diff --git a/caluma/caluma_form/domain_logic.py b/caluma/caluma_form/domain_logic.py index 25a377433..83c255618 100644 --- a/caluma/caluma_form/domain_logic.py +++ b/caluma/caluma_form/domain_logic.py @@ -1,3 +1,4 @@ +from graphlib import TopologicalSorter from typing import Optional from django.db import transaction @@ -6,8 +7,8 @@ from caluma.caluma_core.models import BaseModel from caluma.caluma_core.relay import extract_global_id -from caluma.caluma_form import models, validators -from caluma.caluma_form.utils import recalculate_answers_from_document +from caluma.caluma_form import models, structure, utils, validators +from caluma.caluma_form.utils import update_or_create_calc_answer from caluma.caluma_user.models import BaseUser from caluma.utils import update_model @@ -151,6 +152,19 @@ def post_save(answer: models.Answer) -> models.Answer: # TODO emit events return answer + @staticmethod + def update_calc_dependents(answer): + if not answer.question.calc_dependents: + return + + root_doc = utils.prefetch_document(answer.document.family_id) + struc = structure.FieldSet(root_doc, root_doc.form) + + for question in models.Question.objects.filter( + pk__in=answer.question.calc_dependents + ): + update_or_create_calc_answer(question, root_doc, struc) + @classmethod @transaction.atomic def create( @@ -168,6 +182,8 @@ def create( if answer.question.type == models.Question.TYPE_TABLE: answer.create_answer_documents(documents) + cls.update_calc_dependents(answer) + return answer @classmethod @@ -185,6 +201,8 @@ def update(cls, answer, validated_data, user: Optional[BaseUser] = None): if answer.question.type == models.Question.TYPE_TABLE: answer.create_answer_documents(documents) + cls.update_calc_dependents(answer) + answer.refresh_from_db() return answer @@ -276,7 +294,44 @@ def create( document.meta.pop("_defer_calculation", None) document.save() - recalculate_answers_from_document(document) + SaveDocumentLogic._initialize_calculated_answers(document) + + return document + + @staticmethod + def _initialize_calculated_answers(document): + """ + Initialize all calculated questions in the document. + + In order to do this efficiently, we get all calculated questions with their dependents, + sort them topoligically, and then update their answer. + """ + root_doc = utils.prefetch_document(document.family_id) + struc = structure.FieldSet(root_doc, root_doc.form) + + calculated_questions = ( + models.Form.get_all_questions([(document.family or document).form_id]) + .filter(type=models.Question.TYPE_CALCULATED_FLOAT) + .values("slug", "calc_dependents") + ) + adjacency_list = { + dep["slug"]: dep["calc_dependents"] for dep in calculated_questions + } + ts = TopologicalSorter(adjacency_list) + # TopologicalSorter expects the adjacency_list the "other way around", i.e. + # for every node the incoming nodes should be given. To account for this, we + # just reverse the resulting order. + sorted_question_slugs = list(reversed(list(ts.static_order()))) + + # fetch all related questions in one query, but iterate according + # to pre-established sorting + _questions = models.Question.objects.in_bulk(sorted_question_slugs) + for slug in sorted_question_slugs: + print("question", slug) + update_or_create_calc_answer( + _questions[slug], document, struc, update_dependents=False + ) + return document @staticmethod diff --git a/caluma/caluma_form/models.py b/caluma/caluma_form/models.py index fe695a924..f1bd59b8e 100644 --- a/caluma/caluma_form/models.py +++ b/caluma/caluma_form/models.py @@ -396,9 +396,9 @@ def set_family(self, root_doc): def copy(self, family=None, user=None): """Create a copy including all its answers.""" - from caluma.caluma_form.utils import recalculate_answers_from_document - # defer calculated questions, as many unecessary recomputations will happen otherwise + # no need to update calculated questions, since calculated answers + # are copied as well meta = dict(self.meta) meta["_defer_calculation"] = True @@ -423,7 +423,6 @@ def copy(self, family=None, user=None): new_document.meta.pop("_defer_calculation", None) new_document.save() - recalculate_answers_from_document(new_document) return new_document diff --git a/caluma/caluma_form/signals.py b/caluma/caluma_form/signals.py index 805aaa56e..df79414b8 100644 --- a/caluma/caluma_form/signals.py +++ b/caluma/caluma_form/signals.py @@ -1,7 +1,4 @@ -import itertools - from django.db.models.signals import ( - m2m_changed, post_delete, post_init, post_save, @@ -15,11 +12,7 @@ from caluma.utils import disable_raw from . import models -from .utils import ( - recalculate_answers_from_document, - update_calc_dependents, - update_or_create_calc_answer, -) +from .utils import update_calc_dependents, update_or_create_calc_answer @receiver(pre_create_historical_record, sender=models.HistoricalAnswer) @@ -78,6 +71,7 @@ def save_calc_dependents(sender, instance, **kwargs): update_calc_dependents( instance.slug, old_expr="false", new_expr=instance.calc_expression ) + instance.calc_expression_changed = True elif original.calc_expression != instance.calc_expression: update_calc_dependents( @@ -85,6 +79,9 @@ def save_calc_dependents(sender, instance, **kwargs): old_expr=original.calc_expression, new_expr=instance.calc_expression, ) + instance.calc_expression_changed = True + else: + instance.calc_expression_changed = False @receiver(pre_delete, sender=models.Question) @@ -104,10 +101,8 @@ def remove_calc_dependents(sender, instance, **kwargs): @receiver(post_save, sender=models.Question) @disable_raw @filter_events(lambda instance: instance.type == models.Question.TYPE_CALCULATED_FLOAT) +@filter_events(lambda instance: getattr(instance, "calc_expression_changed", False)) def update_calc_from_question(sender, instance, created, update_fields, **kwargs): - # TODO optimize: only update answers if calc_expression is updated - # needs to happen during save() or __init__() - for document in models.Document.objects.filter(form__questions=instance): update_or_create_calc_answer(instance, document) @@ -120,40 +115,3 @@ def update_calc_from_question(sender, instance, created, update_fields, **kwargs def update_calc_from_form_question(sender, instance, created, **kwargs): for document in instance.form.documents.all(): update_or_create_calc_answer(instance.question, document) - - -@receiver(post_save, sender=models.Answer) -@disable_raw -@filter_events(lambda instance: instance.document and instance.question.calc_dependents) -def update_calc_from_answer(sender, instance, **kwargs): - # If there is no document on the answer it means that it's a default - # answer. They shouldn't trigger a recalculation of a calculated field - # even when they are technically listed as a dependency. - # Also skip non-referenced answers. - if instance.document.family.meta.get("_defer_calculation"): - return - - for question in models.Question.objects.filter( - pk__in=instance.question.calc_dependents - ): - update_or_create_calc_answer(question, instance.document) - - -@receiver(post_save, sender=models.Document) -@disable_raw -# We're only interested in table row forms -@filter_events(lambda instance, created: instance.pk != instance.family_id or created) -def update_calc_from_document(sender, instance, created, **kwargs): - recalculate_answers_from_document(instance) - - -@receiver(m2m_changed, sender=models.AnswerDocument) -def update_calc_from_answerdocument(sender, instance, **kwargs): - dependents = instance.document.form.questions.exclude( - calc_dependents=[] - ).values_list("calc_dependents", flat=True) - - dependent_questions = list(itertools.chain(*dependents)) - - for question in models.Question.objects.filter(pk__in=dependent_questions): - update_or_create_calc_answer(question, instance.document) diff --git a/caluma/caluma_form/structure.py b/caluma/caluma_form/structure.py index 4e1853474..687b1b090 100644 --- a/caluma/caluma_form/structure.py +++ b/caluma/caluma_form/structure.py @@ -254,6 +254,11 @@ def children(self): for question in self.form.questions.all() ] + def set_answer(self, question_slug, answer): + field = self.get_field(question_slug) + if field: + field.answer = answer + def __repr__(self): q_slug = self.question.slug if self.question else None if q_slug: diff --git a/caluma/caluma_form/tests/test_document.py b/caluma/caluma_form/tests/test_document.py index 4aed01b32..109a2b96e 100644 --- a/caluma/caluma_form/tests/test_document.py +++ b/caluma/caluma_form/tests/test_document.py @@ -282,9 +282,6 @@ def test_complex_document_query_performance( """ with django_assert_num_queries(10): - # TODO: This used to be 7 queries with graphene 3.0.0b7. - # it seems that `Form` is queried that wasn't before, and - # some question options as well. result = schema_executor(query, variable_values={"id": str(document.pk)}) assert not result.errors @@ -466,10 +463,8 @@ def test_save_document( # if updating, the resulting document must be the same assert same_id == update - assert ( - len(result.data["saveDocument"]["document"]["answers"]["edges"]) == 1 - if update - else 3 + assert len(result.data["saveDocument"]["document"]["answers"]["edges"]) == ( + 1 if update else 3 ) if not update: assert sorted( @@ -492,7 +487,7 @@ def test_save_document( ) assert (doc.pk == document.pk) == update - assert doc.answers.count() == 1 if update else 3 + assert doc.answers.count() == (1 if update else 3) if not update: assert sorted([str(a.value) for a in doc.answers.iterator()]) == [ "23", @@ -511,269 +506,35 @@ def test_save_document( @pytest.mark.parametrize( "question__type,question__configuration,question__data_source,question__format_validators,answer__value,answer__date,mutation,success", [ - ( - Question.TYPE_INTEGER, - {}, - None, - [], - 1, - None, - "SaveDocumentIntegerAnswer", - True, - ), - ( - Question.TYPE_INTEGER, - {"min_value": 100}, - None, - [], - 1, - None, - "SaveDocumentIntegerAnswer", - False, - ), + (Question.TYPE_INTEGER, {}, None, [], 1, None, "SaveDocumentIntegerAnswer", True), + (Question.TYPE_INTEGER, {"min_value": 100}, None, [], 1, None, "SaveDocumentIntegerAnswer", False), (Question.TYPE_FLOAT, {}, None, [], 2.1, None, "SaveDocumentFloatAnswer", True), - ( - Question.TYPE_FLOAT, - {"min_value": 100.0}, - None, - [], - 1, - None, - "SaveDocumentFloatAnswer", - False, - ), - ( - Question.TYPE_TEXT, - {}, - None, - [], - "Test", - None, - "SaveDocumentStringAnswer", - True, - ), - ( - Question.TYPE_TEXT, - {"max_length": 1}, - None, - [], - "toolong", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_DATE, - {}, - None, - [], - None, - "1900-01-01", - "SaveDocumentDateAnswer", - False, - ), - ( - Question.TYPE_DATE, - {}, - None, - [], - None, - "2019-02-22", - "SaveDocumentDateAnswer", - True, - ), - ( - Question.TYPE_FILES, - {}, - None, - [], - None, - None, - "SaveDocumentFilesAnswer", - False, - ), - ( - Question.TYPE_FILES, - {}, - None, - [], - [{"name": "some-file.pdf"}], - None, - "SaveDocumentFilesAnswer", - True, - ), - ( - Question.TYPE_FILES, - {}, - None, - [], - [{"name": "not-exist.pdf"}], - None, - "SaveDocumentFilesAnswer", - True, - ), - ( - Question.TYPE_TEXT, - {"min_length": 10}, - None, - [], - "tooshort", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_TABLE, - {}, - None, - [], - None, - None, - "SaveDocumentTableAnswer", - True, - ), - ( - Question.TYPE_TEXTAREA, - {}, - None, - [], - "Test", - None, - "SaveDocumentStringAnswer", - True, - ), - ( - Question.TYPE_TEXTAREA, - {"max_length": 1}, - None, - [], - "toolong", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_MULTIPLE_CHOICE, - {}, - None, - [], - ["option-slug"], - None, - "SaveDocumentListAnswer", - True, - ), - ( - Question.TYPE_MULTIPLE_CHOICE, - {}, - None, - [], - ["option-slug", "option-invalid-slug"], - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_CHOICE, - {}, - None, - [], - "option-slug", - None, - "SaveDocumentStringAnswer", - True, - ), - ( - Question.TYPE_CHOICE, - {}, - None, - [], - "invalid-option-slug", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_DYNAMIC_MULTIPLE_CHOICE, - {}, - "MyDataSource", - [], - ["5.5", "1"], - None, - "SaveDocumentListAnswer", - True, - ), - ( - Question.TYPE_DYNAMIC_MULTIPLE_CHOICE, - {}, - "MyDataSource", - [], - ["not in data"], - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_DYNAMIC_CHOICE, - {}, - "MyDataSource", - [], - "5.5", - None, - "SaveDocumentStringAnswer", - True, - ), - ( - Question.TYPE_DYNAMIC_CHOICE, - {}, - "MyDataSource", - [], - "not in data", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_TEXT, - {}, - None, - ["email"], - "some text", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_TEXT, - {}, - None, - ["email"], - "test@example.com", - None, - "SaveDocumentStringAnswer", - True, - ), - ( - Question.TYPE_TEXTAREA, - {}, - None, - ["email"], - "some text", - None, - "SaveDocumentStringAnswer", - False, - ), - ( - Question.TYPE_TEXTAREA, - {}, - None, - ["email"], - "test@example.com", - None, - "SaveDocumentStringAnswer", - True, - ), + (Question.TYPE_FLOAT, {"min_value": 100.0}, None, [], 1, None, "SaveDocumentFloatAnswer", False), + (Question.TYPE_TEXT, {}, None, [], "Test", None, "SaveDocumentStringAnswer", True), + (Question.TYPE_TEXT, {"max_length": 1}, None, [], "toolong", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_DATE, {}, None, [], None, "1900-01-01", "SaveDocumentDateAnswer", False), + (Question.TYPE_DATE, {}, None, [], None, "2019-02-22", "SaveDocumentDateAnswer", True), + (Question.TYPE_FILES, {}, None, [], None, None, "SaveDocumentFilesAnswer", False), + (Question.TYPE_FILES, {}, None, [], [{"name": "some-file.pdf"}], None, "SaveDocumentFilesAnswer", True), + (Question.TYPE_FILES, {}, None, [], [{"name": "not-exist.pdf"}], None, "SaveDocumentFilesAnswer", True), + (Question.TYPE_TEXT, {"min_length": 10}, None, [], "tooshort", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_TABLE, {}, None, [], None, None, "SaveDocumentTableAnswer", True), + (Question.TYPE_TEXTAREA, {}, None, [], "Test", None, "SaveDocumentStringAnswer", True), + (Question.TYPE_TEXTAREA, {"max_length": 1}, None, [], "toolong", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_MULTIPLE_CHOICE, {}, None, [], ["option-slug"], None, "SaveDocumentListAnswer", True), + (Question.TYPE_MULTIPLE_CHOICE, {}, None, [], ["option-slug", "option-invalid-slug"], None, "SaveDocumentStringAnswer", False), + (Question.TYPE_CHOICE, {}, None, [], "option-slug", None, "SaveDocumentStringAnswer", True), + (Question.TYPE_CHOICE, {}, None, [], "invalid-option-slug", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_DYNAMIC_MULTIPLE_CHOICE, {}, "MyDataSource", [], ["5.5", "1"], None, "SaveDocumentListAnswer", True), + (Question.TYPE_DYNAMIC_MULTIPLE_CHOICE, {}, "MyDataSource", [], ["not in data"], None, "SaveDocumentStringAnswer", False), + (Question.TYPE_DYNAMIC_CHOICE, {}, "MyDataSource", [], "5.5", None, "SaveDocumentStringAnswer", True), + (Question.TYPE_DYNAMIC_CHOICE, {}, "MyDataSource", [], "not in data", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_TEXT, {}, None, ["email"], "some text", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_TEXT, {}, None, ["email"], "test@example.com", None, "SaveDocumentStringAnswer", True), + (Question.TYPE_TEXTAREA, {}, None, ["email"], "some text", None, "SaveDocumentStringAnswer", False), + (Question.TYPE_TEXTAREA, {}, None, ["email"], "test@example.com", None, "SaveDocumentStringAnswer", True), ], -) +) # fmt:skip def test_save_document_answer( # noqa:C901 db, snapshot, @@ -1755,3 +1516,30 @@ def test_flat_answer_map(db, form_and_document): assert flat_answer_map["table"] == [ {"column": answers_dict["table"].documents.first().answers.first().value} ] + + +def test_efficient_init_of_calc_questions( + db, schema_executor, form, form_question_factory, question_factory, mocker +): + calc_1 = question_factory( + slug="calc-1", + type=Question.TYPE_CALCULATED_FLOAT, + calc_expression="1", + ) + calc_2 = question_factory( + slug="calc-2", + type=Question.TYPE_CALCULATED_FLOAT, + calc_expression='"calc-1"|answer(0) * 2', + ) + form_question_factory(form=form, question=calc_1) + form_question_factory(form=form, question=calc_2) + + from caluma.caluma_form.jexl import QuestionJexl + + spy = mocker.spy(QuestionJexl, "evaluate") + document = api.save_document(form) + # twice for calc value, once for hidden state of calc-1 + assert spy.call_count == 3 + + calc_ans = document.answers.get(question_id="calc-2") + assert calc_ans.value == 2 diff --git a/caluma/caluma_form/tests/test_question.py b/caluma/caluma_form/tests/test_question.py index 7f8a1a974..921a56f89 100644 --- a/caluma/caluma_form/tests/test_question.py +++ b/caluma/caluma_form/tests/test_question.py @@ -8,6 +8,7 @@ ) from .. import api, models, serializers +from ..jexl import QuestionJexl @pytest.mark.parametrize( @@ -1057,9 +1058,6 @@ def test_nested_calculated_question( question__calc_expression=expr, ) - calc_ans = document.answers.get(question_id="calc") - assert calc_ans.value == expected - for dep in calc_deps: questions[dep].refresh_from_db() assert questions[dep].calc_dependents == ["calc"] @@ -1110,7 +1108,7 @@ def test_recursive_calculated_question( def test_calculated_question_update_calc_expr( - db, schema_executor, form_and_document, form_question_factory + db, schema_executor, form_and_document, form_question_factory, mocker ): form, document, questions_dict, answers_dict = form_and_document(True, True) @@ -1130,12 +1128,20 @@ def test_calculated_question_update_calc_expr( calc_ans = document.answers.get(question_id="calc_question") assert calc_ans.value == 101 - + # spying on update_or_create_calc_answer doesn't seem to work, so we spy on QuestionJexl.evaluate instead + spy = mocker.spy(QuestionJexl, "evaluate") calc_question.calc_expression = "'sub_question'|answer -1" calc_question.save() + assert spy.call_count > 0 + call_count = spy.call_count calc_ans.refresh_from_db() assert calc_ans.value == 99 + calc_question.label = "New Label" + calc_question.save() + # if the calc expression is not changed, no jexl evaluation should be done + assert spy.call_count == call_count + def test_calculated_question_answer_document( db, @@ -1165,19 +1171,24 @@ def test_calculated_question_answer_document( question__calc_expression="'table'|answer|mapby('column')[0] + 'table'|answer|mapby('column')[1]", ).question - calc_ans = document.answers.get(question_id="calc_question") - assert calc_ans.value is None - # adding another row will make make the expression valid row_doc = document_factory(form=row_form, family=document) column_a2 = answer_factory(document=row_doc, question_id=column.slug, value=200) - table_a.documents.add(row_doc) - calc_ans.refresh_from_db() + api.save_answer( + question=table, + document=document, + documents=list(table_a.documents.all()) + [row_doc], + ) + + calc_ans = document.answers.get(question_id="calc_question") assert calc_ans.value == 300 - column_a2.value = 100 - column_a2.save() + api.save_answer( + question=column_a2.question, + document=row_doc, + value=100, + ) calc_ans.refresh_from_db() column.refresh_from_db() assert column.calc_dependents == ["calc_question"] @@ -1185,7 +1196,11 @@ def test_calculated_question_answer_document( # removing the row will make it invalid again table_a.documents.remove(row_doc) - row_doc.delete() + api.save_answer( + question=table, + document=document, + documents=[table_a.documents.first()], + ) calc_ans.refresh_from_db() assert calc_ans.value is None @@ -1220,3 +1235,25 @@ def test_save_action_button_question(db, snapshot, question, schema_executor): result = schema_executor(query, variable_values=inp) assert not bool(result.errors) snapshot.assert_match(result.data) + + +def test_init_of_calc_questions_queries( + db, + form, + form_and_document, + form_question_factory, + django_assert_num_queries, +): + (form, document, questions_dict, _) = form_and_document( + use_table=True, use_subform=True, table_row_count=10 + ) + + form_question_factory( + form=form, + question__slug="calc_question", + question__type=models.Question.TYPE_CALCULATED_FLOAT, + question__calc_expression="'table'|answer|mapby('column')|sum + 'top_question'|answer + 'sub_question'|answer", + ) + + with django_assert_num_queries(35): + api.save_answer(questions_dict["top_question"], document, value="1") diff --git a/caluma/caluma_form/tests/test_recalculate_calc_answers_command.py b/caluma/caluma_form/tests/test_recalculate_calc_answers_command.py index 6b0a3e1ae..1102daceb 100644 --- a/caluma/caluma_form/tests/test_recalculate_calc_answers_command.py +++ b/caluma/caluma_form/tests/test_recalculate_calc_answers_command.py @@ -57,6 +57,8 @@ def test_recalculate_calc_answers( save_answer(question=dep1_main, value=10, document=main_doc) row_doc = save_document(form=row_form) save_answer(question=dep2_row, value=13, document=row_doc) + # make sure table questions' calc dependents are up to date + table_question.refresh_from_db() save_answer(table_question, document=main_doc, value=[str(row_doc.pk)]) calc_answer = main_doc.answers.get(question_id="calc_question") diff --git a/caluma/caluma_form/tests/test_validators.py b/caluma/caluma_form/tests/test_validators.py index 939c9131b..c353f14cc 100644 --- a/caluma/caluma_form/tests/test_validators.py +++ b/caluma/caluma_form/tests/test_validators.py @@ -3,6 +3,7 @@ from rest_framework.exceptions import ValidationError from ...caluma_core.tests import extract_serializer_input_fields +from ...caluma_form import api from ...caluma_form.models import DynamicOption, Question from .. import serializers, structure from ..jexl import QuestionMissing @@ -59,10 +60,8 @@ def test_validate_special_fields( (Question.TYPE_CALCULATED_FLOAT, "false"), ], ) -def test_validate_calc_fields( - db, form_question, question, document_factory, answer_factory, admin_user -): - document = document_factory(form=form_question.form) +def test_validate_calc_fields(db, form_question, question, admin_user): + document = api.save_document(form=form_question.form) assert document.answers.filter(question_id=question.pk).exists() DocumentValidator().validate(document, admin_user) diff --git a/caluma/caluma_form/utils.py b/caluma/caluma_form/utils.py index b7531e4e5..f7ac560bd 100644 --- a/caluma/caluma_form/utils.py +++ b/caluma/caluma_form/utils.py @@ -1,7 +1,87 @@ +from django.db.models import Prefetch + from caluma.caluma_form import models, structure from caluma.caluma_form.jexl import QuestionJexl +def prefetch_document(document_id): + """Fetch a document while prefetching the entire structure. + + This is needed to reduce the query count when almost all the form data + is needed for a given document, e.g. when recalculating calculated + answers: in order to evaluate calc expressions the complete document + structure is needed, which would otherwise result in a lot of queries. + """ + return ( + models.Document.objects.filter(pk=document_id) + .prefetch_related(*_build_document_prefetch_statements(prefetch_options=True)) + .first() + ) + + +def _build_document_prefetch_statements(prefix="", prefetch_options=False): + question_queryset = models.Question.objects.select_related( + "sub_form", "row_form" + ).order_by("-formquestion__sort") + + if prefetch_options: + question_queryset = question_queryset.prefetch_related( + Prefetch( + "options", + queryset=models.Option.objects.order_by("-questionoption__sort"), + ) + ) + + if prefix: # pragma: no cover + prefix += "__" + + return [ + f"{prefix}answers", + f"{prefix}dynamicoption_set", + Prefetch( + f"{prefix}answers__answerdocument_set", + queryset=models.AnswerDocument.objects.select_related( + "document__form", "document__family" + ) + .prefetch_related("document__answers", "document__form__questions") + .order_by("-sort"), + ), + Prefetch( + # root form -> questions + f"{prefix}form__questions", + queryset=question_queryset.prefetch_related( + Prefetch( + # root form -> row forms -> questions + "row_form__questions", + queryset=question_queryset, + ), + Prefetch( + # root form -> sub forms -> questions + "sub_form__questions", + queryset=question_queryset.prefetch_related( + Prefetch( + # root form -> sub forms -> row forms -> questions + "row_form__questions", + queryset=question_queryset, + ), + Prefetch( + # root form -> sub forms -> sub forms -> questions + "sub_form__questions", + queryset=question_queryset.prefetch_related( + Prefetch( + # root form -> sub forms -> sub forms -> row forms -> questions + "row_form__questions", + queryset=question_queryset, + ), + ), + ), + ), + ), + ), + ), + ] + + def update_calc_dependents(slug, old_expr, new_expr): jexl = QuestionJexl() old_q = set( @@ -28,10 +108,14 @@ def update_calc_dependents(slug, old_expr, new_expr): question.save() -def update_or_create_calc_answer(question, document): +def update_or_create_calc_answer( + question, document, struc=None, update_dependents=True +): root_doc = document.family - struc = structure.FieldSet(root_doc, root_doc.form) + if not struc: + struc = structure.FieldSet(root_doc, root_doc.form) + field = struc.get_field(question.slug) # skip if question doesn't exist in this document structure @@ -47,15 +131,14 @@ def update_or_create_calc_answer(question, document): # be invalid, in which case we return None value = jexl.evaluate(field.question.calc_expression, raise_on_error=False) - models.Answer.objects.update_or_create( + answer, _ = models.Answer.objects.update_or_create( question=question, document=field.document, defaults={"value": value} ) + # also save new answer to structure for reuse + struc.set_answer(question.slug, answer) - -def recalculate_answers_from_document(instance): - if (instance.family or instance).meta.get("_defer_calculation"): - return - for question in models.Form.get_all_questions( - [(instance.family or instance).form_id] - ).filter(type=models.Question.TYPE_CALCULATED_FLOAT): - update_or_create_calc_answer(question, instance) + if update_dependents: + for _question in models.Question.objects.filter( + pk__in=field.question.calc_dependents + ): + update_or_create_calc_answer(_question, document, struc) diff --git a/caluma/conftest.py b/caluma/conftest.py index 4485d84de..6f6c0c51f 100644 --- a/caluma/conftest.py +++ b/caluma/conftest.py @@ -272,7 +272,7 @@ def fallback_factory(factory, **kwargs): return existing return factory(**kwargs) - def factory(use_table=False, use_subform=False): + def factory(use_table=False, use_subform=False, table_row_count=1): form = fallback_factory( form_factory, slug="top_form", meta={"is-top-form": True, "level": 0} ) @@ -324,11 +324,12 @@ def factory(use_table=False, use_subform=False): document=document, question=questions["table"] ) - row_doc = document_factory(form=row_form, family=document) - answers["column"] = answer_factory( - document=row_doc, question=questions["column"] - ) - answers["table"].documents.add(row_doc) + for _ in range(table_row_count): + row_doc = document_factory(form=row_form, family=document) + answers["column"] = answer_factory( + document=row_doc, question=questions["column"] + ) + answers["table"].documents.add(row_doc) if use_subform: sub_form = fallback_factory(