diff --git a/backoffice/backoffice/authors/airflow_utils.py b/backoffice/backoffice/authors/airflow_utils.py index d389078ceb..c6b1fd0e75 100644 --- a/backoffice/backoffice/authors/airflow_utils.py +++ b/backoffice/backoffice/authors/airflow_utils.py @@ -159,15 +159,18 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): :param workflow_id: workflow_id for dags that should be restarted :param workflow_type: type of workflow the will be restarted - :param params: parameters of new dag execution + :param params: parameters of new dag execution, if not provided will be fetched from the workflow :returns: request response """ + conf = params if params else fetch_conf_workflow_dag(workflow_id, workflow_type) - data = fetch_data_workflow_dag(workflow_id, workflow_type) delete_workflow_dag_runs(workflow_id, workflow_type) return trigger_airflow_dag( - WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data + WORKFLOW_DAGS[workflow_type].initialize, + str(workflow_id), + workflow=conf.get("workflow"), + extra_data=conf.get("data"), ) @@ -183,7 +186,7 @@ def delete_workflow_dag_runs(workflow_id, workflow_type): delete_workflow_dag(dag_id, str(workflow_id)) -def fetch_data_workflow_dag(workflow_id, workflow_type): +def fetch_conf_workflow_dag(workflow_id, workflow_type): """Fetches Data that the workflow ran with :param workflow_id: workflow_id for dag to get data of @@ -194,4 +197,4 @@ def fetch_data_workflow_dag(workflow_id, workflow_type): executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) _, dag = next(iter(executed_dags_for_workflow.items())) - return dag["conf"].get("data") + return dag["conf"] diff --git a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_data_workflow_dag.yaml b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_conf_workflow_dag.yaml similarity index 70% rename from backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_data_workflow_dag.yaml rename to backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_conf_workflow_dag.yaml index 5e3295de8d..893d67e066 100644 --- a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_data_workflow_dag.yaml +++ b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_fetch_conf_workflow_dag.yaml @@ -1,7 +1,8 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}' + "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow": + {"id": "id"}}}' headers: Accept: - '*/*' @@ -10,7 +11,7 @@ interactions: Connection: - keep-alive Content-Length: - - '145' + - '171' Content-Type: - application/json method: POST @@ -18,25 +19,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:25.361392+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:25.361392+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:25.361392+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:25.361392+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T11:02:49.477410+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T11:02:49.477410+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T11:02:49.477410+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T11:02:49.477410+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:25 GMT + - Fri, 20 Dec 2024 11:02:49 GMT Server: - gunicorn X-Robots-Tag: @@ -58,25 +59,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:25.361392+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:25.361392+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:25.361392+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:25.361392+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T11:02:49.477410+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T11:02:49.477410+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T11:02:49.477410+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T11:02:49.477410+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:25 GMT + - Fri, 20 Dec 2024 11:02:49 GMT Server: - gunicorn X-Robots-Tag: @@ -110,7 +111,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:25 GMT + - Fri, 20 Dec 2024 11:02:49 GMT Server: - gunicorn X-Robots-Tag: @@ -144,7 +145,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:25 GMT + - Fri, 20 Dec 2024 11:02:49 GMT Server: - gunicorn X-Robots-Tag: @@ -176,7 +177,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:25 GMT + - Fri, 20 Dec 2024 11:02:49 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml index 6527a4c03e..389fd0eb11 100644 --- a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml +++ b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml @@ -1,7 +1,8 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}' + "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow": + {"id": "id"}}}' headers: Accept: - '*/*' @@ -10,7 +11,7 @@ interactions: Connection: - keep-alive Content-Length: - - '145' + - '171' Content-Type: - application/json method: POST @@ -18,25 +19,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:49 GMT + - Fri, 20 Dec 2024 10:59:09 GMT Server: - gunicorn X-Robots-Tag: @@ -58,25 +59,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:09 GMT Server: - gunicorn X-Robots-Tag: @@ -110,7 +111,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:09 GMT Server: - gunicorn X-Robots-Tag: @@ -144,7 +145,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:09 GMT Server: - gunicorn X-Robots-Tag: @@ -166,25 +167,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:49.928543+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:49.928543+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:49.928543+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T10:59:09.785487+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T10:59:09.785487+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T10:59:09.785487+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: @@ -218,7 +219,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: @@ -252,7 +253,7 @@ interactions: Content-Type: - application/problem+json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: @@ -284,7 +285,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: @@ -294,7 +295,8 @@ interactions: message: NO CONTENT - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}' + "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow": + {"id": "id"}}}' headers: Accept: - '*/*' @@ -303,7 +305,7 @@ interactions: Connection: - keep-alive Content-Length: - - '145' + - '171' Content-Type: - application/json method: POST @@ -311,25 +313,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:50.810267+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:50.810267+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:50.810267+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:50.810267+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-12-20T10:59:10.350095+00:00\",\n + \ \"data_interval_start\": \"2024-12-20T10:59:10.350095+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-12-20T10:59:10.350095+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-12-20T10:59:10.350095+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: @@ -361,7 +363,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:50 GMT + - Fri, 20 Dec 2024 10:59:10 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/authors/tests/test_airflow_utils.py b/backoffice/backoffice/authors/tests/test_airflow_utils.py index caca9fe8e6..ed67c5f860 100644 --- a/backoffice/backoffice/authors/tests/test_airflow_utils.py +++ b/backoffice/backoffice/authors/tests/test_airflow_utils.py @@ -17,8 +17,8 @@ def setUp(self): self.response = airflow_utils.trigger_airflow_dag( self.dag_id, str(self.workflow_id), - self.extra_data, - self.workflow_serialized, + extra_data=self.extra_data, + workflow=self.workflow_serialized, ) def tearDown(self): @@ -69,15 +69,19 @@ def test_restart_workflow_dags(self): self.workflow_id, self.workflow_type ) self.assertEqual(response.status_code, 200) + json_content = json.loads(response.content) + self.assertEqual(json_content["conf"]["data"], self.extra_data) + self.assertEqual(json_content["conf"]["workflow"], self.workflow_serialized) @pytest.mark.vcr def test_delete_workflow_dag_runs(self): airflow_utils.delete_workflow_dag_runs(self.workflow_id, self.workflow_type) @pytest.mark.vcr - def test_fetch_data_workflow_dag(self): - result = airflow_utils.fetch_data_workflow_dag( + def test_fetch_conf_workflow_dag(self): + result = airflow_utils.fetch_conf_workflow_dag( self.workflow_id, self.workflow_type ) - self.assertEqual(result, {"test": "test"}) + self.assertEqual(result["data"], {"test": "test"}) + self.assertEqual(result["workflow"], {"id": "id"})