update flyway

govuk-one-login · Oct 24, 2024 · c5dde91 · c5dde91
1 parent dcef861
commit c5dde91
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 38 deletions.
diff --git a/athena-scripts/raw_stage_optimisation_solution/scripts/core_preprocessing_functions.py b/athena-scripts/raw_stage_optimisation_solution/scripts/core_preprocessing_functions.py
@@ -37,8 +37,7 @@ def get_max_timestamp(app, stage_database, stage_target_table):
                             raise Exception("Stage table does not contain the timestamp column.")
 
         else:
-            raise Exception("Error returned querying the raw table for the min(year,month,day) value.")
-
+            return 0
     except Exception as e:
         print(f"Exception Error retrieving max timestamp: {str(e)}")
         return None

diff --git a/athena-scripts/raw_stage_optimisation_solution/scripts/raw_to_stage_process_glue_job.py b/athena-scripts/raw_stage_optimisation_solution/scripts/raw_to_stage_process_glue_job.py
@@ -112,16 +112,16 @@ def main():
             raise ValueError("Function 'get_max_timestamp' returned None, which is not allowed.")
         print(f'retrieved timestamp filter value: {filter_timestamp}')
 
-        adjusted_filter_processed_dt = adjust_with_buffer(filter_processed_dt, 1)
-        adjusted_filter_timestamp = adjust_with_buffer(filter_timestamp, 20 * 60)
+        # adjusted_filter_processed_dt = adjust_with_buffer(filter_processed_dt, 1)
+        # adjusted_filter_timestamp = adjust_with_buffer(filter_timestamp, 20 * 60)
 
 
         # Generate the raw data select criteria
         raw_sql_select = generate_raw_select_filter(json_data, 
                                                     args['raw_database'], 
                                                     args['raw_source_table'],
-                                                    adjusted_filter_processed_dt,
-                                                    adjusted_filter_timestamp)
+                                                    filter_processed_dt,
+                                                    filter_timestamp)
 
         if raw_sql_select is None:
             raise ValueError("Function 'generate_raw_select_filter' returned None, which is not allowed.")

diff --git a/...ft-scripts/flyway/migrations/dap_txma_reporting_db_refactored/R__grants_for_procedure.sql b/...ft-scripts/flyway/migrations/dap_txma_reporting_db_refactored/R__grants_for_procedure.sql
@@ -1,13 +1,13 @@
-GRANT EXECUTE ON procedure conformed_refactored.update_dap_data_mart() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.dim_event_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.dim_journey_channel_refactored_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.dim_relying_party_refactored_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.dim_user_journey_event_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.dim_user_refactored_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.event_extensions_refactored_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.fact_user_journey_event_refactored_upsert() TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.redshift_date_dim(VARCHAR,VARCHAR) TO "IAMR:production-dap-redshift-processing-role";
-GRANT EXECUTE ON procedure conformed_refactored.update_event_batch_table() TO "IAMR:production-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.update_dap_data_mart() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.dim_event_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.dim_journey_channel_refactored_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.dim_relying_party_refactored_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.dim_user_journey_event_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.dim_user_refactored_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.event_extensions_refactored_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.fact_user_journey_event_refactored_upsert() TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.redshift_date_dim(VARCHAR,VARCHAR) TO "IAMR:dev-dap-redshift-processing-role";
+GRANT EXECUTE ON procedure conformed_refactored.update_event_batch_table() TO "IAMR:dev-dap-redshift-processing-role";
 
 GRANT ALL ON DATABASE "dap_txma_reporting_db_refactored" TO GROUP dap_elt_processing;
 GRANT ALL ON SCHEMA "conformed_refactored" TO GROUP dap_elt_processing;
@@ -21,4 +21,4 @@ Database object privileges to group
 
 GRANT ALL ON DATABASE "dap_txma_reporting_db_refactored" TO GROUP dap_elt_processing;
 GRANT ALL ON SCHEMA "audit_refactored" TO GROUP dap_elt_processing;
-GRANT ALL ON ALL TABLES IN SCHEMA "audit_refactored" TO GROUP dap_elt_processing;
+GRANT ALL ON ALL TABLES IN SCHEMA "audit_refactored" TO GROUP dap_elt_processing;
diff --git a/redshift-scripts/setup_process/redshift_setup_process_scripts.sql b/redshift-scripts/setup_process/redshift_setup_process_scripts.sql
@@ -30,9 +30,9 @@ ii. IAM_ROLE
 --**REPLACE {aws-account-id}**
 CREATE EXTERNAL SCHEMA IF NOT EXISTS dap_txma_stage
 FROM DATA CATALOG
-DATABASE '{env}-txma-stage'
+DATABASE 'dev-txma-stage'
 REGION 'eu-west-2'
-IAM_ROLE 'arn:aws:iam::{aws-account-id}:role/{env}-redshift-serverless-role';
+IAM_ROLE 'arn:aws:iam::563887642259:role/dev-redshift-serverless-role';
 
 
 /*
@@ -67,7 +67,7 @@ copy the contents of the file: redshift-scripts/setup_process/sp_conformed_date_
 -- paste into the redshift query editor
 
 
-click [Run] button to create the stored procedure: conformed.redshift_date_dim
+-- click [Run] button to create the stored procedure: conformed.redshift_date_dim
 
 -- run the following cmd once confirmed SP has been created
 -- passing in the start, end dates for the date range to populate
@@ -102,7 +102,7 @@ Create IAM user (used by the Redshift Step Function)
 */
 
 --**REPLACE {env}**
-CREATE USER "IAMR:{env}-dap-redshift-processing-role" PASSWORD DISABLE;
+CREATE USER "IAMR:dev-dap-redshift-processing-role" PASSWORD DISABLE;
 
 
 
@@ -216,16 +216,16 @@ User association to group
 */
 
 --**REPLACE {env}**
-ALTER GROUP dap_elt_processing ADD USER "IAMR:{env}-dap-redshift-processing-role";
+ALTER GROUP dap_elt_processing ADD USER "IAMR:dev-dap-redshift-processing-role";
 
 
 /*
 Alter table ownership
 */
 
 --**REPLACE {env}**
-ALTER TABLE dap_txma_reporting_db.conformed.ref_events OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.conformed.ref_relying_parties OWNER TO "IAMR:{env}-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.conformed.ref_events OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.conformed.ref_relying_parties OWNER TO "IAMR:dev-dap-redshift-processing-role";
 
 
 /*
@@ -234,20 +234,20 @@ Alter table ownership
 
 --**REPLACE {env}**
 
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_creation_1 OWNER TO "IAMR:{env}-dap-redshift-processing-role";	
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_orchestration_2	OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_user_login_3 OWNER TO "IAMR:{env}-dap-redshift-processing-role";	
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_dcmaw_cri_4 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_mfa_5 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_management_6 OWNER TO "IAMR:{env}-dap-redshift-processing-role";	
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_address_7 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_driving_license_8 OWNER TO "IAMR:{env}-dap-redshift-processing-role";	
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_fraud_9 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_journey_10 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_kbv_11 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_passport_12 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_cic_13 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
-ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_f2f_14 OWNER TO "IAMR:{env}-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_creation_1 OWNER TO "IAMR:dev-dap-redshift-processing-role";	
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_orchestration_2	OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_user_login_3 OWNER TO "IAMR:dev-dap-redshift-processing-role";	
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_dcmaw_cri_4 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_mfa_5 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_auth_account_management_6 OWNER TO "IAMR:dev-dap-redshift-processing-role";	
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_address_7 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_driving_license_8 OWNER TO "IAMR:dev-dap-redshift-processing-role";	
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_fraud_9 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_journey_10 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_kbv_11 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_passport_12 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_cic_13 OWNER TO "IAMR:dev-dap-redshift-processing-role";
+ALTER TABLE dap_txma_reporting_db.audit.err_duplicate_event_id_ipv_cri_f2f_14 OWNER TO "IAMR:dev-dap-redshift-processing-role";
 
 -- run the following cmds once confirmed SPs has been created