try inline config

govuk-one-login · Jan 10, 2025 · 8b36c2f · 8b36c2f
1 parent 0c914af
commit 8b36c2f
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/iac/main/resources/raw.yml b/iac/main/resources/raw.yml
@@ -179,11 +179,18 @@ RawLayerTestCrawler:
         - Path: !Sub 's3://${RawLayerBucket}/txma/'
     DatabaseName: !Ref RawGlueDatabase
     CrawlerSecurityConfiguration: !Ref GlueSecurityConfig
+    PartitionKeys:
+      - Name: year
+        Type: string
+      - Name: month
+        Type: string
+      - Name: day
+        Type: string
+    TableName: txma-poc
     RecrawlPolicy:
       RecrawlBehavior: CRAWL_EVERYTHING
     SchemaChangePolicy:
       UpdateBehavior: UPDATE_IN_DATABASE
-      DeleteBehavior: DELETE_FROM_DATABASE
 
 TxmaCrawlerInvoke:
   # checkov:skip=CKV_AWS_116: DLQ not needed as this lambda failing will cause state machine to fail

diff --git a/src/handlers/trigger-txma-crawler/handler.ts b/src/handlers/trigger-txma-crawler/handler.ts
@@ -17,6 +17,25 @@ export const handler = async (): Promise<unknown> => {
         Path: path,
       })),
     },
+    Configuration: JSON.stringify({
+      Version: 1.0,
+      CrawlerOutput: {
+        Partitions: {
+          AddOrUpdateBehavior: 'InheritFromTable',
+        },
+        TableLevelConfiguration: [
+          {
+            DatabaseName: 'dev-txma-raw',
+            TableName: 'txma-poc',
+            Path: 's3://dev-txma-raw/txma/',
+          },
+        ],
+      },
+    }),
+    TablePrefix: '',
+    RecrawlPolicy: {
+      RecrawlBehavior: 'CRAWL_NEW_FOLDERS_ONLY',
+    },
   });
   try {
     const response = await glueClient.send(updateCrawlerCommand);