DAC-2372 Implement database migration workflow (#568)

Add run flyway command lambda handler and IaC code
Add run flyway command lambda layer files and IaC code
Make npm build script build ayer using new build-flyway-layer script as well as lambdas
Make checkout actions in deployment tasks in workflows use the lfs flag
Add Git LFS to repository in order to store large flyway tar file and redshift driver
Extract getSecret functionality and RedshiftSecret type into shared
Add tests and test resources for new lambda
Simplify parseS3ResponseAsObject lambda and update athena get config test that uses it

.gitattributes

@@ -0,0 +1,2 @@
+*.jar filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text

.github/workflows/deploy-to-aws.yml

@@ -47,6 +47,8 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+        with:
+          lfs: true
       - name: Node setup
         uses: actions/setup-node@v4
         with:
@@ -87,6 +89,8 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+        with:
+          lfs: true
       - name: Node setup
         uses: actions/setup-node@v4
         with:

.github/workflows/sam-deploy.yml

@@ -62,6 +62,8 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+        with:
+          lfs: true
       - name: Node setup
         uses: actions/setup-node@v4
         with:
@@ -102,6 +104,8 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+        with:
+          lfs: true
       - name: Node setup
         uses: actions/setup-node@v4
         with:

.gitignore

@@ -10,3 +10,4 @@ template.yaml
 reports
 iac-dist/
 .vscode
+layer-dist/

.husky/post-checkout

@@ -0,0 +1,3 @@
+# git-lfs hook
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-checkout "$@"

.husky/post-commit

@@ -0,0 +1,3 @@
+# git-lfs hook
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-commit "$@"

.husky/post-merge

@@ -0,0 +1,3 @@
+# git-lfs hook
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-merge "$@"

.husky/pre-push

@@ -1 +1,5 @@
 npm run test
+
+# git-lfs hook
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs pre-push "$@"

.prettierignore

@@ -8,3 +8,6 @@ test-report/
 .husky/
 *.properties
 Dockerfile
+*.sql
+*.jar
+*.tar.gz

README.md

@@ -30,13 +30,28 @@ You may need to install `gpg` first - on a GDS Mac open the terminal and run `br
 #### Set up husky hooks
 
 [Husky](https://typicode.github.io/husky) is used to run [githooks](https://git-scm.com/docs/githooks), specifically `pre-commit` and `pre-push`.
-To install the hooks run `npm run husky:install`. After this, the hooks defined under the [.husky](.husky) directory will automatically run when you commit or push.
+To install the hooks run `npm run husky:install`. After this, the hooks defined under the [.husky](.husky) directory will automatically run when you commit or push.*
 The [lint-staged](https://github.com/okonet/lint-staged) library is used to only run certain tasks if certain files are modified.
 
 Config can be found in the `lint-staged` block in [package.json](package.json). Note that `lint-staged` works by passing
 a list of the matched staged files to the command defined, which is why the commands in `package.json` are e.g. `prettier --write`, with no file, directory or glob arguments.
 (usually if you wanted to run prettier you would need such an argument, e.g.`prettier --write .` or `prettier --check src`. More information can be found [here](https://github.com/okonet/lint-staged#configuration).
 
+* Git LFS hooks also live in this directory - see section below
+
+#### Set up Git LFS
+
+If you intend to make changes to any of the large binary files in this repository (currently just `*.tar.gz` and `*.jar`) then you will need to install [Git LFS](https://git-lfs.com).
+This is necessary as [GitHub blocks files larger than 100 MiB](https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-large-files-on-github).
+
+If you do not install Git LFS you will only get the pointer files and not the actual data. **This is not a problem unless you want to edit these files.**
+See [this section of the GitHub docs](https://docs.github.com/en/repositories/working-with-files/managing-large-files) for more information
+
+Git LFS also uses hooks, specifically `post-checkout`, `post-commit`, `post-merge` and `pre-push`.
+In the case of the latter, husky also uses this hook which is why the file at [.husky/_/pre-push](.husky/_/pre-push) contains both husky and Git LFS code.
+Note that the Git LFS hooks are in the husky directory because husky was installed in the repository before Git LFS and so that directory structure was already in place.
+Manually editing the hooks was necessary due to the clash on `pre-push`, and [this comment](https://github.com/typicode/husky/issues/108#issuecomment-1432554983) was the general direction taken.
+
 ## Repository structure
 
 #### Lambdas
@@ -45,6 +60,7 @@ The lambdas and supporting code are written in [TypeScript](https://www.typescri
 
 Individual lambda handlers (and unit tests) can be found in subdirectories of the [src/handlers](src/handlers) directory.
 Common and utility code can be found in the [src/shared](src/shared) directory.
+Lambda layers can be found in subdirectories of the [src/layers](src/layers) directory.
 
 In addition, files to support running lambdas with `sam local invoke` are in the [sam-local-examples](sam-local-examples) directory.
 
@@ -133,7 +149,7 @@ In addition, [checkov](https://www.checkov.io) can find misconfigurations. Prett
 
 #### Lambdas
 
-* `npm run build` - build (transpile, bundle, etc.) lambdas into the [dist](dist) directory
+* `npm run build` - build (transpile, bundle, etc.) lambdas into the [dist](dist) directory and build the flyway lambda layer into the [layer-dist](layer-dist) directory
 
 Lambdas can be run locally with [sam local invoke](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-local-invoke.html). A few prerequisites:
 

iac/main/resources/redshift.yml

@@ -90,12 +90,6 @@ RedshiftSecretRotationLambda:
     Policies:
       - AWSLambdaBasicExecutionRole
       - Statement:
-          - Effect: Allow
-            Action:
-              - 'sqs:DeleteMessage'
-              - 'sqs:GetQueueAttributes'
-              - 'sqs:ReceiveMessage'
-            Resource: !If [UsePlaceholderTxMAQueue, !GetAtt EventConsumerQueue.Arn, '{{resolve:ssm:TxMAEventQueueARN}}']
           - Effect: Allow
             Action:
               - secretsmanager:DescribeSecret
@@ -264,22 +258,54 @@ RedshiftMigrationRole:
           Version: 2012-10-17
           Statement:
             - Effect: Allow
-              Action: secretsmanager:ListSecrets
-              Resource: '*'
-            - Effect: Allow
-              Action: secretsmanager:GetSecretValue
-              Resource: !Ref RedshiftSecret
-            - Effect: Allow
-              Action: redshift-serverless:GetWorkgroup
-              Resource: !GetAtt RedshiftServerlessWorkgroup.Workgroup.WorkgroupArn
-            - Effect: Allow
-              Action: kms:Decrypt
-              Resource: !GetAtt KmsKey.Arn
-            - Effect: Allow
-              Action: ssm:StartSession
-              Resource:
-                - arn:aws:ssm:eu-west-2::document/AWS-StartPortForwardingSessionToRemoteHost
-                - !Sub arn:aws:ec2:eu-west-2:${AWS::AccountId}:instance/${RedshiftAccessEC2}
-            - Effect: Allow
-              Action: ec2:DescribeInstances
-              Resource: '*'
+              Action: lambda:InvokeFunction
+              Resource: !GetAtt RunFlywayCommandLambda.Arn
+
+RunFlywayCommandLambda:
+  # checkov:skip=CKV_AWS_116: DLQ not needed
+  Type: AWS::Serverless::Function
+  Properties:
+    FunctionName: run-flyway-command
+    Handler: run-flyway-command.handler
+    Policies:
+      - AWSLambdaBasicExecutionRole
+      - Statement:
+          - Effect: Allow
+            Action: secretsmanager:GetSecretValue
+            Resource: !Ref RedshiftSecret
+          - Effect: Allow
+            Action:
+              - kms:Decrypt
+              - kms:DescribeKey
+              - kms:GenerateDataKey
+            Resource: !GetAtt KmsKey.Arn
+            Condition:
+              StringEquals:
+                kms:EncryptionContext:SecretARN: !Ref RedshiftSecret
+    ReservedConcurrentExecutions: 10
+    Environment:
+      # checkov:skip=CKV_AWS_173: These environment variables do not require encryption
+      Variables:
+        REDSHIFT_SECRET_ID: !Ref RedshiftSecret
+        ENVIRONMENT: !Ref Environment
+    Tags:
+      Environment: !Ref Environment
+    MemorySize: 512
+    Timeout: 600
+    VpcConfig:
+      SecurityGroupIds:
+        - !Ref LambdaSecurityGroup
+        - !Ref RedshiftAccessEC2SecurityGroup
+      SubnetIds:
+        - !Ref SubnetForDAP1
+        - !Ref SubnetForDAP2
+        - !Ref SubnetForDAP3
+    Layers:
+      - !Ref FlywayLayer
+
+FlywayLayer:
+  Type: AWS::Serverless::LayerVersion
+  Properties:
+    ContentUri: layer-dist/flyway
+    LayerName: !Sub ${Environment}-dap-lambda-layer-flyway
+    RetentionPolicy: Delete

package.json

@@ -48,7 +48,7 @@
     "typescript": "^5.3.3"
   },
   "scripts": {
-    "build": "scripts/build-lambdas.sh",
+    "build": "scripts/build-lambdas.sh && scripts/build-flyway-layer.sh",
     "check": "npm run format:check && npm run lint:check",
     "format:check": "prettier --check src tests",
     "format:fix": "prettier --write src tests",

redshift-scripts/migrations/V1__add_hello_table.sql

@@ -0,0 +1 @@
+CREATE TABLE hello (id int);

scripts/build-flyway-layer.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+ROOT=$(pwd)
+LAYER_DIST="$ROOT/layer-dist/flyway"
+LAYER_SRC="$ROOT/src/layers/flyway"
+BIN_DIR="$LAYER_DIST"/bin
+FLYWAY_DIR="$LAYER_DIST"/flyway
+
+rm -rf "$ROOT/layer-dist"
+mkdir -p "$BIN_DIR"
+mkdir -p "$FLYWAY_DIR"
+
+echo "Building layers/flyway"
+
+cp -R "$LAYER_SRC"/run-flyway "$BIN_DIR"
+
+tar -xzf "$LAYER_SRC"/flyway-commandline-*-linux-x64.tar.gz -C "$FLYWAY_DIR" --strip-components 1
+
+# delete almost everything in drivers/ to reduce the size of the lambda so it will fit within the AWS deployment package limit
+# see https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html
+mv "$FLYWAY_DIR"/drivers/jackson-data*.jar "$LAYER_DIST"
+rm -r "$FLYWAY_DIR"/drivers/*
+mv "$LAYER_DIST"/jackson-data*.jar "$FLYWAY_DIR"/drivers
+
+# add redshift driver
+cp "$LAYER_SRC"/redshift-jdbc*.jar "$FLYWAY_DIR"/drivers
+
+# further reduce package size by deleting a large and apparently unneeded directory from lib/
+rm -rf "$FLYWAY_DIR"/lib/rgcompare
+
+# remove jre/legal/ as it is full of broken symlinks that cause sam deploy to exit with an error
+rm -rf "$FLYWAY_DIR"/jre/legal
+
+# add migrations
+mkdir -p "$FLYWAY_DIR"/sql
+cp redshift-scripts/migrations/*.sql "$FLYWAY_DIR"/sql

scripts/build-lambdas.sh

@@ -10,7 +10,7 @@ mkdir -p "$DIST"
 for dir in "$SRC"/*; do
   srcPath="${dir}/handler.ts"
   lambdaName="${dir##*/}"
-  echo "Building $lambdaName"
+  echo "Building handlers/$lambdaName"
   esbuild "$srcPath" --bundle --minify --sourcemap --platform=node --target=es2020 --outfile="$DIST/$lambdaName.js" --log-level=warning \
     --external:better-sqlite3 --external:better-mysql2 --external:mysql* --external:oracledb --external:pg-query-stream --external:sqlite3 --external:tedious
 done

src/handlers/athena-get-config/handler.spec.ts

@@ -61,7 +61,7 @@ test('bad json', async () => {
   mockS3Client.resolves({ Body: mockS3BodyStream({ stringValue: 'hi' }) });
 
   await expect(handler(TEST_EVENT)).rejects.toThrow(
-    'Error parsing JSON string "hi". Original error: SyntaxError: Unexpected token h in JSON at position 0',
+    'Error parsing JSON string "hi" - Unexpected token h in JSON at position 0',
   );
   expect(mockS3Client.calls()).toHaveLength(1);
 });

src/handlers/redshift-rotate-secret/database-access.ts

@@ -1,4 +1,4 @@
-import type { RedshiftSecret } from './handler';
+import type { RedshiftSecret } from '../../shared/types/secrets-manager';
 import type { Knex } from 'knex';
 import { knex } from 'knex';
 import type { Logger } from '@aws-lambda-powertools/logger';

src/handlers/redshift-rotate-secret/handler.spec.ts

@@ -8,7 +8,8 @@ import {
   UpdateSecretVersionStageCommand,
 } from '@aws-sdk/client-secrets-manager';
 import { databaseAccess, handler } from './handler';
-import type { RedshiftSecret, RotateSecretStep, SecretRotationStage } from './handler';
+import type { RotateSecretStep } from './handler';
+import type { RedshiftSecret, SecretRotationStage } from '../../shared/types/secrets-manager';
 import type { Database } from './database-access';
 
 const mockSecretsManagerClient = mockClient(SecretsManagerClient);

src/handlers/redshift-rotate-secret/handler.ts

@@ -5,12 +5,13 @@ import type { DescribeSecretCommandOutput } from '@aws-sdk/client-secrets-manage
 import {
   DescribeSecretCommand,
   GetRandomPasswordCommand,
-  GetSecretValueCommand,
   PutSecretValueCommand,
   UpdateSecretVersionStageCommand,
 } from '@aws-sdk/client-secrets-manager';
 import * as crypto from 'node:crypto';
 import { DatabaseAccess } from './database-access';
+import { getSecret } from '../../shared/secrets-manager/get-secret';
+import type { RedshiftSecret, SecretRotationStage } from '../../shared/types/secrets-manager';
 
 const logger = getLogger('lambda/redshift-rotate-secret');
 
@@ -22,18 +23,6 @@ interface RotateSecretEvent {
   ClientRequestToken: string;
 }
 
-// based on https://docs.aws.amazon.com/secretsmanager/latest/userguide/reference_secret_json_structure.html#reference_secret_json_structure_RS
-export interface RedshiftSecret {
-  engine: 'redshift';
-  host: string;
-  username: string;
-  password: string;
-  dbname: string;
-  port: string;
-}
-
-export type SecretRotationStage = 'AWSPREVIOUS' | 'AWSCURRENT' | 'AWSPENDING';
-
 export const databaseAccess = new DatabaseAccess(logger);
 
 /**
@@ -88,7 +77,7 @@ const rotateSecret = async (event: RotateSecretEvent): Promise<void> => {
 // generate a new secret
 const createSecret = async (event: RotateSecretEvent): Promise<void> => {
   try {
-    await getSecret(event, 'AWSPENDING');
+    await getRedshiftSecret(event, 'AWSPENDING');
     logger.info('createSecret: Successfully retrieved secret');
   } catch (error) {
     await updateSecretPassword(event);
@@ -98,7 +87,7 @@ const createSecret = async (event: RotateSecretEvent): Promise<void> => {
 
 // set the pending secret in the database
 const setSecret = async (event: RotateSecretEvent): Promise<void> => {
-  const pendingSecret = await getSecret(event, 'AWSPENDING');
+  const pendingSecret = await getRedshiftSecret(event, 'AWSPENDING');
   let loginSecret = pendingSecret;
   logger.info('setSecret: Trying connection with AWSPENDING secret');
   let connection = await databaseAccess.getDatabaseConnection(loginSecret);
@@ -107,12 +96,12 @@ const setSecret = async (event: RotateSecretEvent): Promise<void> => {
     return;
   }
 
-  loginSecret = await getSecret(event, 'AWSCURRENT');
+  loginSecret = await getRedshiftSecret(event, 'AWSCURRENT');
   logger.info('setSecret: Trying connection with AWSCURRENT secret');
   connection = await databaseAccess.getDatabaseConnection(loginSecret);
 
   if (connection === null) {
-    loginSecret = await getSecret(event, 'AWSPREVIOUS');
+    loginSecret = await getRedshiftSecret(event, 'AWSPREVIOUS');
     logger.info('setSecret: Trying connection with AWSPREVIOUS secret');
     connection = await databaseAccess.getDatabaseConnection(loginSecret);
 
@@ -135,7 +124,7 @@ const setSecret = async (event: RotateSecretEvent): Promise<void> => {
 
 // test the pending secret against the database
 const testSecret = async (event: RotateSecretEvent): Promise<void> => {
-  const secret = await getSecret(event, 'AWSPENDING');
+  const secret = await getRedshiftSecret(event, 'AWSPENDING');
   const connection = await databaseAccess.getDatabaseConnection(secret);
   if (connection === null) {
     logAndThrow('testSecret: Unable to log into database with pending secret');
@@ -163,21 +152,8 @@ const finishSecret = async (event: RotateSecretEvent, versions: Record<string, s
   logger.info(`finishSecret: Successfully set AWSCURRENT stage to version ${event.ClientRequestToken} for secret`);
 };
 
-const getSecret = async (event: RotateSecretEvent, stage: SecretRotationStage): Promise<RedshiftSecret> => {
-  try {
-    const secretString = await secretsManagerClient
-      .send(
-        new GetSecretValueCommand({
-          SecretId: event.SecretId,
-          VersionStage: stage,
-          ...(stage === 'AWSPENDING' ? { VersionId: event.ClientRequestToken } : {}),
-        }),
-      )
-      .then(response => ensureDefined(() => response.SecretString));
-    return JSON.parse(secretString);
-  } catch (error) {
-    throw new Error(`Error getting secret - ${getErrorMessage(error)}`);
-  }
+const getRedshiftSecret = async (event: RotateSecretEvent, stage: SecretRotationStage): Promise<RedshiftSecret> => {
+  return await getSecret(event.SecretId, stage, event.ClientRequestToken);
 };
 
 const describeSecret = async (event: RotateSecretEvent): Promise<DescribeSecretCommandOutput> => {
@@ -211,7 +187,7 @@ const getRandomPassword = async (): Promise<string> => {
 
 const updateSecretPassword = async (event: RotateSecretEvent): Promise<void> => {
   try {
-    const currentSecret = await getSecret(event, 'AWSCURRENT');
+    const currentSecret = await getRedshiftSecret(event, 'AWSCURRENT');
     const newPassword = await getRandomPassword();
     await secretsManagerClient.send(
       new PutSecretValueCommand({