From ea02288d292d91b3004f698342a46767347b7f22 Mon Sep 17 00:00:00 2001 From: hdavey-gds <129174608+hdavey-gds@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:30:50 +0000 Subject: [PATCH] DAC-1889 Provide interface for analysts to use Quicksight user lambda (#490) Add new get-quicksight-user-spreadsheet script which uses google APIs to get data from the users spreadsheet Add new quicksight-add-users-from-spreadsheet lambda which parses the user spreadsheet and invokes the quicksight-add-users lambda Add new add-quicksight-users workflow to allow running the new functionality from github --- .github/workflows/add-quicksight-users.yml | 51 +++ README.md | 1 + .../resources/quicksight-access.yml | 67 +++ package-lock.json | 297 +++++++++++++- package.json | 2 + scripts/get-quicksight-user-spreadsheet.mjs | 46 +++ .../handler.spec.ts | 201 +++++++++ .../handler.ts | 165 ++++++++ .../quicksight-add-users/handler.spec.ts | 10 +- src/handlers/quicksight-add-users/handler.ts | 8 +- src/handlers/quicksight-sync-users/handler.ts | 4 +- src/handlers/test-support/handler.ts | 14 +- src/shared/utils/test-utils.ts | 10 + src/shared/utils/utils.ts | 25 ++ .../user-spreadsheet-data-gds.json | 189 +++++++++ .../user-spreadsheet-data-rp.json | 384 ++++++++++++++++++ 16 files changed, 1442 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/add-quicksight-users.yml create mode 100644 scripts/get-quicksight-user-spreadsheet.mjs create mode 100644 src/handlers/quicksight-add-users-from-spreadsheet/handler.spec.ts create mode 100644 src/handlers/quicksight-add-users-from-spreadsheet/handler.ts create mode 100644 src/test-resources/user-spreadsheet-data-gds.json create mode 100644 src/test-resources/user-spreadsheet-data-rp.json diff --git a/.github/workflows/add-quicksight-users.yml b/.github/workflows/add-quicksight-users.yml new file mode 100644 index 000000000..6e89d5857 --- /dev/null +++ b/.github/workflows/add-quicksight-users.yml @@ -0,0 +1,51 @@ +name: ✳️ Add users to Quicksight + +on: + workflow_dispatch: + inputs: + dryRun: + type: boolean + required: true + description: If true, this action only prints the users it thinks need adding + default: false + environment: + type: string + required: true + description: AWS Environment + options: [DEV, TEST, FEATURE, PRODUCTION] + type: + type: choice + required: true + description: Type of user (determines which spreadsheet sheet to read) + options: [GDS, RP] + +jobs: + get-spreadsheet-users-and-invoke-lambda: + # These permissions are needed to interact with GitHub's OIDC Token endpoint (enabling the aws-actions/configure-aws-credentials action) + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Node setup + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + - name: Install node packages + run: npm ci + - name: Create users file + run: node scripts/get-quicksight-user-spreadsheet.mjs ${{ inputs.type }} ${{ secrets.GOOGLE_CLOUD_SERVICE_ACCOUNT_CREDENTIALS }} > spreadsheet.json + - name: Assume AWS add users lambda invoke role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets[format('ADD_USERS_LAMBDA_INVOKE_ROLE_{0}', inputs.environment)] }} + - name: Invoke lambda + run: | + PAYLOAD=$(echo "{\"dryRun\": ${{ inputs.dryRun }}, \"spreadsheet\": $(cat spreadsheet.json)}") + ENCODED=$(echo "$PAYLOAD" | openssl base64) + aws --region eu-west-2 lambda invoke --function-name quicksight-add-users-from-spreadsheet --payload "$ENCODED" out.json + cat out.json diff --git a/README.md b/README.md index 4d92cbf4a..060c3980e 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Below is a list of workflows. The ✳️ symbol at the start of a workflow name | ✳️ Upload testing images to ECR | upload-testing-images.yml | | Builds one or more testing dockerfiles in `tests/scripts/` and uploads the images to ECR. Which dockerfiles to build can be specified via inputs | | SonarCloud Code Analysis | code-quality-sonarcloud.yml | | Runs a SonarCloud analysis on the repository | | ✳️ Run flyway command on redshift | run-flyway-command.yml | | Runs a specified flyway command on the redshift database in a specified environment | +| ✳️ Add Quicksight users | add-quicksight-users.yml | | Reads the DAP account management spreadsheet and attempts to add users to Cognito and Quicksight | ## Testing diff --git a/iac/quicksight-access/resources/quicksight-access.yml b/iac/quicksight-access/resources/quicksight-access.yml index 517f67fa8..66d9cf6c5 100644 --- a/iac/quicksight-access/resources/quicksight-access.yml +++ b/iac/quicksight-access/resources/quicksight-access.yml @@ -207,6 +207,73 @@ QuicksightAddUsersLambdaFunction: - Fn::ImportValue: !Sub ${Environment}-dap-vpc-ProtectedSubnetIdB - Fn::ImportValue: !Sub ${Environment}-dap-vpc-ProtectedSubnetIdC +QuicksightAddUsersFromSpreadsheetLambdaFunction: + # checkov:skip=CKV_AWS_116: DLQ not needed as this is a manually invoked action + Type: AWS::Serverless::Function + Condition: IsQuicksightEnvironment + Properties: + FunctionName: quicksight-add-users-from-spreadsheet + Handler: quicksight-add-users-from-spreadsheet.handler + Policies: + - AWSLambdaBasicExecutionRole + - Statement: + - Effect: Allow + Action: lambda:InvokeFunction + Resource: !GetAtt QuicksightAddUsersLambdaFunction.Arn + - Effect: Allow + Action: cognito-idp:AdminGetUser + Resource: !GetAtt QuicksightAccessUserPool.Arn + - Effect: Allow + Action: + - quicksight:DescribeUser + - quicksight:ListUserGroups + Resource: '*' + ReservedConcurrentExecutions: 10 + Environment: + # checkov:skip=CKV_AWS_173: These environment variables do not require encryption + Variables: + ENVIRONMENT: !Ref Environment + USER_POOL_ID: !Ref QuicksightAccessUserPool + Tags: + Environment: !Ref Environment + MemorySize: 512 + # this lambda lives in the protected subnets of a different VPC than the main application lambdas + # because it needs (limited) internet access to call cognito and quicksight APIs and these services do not have VPC endpoints + # it also needs to invoke the quicksight-add-users lambda and must be in this VPC to use the lambda VPC endpoint + # see https://govukverify.atlassian.net/wiki/spaces/PLAT/pages/3531735041/VPC + VpcConfig: + SecurityGroupIds: + - Fn::ImportValue: !Sub ${Environment}-dap-vpc-AWSServicesEndpointSecurityGroupId + SubnetIds: + - Fn::ImportValue: !Sub ${Environment}-dap-vpc-ProtectedSubnetIdA + - Fn::ImportValue: !Sub ${Environment}-dap-vpc-ProtectedSubnetIdB + - Fn::ImportValue: !Sub ${Environment}-dap-vpc-ProtectedSubnetIdC + +QuicksightAddUsersFromSpreadsheetInvokeRole: + Type: AWS::IAM::Role + Condition: IsQuicksightEnvironment + Properties: + RoleName: !Sub ${Environment}-dap-quicksight-add-users-invoke-role + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Federated: !Sub arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com + Action: 'sts:AssumeRoleWithWebIdentity' + Condition: + StringLike: + 'token.actions.githubusercontent.com:sub': + - repo:govuk-one-login/data-analytics-platform:ref:refs/heads/* + - repo:govuk-one-login/data-analytics-platform:environment:* + Policies: + - PolicyName: !Sub ${Environment}-dap-quicksight-add-users-invoke-policy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: lambda:InvokeFunction + Resource: !GetAtt QuicksightAddUsersFromSpreadsheetLambdaFunction.Arn webAcl: Type: 'AWS::WAFv2::WebACL' Condition: IsQuicksightEnvironment diff --git a/package-lock.json b/package-lock.json index 470bc7237..69bb91cac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,8 @@ "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "eslint-config-standard-with-typescript": "^43.0.0", + "gaxios": "^6.1.1", + "googleapis": "^129.0.0", "html-minifier": "^4.0.0", "husky": "^8.0.3", "jest": "^29.7.0", @@ -5197,6 +5199,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -5678,6 +5692,35 @@ "node": ">=0.10.0" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/bootstrap": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.3.1.tgz", @@ -5766,6 +5809,12 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "dev": true + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -5807,7 +5856,6 @@ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", "dev": true, - "peer": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -6523,6 +6571,15 @@ "node": ">=6.0.0" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.494", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.494.tgz", @@ -7621,6 +7678,12 @@ "node": ">=8" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, "node_modules/extend-shallow": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", @@ -7914,6 +7977,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.1.1.tgz", + "integrity": "sha512-bw8smrX+XlAoo9o1JAksBwX+hi/RG15J+NTSxmNPIclKC3ZVK6C2afwY8OSdRvOK0+ZLecUJYtj2MmjOt3Dm0w==", + "dev": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dev": true, + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -7949,7 +8040,6 @@ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", "dev": true, - "peer": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -8084,6 +8174,66 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/google-auth-library": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.4.1.tgz", + "integrity": "sha512-Chs7cuzDuav8W/BXOoRgSXw4u0zxYtuqAHETDR5Q6dG1RwNwz7NUKjsDDHAsBV3KkiiJBtJqjbzy1XU1L41w1g==", + "dev": true, + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis": { + "version": "129.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-129.0.0.tgz", + "integrity": "sha512-gFatrzby+oh/GxEeMhJOKzgs9eG7yksRcTon9b+kPie4ZnDSgGQ85JgtUaBtLSBkcKpUKukdSP6Km1aCjs4y4Q==", + "dev": true, + "dependencies": { + "google-auth-library": "^9.0.0", + "googleapis-common": "^7.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis-common": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.0.1.tgz", + "integrity": "sha512-mgt5zsd7zj5t5QXvDanjWguMdHAcJmmDrF9RkInCecNsyV7S7YtGqm5v2IWONNID88osb7zmx5FtrAP12JfD0w==", + "dev": true, + "dependencies": { + "extend": "^3.0.2", + "gaxios": "^6.0.3", + "google-auth-library": "^9.0.0", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis-common/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -8109,6 +8259,19 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gtoken": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.0.1.tgz", + "integrity": "sha512-KcFVtoP1CVFtQu0aSk3AyAt2og66PFhZAlkUOuWKwzMLoulHXG5W5wE5xAnHb+yl3/wEFoqGW7/cDGMU8igDZQ==", + "dev": true, + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -8158,7 +8321,6 @@ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", "dev": true, - "peer": true, "engines": { "node": ">= 0.4" }, @@ -8171,7 +8333,6 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", "dev": true, - "peer": true, "engines": { "node": ">= 0.4" }, @@ -8339,6 +8500,19 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -12117,6 +12291,15 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dev": true, + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -12153,6 +12336,27 @@ "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dev": true, + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dev": true, + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -12895,6 +13099,26 @@ "lower-case": "^1.1.1" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -13033,7 +13257,6 @@ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", "dev": true, - "peer": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -13607,6 +13830,21 @@ } ] }, + "node_modules/qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -13867,6 +14105,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/safe-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", @@ -14295,7 +14553,6 @@ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", "dev": true, - "peer": true, "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -15028,6 +15285,12 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true + }, "node_modules/ts-api-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.2.tgz", @@ -15364,6 +15627,12 @@ "deprecated": "Please see https://github.com/lydell/urix#deprecated", "dev": true }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==", + "dev": true + }, "node_modules/use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", @@ -15405,6 +15674,22 @@ "makeerror": "1.0.12" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index a10a63b12..0d0d7a158 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,8 @@ "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "eslint-config-standard-with-typescript": "^43.0.0", + "gaxios": "^6.1.1", + "googleapis": "^129.0.0", "html-minifier": "^4.0.0", "husky": "^8.0.3", "jest": "^29.7.0", diff --git a/scripts/get-quicksight-user-spreadsheet.mjs b/scripts/get-quicksight-user-spreadsheet.mjs new file mode 100644 index 000000000..306598142 --- /dev/null +++ b/scripts/get-quicksight-user-spreadsheet.mjs @@ -0,0 +1,46 @@ +import { Auth, google } from 'googleapis'; + +const USER_TYPE = process.argv[2]; +if (USER_TYPE === undefined || (USER_TYPE !== 'GDS' && USER_TYPE !== 'RP')) { + throw new Error('User type required as parameter 1 (and must be one of [GDS, RP])'); +} + +const GOOGLE_CLOUD_SERVICE_ACCOUNT_CREDENTIALS = process.argv[3]; +if (GOOGLE_CLOUD_SERVICE_ACCOUNT_CREDENTIALS === undefined) { + throw new Error('Base64 encoded service account credentials required as parameter 2'); +} + +const getAuth = async () => { + try { + const credentials = Buffer.from(GOOGLE_CLOUD_SERVICE_ACCOUNT_CREDENTIALS, 'base64').toString('utf-8'); + const auth = new Auth.GoogleAuth({ + credentials: JSON.parse(credentials), + scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'], + }); + return await auth.getClient(); + } catch (e) { + console.error('Error getting authentication', e instanceof Error ? e.message : JSON.stringify(e)); + } +}; + +// we must use the spreadsheets.get API for this operation (https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/get) +// this API returns various metadata we don't want (some of which we filter out with the field mask), but +// if we just use the simpler spreadsheets.values.get API (https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get) +// then we just get plain text back and are unable to see if rows have strikethrough text +const getSpreadsheetData = async () => { + try { + const auth = await getAuth(); + const sheets = google.sheets({ version: 'v4', auth }); + return await sheets.spreadsheets.get({ + spreadsheetId: '1VK5ZNMzh4NrHNrsu1s0GnWhwcPoDVNZiGdQ4IONN2tI', + ranges: [ + USER_TYPE === 'GDS' ? "'Internal Quicksight reader accounts'!A:C" : "'RP Quicksight reader accounts'!A:D", + ], + fields: 'sheets.data.rowData.values(userEnteredValue,userEnteredFormat)', + }); + } catch (e) { + console.error('Error getting spreadsheet contents', e instanceof Error ? e.message : JSON.stringify(e)); + } +}; + +getSpreadsheetData().then(response => console.log(JSON.stringify(response.data))); diff --git a/src/handlers/quicksight-add-users-from-spreadsheet/handler.spec.ts b/src/handlers/quicksight-add-users-from-spreadsheet/handler.spec.ts new file mode 100644 index 000000000..90f92b434 --- /dev/null +++ b/src/handlers/quicksight-add-users-from-spreadsheet/handler.spec.ts @@ -0,0 +1,201 @@ +import { mockClient } from 'aws-sdk-client-mock'; +import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; +import { getTestResource, mockCognitoUser } from '../../shared/utils/test-utils'; +import type { sheets_v4 } from 'googleapis'; +import { handler } from './handler'; +import type { AddUserResult, AddUsersEvent } from '../quicksight-add-users/handler'; +import { + AdminGetUserCommand, + CognitoIdentityProviderClient, + UserNotFoundException, +} from '@aws-sdk/client-cognito-identity-provider'; +import { + DescribeUserCommand, + ListUserGroupsCommand, + QuickSightClient, + ResourceNotFoundException, +} from '@aws-sdk/client-quicksight'; +import type { Context } from 'aws-lambda'; +import { encodeObject } from '../../shared/utils/utils'; +import type { LambdaInvokeResponse } from '../../shared/utils/utils'; + +const ACCOUNT_ID = '123456789012'; + +const CONTEXT: Context = { + invokedFunctionArn: `arn:aws:lambda:eu-west-2:${ACCOUNT_ID}:function:LambdaFunctionName`, +} as unknown as Context; + +const USER_POOL_ID = 'user-pool-id'; + +const mockLambdaClient = mockClient(LambdaClient); +const mockCognitoClient = mockClient(CognitoIdentityProviderClient); +const mockQuicksightClient = mockClient(QuickSightClient); + +let GDS_USER_SHEET: sheets_v4.Schema$Spreadsheet; +let RP_USER_SHEET: sheets_v4.Schema$Spreadsheet; + +beforeAll(async () => { + GDS_USER_SHEET = JSON.parse(await getTestResource('user-spreadsheet-data-gds.json')); + RP_USER_SHEET = JSON.parse(await getTestResource('user-spreadsheet-data-rp.json')); + process.env.USER_POOL_ID = USER_POOL_ID; +}); + +beforeEach(() => { + mockLambdaClient.reset(); + mockLambdaClient.on(InvokeCommand).callsFakeOnce(input => ({ + StatusCode: 200, + ExecutedVersion: 1, + FunctionError: undefined, + LogResult: undefined, + Payload: encodeObject(JSON.parse(input.Payload).requests), + })); + + // give these clients default responses that indicate the user exists in neither service + mockCognitoClient.reset(); + mockQuicksightClient.reset(); + mockCognitoClient + .callsFake(input => { + throw new Error(`Unexpected Cognito request - ${JSON.stringify(input)}`); + }) + .on(AdminGetUserCommand) + .rejects(new UserNotFoundException({ message: '', $metadata: {} })); + mockQuicksightClient + .callsFake(input => { + throw new Error(`Unexpected Quicksight request - ${JSON.stringify(input)}`); + }) + .on(DescribeUserCommand) + .rejects(new ResourceNotFoundException({ message: '', $metadata: {} })) + .on(ListUserGroupsCommand) + .resolves({ GroupList: [] }); +}); + +test('test gds users', async () => { + const expectedEmails = [ + 'user.one@digital.cabinet-office.gov.uk', + 'user.two@digital.cabinet-office.gov.uk', + 'user.three@digital.cabinet-office.gov.uk', + 'user.four@digital.cabinet-office.gov.uk', + ]; + + const payload = await getLambdaPayload(GDS_USER_SHEET); + expect(payload).toBeDefined(); + expect(payload).toHaveLength(4); + expect(payload.map(request => request.username)).toEqual(expect.arrayContaining(expectedEmails)); + expect(payload.map(request => request.email)).toEqual(expect.arrayContaining(expectedEmails)); + expect( + payload.map(request => request.quicksightGroups).every(groups => groups.length === 1 && groups[0] === 'gds-users'), + ).toBe(true); +}); + +test('test rp users', async () => { + // there is a user.five in the file but it has strikethrough text so shouldn't come through + const expectedEmails = [ + 'user.one@dbs.gov.uk', + 'user.two@dbs.gov.uk', + 'user.three@dbs.gov.uk', + 'user.four@dvsa.gov.uk', + 'user.six@dvsa.gov.uk', + ]; + + const payload = await getLambdaPayload(RP_USER_SHEET); + expect(payload).toBeDefined(); + expect(payload).toHaveLength(5); + expect(payload.map(request => request.username)).toEqual(expect.arrayContaining(expectedEmails)); + expect(payload.map(request => request.email)).toEqual(expect.arrayContaining(expectedEmails)); + expect( + payload + .slice(0, 3) + .map(request => request.quicksightGroups) + .every(groups => groups.length === 1 && groups[0] === 'dbs'), + ).toBe(true); + expect( + payload + .slice(3) + .map(request => request.quicksightGroups) + .every(groups => groups.length === 1 && groups[0] === 'dvsa'), + ).toBe(true); +}); + +test('filter out users', async () => { + // there is a user.five in the file but it has strikethrough text so shouldn't come through + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const expectedEmails = ['user.two@dbs.gov.uk', 'user.four@dvsa.gov.uk', 'user.six@dvsa.gov.uk']; + + // cognito and quicksight responses to indicate users one and three have accounts in both which should mean they are filtered out + // user six also has an account but only in quicksight (which will need manual resolution but should not result in the user being filtered at this stage) + mockCognitoClient.reset(); + mockQuicksightClient.reset(); + mockCognitoClient + .on(AdminGetUserCommand) + .rejects(new UserNotFoundException({ message: '', $metadata: {} })) + .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user.one@dbs.gov.uk' }) + .resolvesOnce(mockCognitoUser('user.one@dbs.gov.uk', 'user.one@dbs.gov.uk')) + .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user.three@dbs.gov.uk' }) + .resolvesOnce(mockCognitoUser('user.three@dbs.gov.uk', 'user.three@dbs.gov.uk')); + + mockQuicksightClient + .on(DescribeUserCommand) + .rejects(new ResourceNotFoundException({ message: '', $metadata: {} })) + .on(DescribeUserCommand, { UserName: 'user.one@dbs.gov.uk' }) + .resolvesOnce({ User: { UserName: 'user.one@dbs.gov.uk', Email: 'user.one@dbs.gov.uk' } }) + .on(DescribeUserCommand, { UserName: 'user.three@dbs.gov.uk' }) + .resolvesOnce({ User: { UserName: 'user.three@dbs.gov.uk', Email: 'user.three@dbs.gov.uk' } }) + .on(DescribeUserCommand, { UserName: 'user.six@dvsa.gov.uk' }) + .resolvesOnce({ User: { UserName: 'user.six@dvsa.gov.uk', Email: 'user.six@dvsa.gov.uk' } }) + .on(ListUserGroupsCommand) + .resolves({ GroupList: [] }); + + const payload = await getLambdaPayload(RP_USER_SHEET); + expect(payload).toBeDefined(); + expect(payload).toHaveLength(3); + expect(payload.map(request => request.username)).toEqual(expect.arrayContaining(expectedEmails)); + expect(payload.map(request => request.email)).toEqual(expect.arrayContaining(expectedEmails)); +}); + +test('dry run', async () => { + const expectedEmails = [ + 'user.two@digital.cabinet-office.gov.uk', + 'user.three@digital.cabinet-office.gov.uk', + 'user.four@digital.cabinet-office.gov.uk', + ]; + + // cognito and quicksight responses to indicate user one has an account in both which should mean they are filtered out + // user three also has an account but only in cognito (which will need manual resolution but should not result in the user being filtered at this stage) + mockCognitoClient.reset(); + mockQuicksightClient.reset(); + mockCognitoClient + .on(AdminGetUserCommand) + .rejects(new UserNotFoundException({ message: '', $metadata: {} })) + .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user.one@digital.cabinet-office.gov.uk' }) + .resolvesOnce(mockCognitoUser('user.one@digital.cabinet-office.gov.uk', 'user.one@digital.cabinet-office.gov.uk')) + .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user.three@digital.cabinet-office.gov.uk' }) + .resolvesOnce( + mockCognitoUser('user.three@digital.cabinet-office.gov.uk', 'user.three@digital.cabinet-office.gov.uk'), + ); + + mockQuicksightClient + .on(DescribeUserCommand) + .rejects(new ResourceNotFoundException({ message: '', $metadata: {} })) + .on(DescribeUserCommand, { UserName: 'user.one@digital.cabinet-office.gov.uk' }) + .resolvesOnce({ + User: { UserName: 'user.one@digital.cabinet-office.gov.uk', Email: 'user.one@digital.cabinet-office.gov.uk' }, + }) + .on(ListUserGroupsCommand) + .resolves({ GroupList: [] }); + + const payload = (await handler({ spreadsheet: GDS_USER_SHEET, dryRun: true }, CONTEXT)) as AddUsersEvent; + expect(payload).toBeDefined(); + expect(payload.requests).toHaveLength(3); + expect(payload.requests.map(request => request.username)).toEqual(expect.arrayContaining(expectedEmails)); + expect(payload.requests.map(request => request.email)).toEqual(expect.arrayContaining(expectedEmails)); + + expect(mockLambdaClient.calls()).toHaveLength(0); +}); + +const getLambdaPayload = async (sheet: sheets_v4.Schema$Spreadsheet): Promise => { + const lambdaInput = await handler({ spreadsheet: sheet }, CONTEXT); + expect(lambdaInput).toBeDefined(); + const payload = (lambdaInput as LambdaInvokeResponse).payload as AddUserResult[]; + expect(payload).toBeDefined(); + return payload; +}; diff --git a/src/handlers/quicksight-add-users-from-spreadsheet/handler.ts b/src/handlers/quicksight-add-users-from-spreadsheet/handler.ts new file mode 100644 index 000000000..e61eda6b6 --- /dev/null +++ b/src/handlers/quicksight-add-users-from-spreadsheet/handler.ts @@ -0,0 +1,165 @@ +import { getLogger } from '../../shared/powertools'; +import type { sheets_v4 } from 'googleapis'; +import { InvokeCommand } from '@aws-sdk/client-lambda'; +import { getAccountId, getEnvironmentVariable, getErrorMessage, lambdaInvokeResponse } from '../../shared/utils/utils'; +import type { LambdaInvokeResponse } from '../../shared/utils/utils'; +import { lambdaClient } from '../../shared/clients'; +import { getUserStatus } from '../../shared/quicksight-access/user-status'; +import type { Context } from 'aws-lambda'; +import type { AddUsersEvent } from '../quicksight-add-users/handler'; + +const logger = getLogger('lambda/quicksight-add-users-from-spreadsheet'); + +const EXPECTED_COLUMNS = new Map([ + [0, 'Name'], + [1, 'Email'], + [2, 'Type'], + [3, 'Relying Party'], +]); + +interface AddUsersFromSpreadsheetEvent { + spreadsheet: sheets_v4.Schema$Spreadsheet; + dryRun?: boolean; +} + +type AddUsersFromSpreadsheetResult = AddUsersEvent | LambdaInvokeResponse; + +interface SpreadsheetRow { + Name: string; + Email: string; + Type: string; + RelyingParty: string; +} + +export const handler = async ( + event: AddUsersFromSpreadsheetEvent, + context: Context, +): Promise => { + try { + const rowData = getSpreadsheetRows(event.spreadsheet); + const users = getUsersFromRows(rowData); + logger.info('Parsed user rows from spreadsheet', { users }); + const toBeAdded = await getUsersWithoutAccounts(users, context); + return await sendToAddUsersLambda(toBeAdded, event.dryRun); + } catch (error) { + logger.error('Error preparing to add users', { error }); + throw error; + } +}; + +const sendToAddUsersLambda = async ( + users: SpreadsheetRow[], + dryRun?: boolean, +): Promise => { + const addUsersEvent = { + requests: users.map(user => ({ + username: user.Email, + email: user.Email, + quicksightGroups: user.RelyingParty.length > 0 ? [user.RelyingParty.toLowerCase()] : ['gds-users'], + })), + }; + + if (dryRun === true) { + return addUsersEvent; + } + + logger.info('Sending event to add users lambda', { addUsersEvent }); + const request = new InvokeCommand({ + FunctionName: 'quicksight-add-users', + Payload: JSON.stringify(addUsersEvent), + LogType: 'Tail', + InvocationType: 'RequestResponse', + }); + + try { + const response = await lambdaClient.send(request); + return lambdaInvokeResponse(response); + } catch (error) { + throw new Error(`Error calling add users lambda - ${getErrorMessage(error)}`); + } +}; + +// brief check so the add users lambda is not called with an entire spreadsheet worth of users +// only filter out the case where user has accounts in both cognito and quicksight as this is going to be most common by some way +// the add user lambda can handle the edge-cases where the users exists in only one of the two services +const getUsersWithoutAccounts = async (users: SpreadsheetRow[], context: Context): Promise => { + try { + const accountId = getAccountId(context); + const userPoolId = getEnvironmentVariable('USER_POOL_ID'); + const maybeUsers = await Promise.all( + users.map(async user => { + const status = await getUserStatus(user.Email, userPoolId, accountId); + return status.existsInBoth() ? undefined : user; + }), + ); + // have to use the type guard otherwise typescript thinks the filter() output is (SpreadsheetRow | undefined)[] + // see https://www.benmvp.com/blog/filtering-undefined-elements-from-array-typescript + return maybeUsers.filter((user): user is SpreadsheetRow => user !== undefined); + } catch (error) { + throw new Error(`Error getting users without accounts - ${getErrorMessage(error)}`); + } +}; + +const getUsersFromRows = (rows: sheets_v4.Schema$RowData[]): SpreadsheetRow[] => { + const columnNames = getColumnNames(rows); + const columnValues = getColumnValues(rows, columnNames); + + const getColumnValue = (cells: sheets_v4.Schema$CellData[], columnName: string): string => { + return getCellValue(cells[columnNames.indexOf(columnName)]).trim(); + }; + + return columnValues.map(cells => ({ + Name: getColumnValue(cells, 'Name'), + Email: getColumnValue(cells, 'Email'), + Type: getColumnValue(cells, 'Type'), + RelyingParty: getColumnValue(cells, 'Relying Party'), + })); +}; + +const getColumnNames = (rows: sheets_v4.Schema$RowData[]): string[] => { + const columnNames = getRowCells(rows[0]).map(getCellValue); + const columnCount = columnNames.length; + if (columnCount < 3) { + throw new Error(`Expected 3 or 4 columns - ${JSON.stringify(columnNames)}`); + } + + const validColumns = columnNames.every((name, index) => EXPECTED_COLUMNS.get(index) === name); + if (!validColumns) { + throw new Error(`One or more columns missing or in wrong order - ${JSON.stringify(columnNames)}`); + } + return columnNames; +}; + +const getColumnValues = (rows: sheets_v4.Schema$RowData[], columnNames: string[]): sheets_v4.Schema$CellData[][] => { + // slice(1) to exclude the first row which is the row of column names + return ( + rows + .slice(1) + .map(getRowCells) + // remove lines without values for all properties (e.g. a - line separating 2 blocks of RPs) + .filter(cells => cells.length === columnNames.length) + // remove lines with any strikethrough text as this indicates the user is no longer to be added + .filter(cells => !cells.some(cell => isCellStrikethrough(cell))) + ); +}; + +const getSpreadsheetRows = (spreadsheet: sheets_v4.Schema$Spreadsheet): sheets_v4.Schema$RowData[] => { + // should only be one sheet as we explicitly request either the GDS or RP user sheets + const sheet = spreadsheet?.sheets?.at(0); + if (sheet === undefined) { + throw new Error(`Spreadsheet sheet is missing or undefined - ${JSON.stringify(spreadsheet)}`); + } + // should only be one set of data as you get one per requested range and we only request one range + const rowData = sheet?.data?.at(0)?.rowData; + if (rowData === undefined) { + throw new Error(`Spreadsheet row data is missing or undefined - ${JSON.stringify(spreadsheet)}`); + } + return rowData; +}; + +const getRowCells = (row: sheets_v4.Schema$RowData): sheets_v4.Schema$CellData[] => row.values ?? []; + +const getCellValue = (cell: sheets_v4.Schema$CellData): string => cell?.userEnteredValue?.stringValue ?? ''; + +const isCellStrikethrough = (cell: sheets_v4.Schema$CellData): boolean => + cell?.userEnteredFormat?.textFormat?.strikethrough ?? false; diff --git a/src/handlers/quicksight-add-users/handler.spec.ts b/src/handlers/quicksight-add-users/handler.spec.ts index 6705c2622..f72e515cb 100644 --- a/src/handlers/quicksight-add-users/handler.spec.ts +++ b/src/handlers/quicksight-add-users/handler.spec.ts @@ -1,7 +1,6 @@ import { handler } from './handler'; import type { Context } from 'aws-lambda'; import { mockClient } from 'aws-sdk-client-mock'; -import type { AttributeType } from '@aws-sdk/client-cognito-identity-provider'; import { AdminCreateUserCommand, AdminGetUserCommand, @@ -16,6 +15,7 @@ import { RegisterUserCommand, ResourceNotFoundException, } from '@aws-sdk/client-quicksight'; +import { mockCognitoUser } from '../../shared/utils/test-utils'; const ACCOUNT_ID = '123456789012'; @@ -128,11 +128,11 @@ test('user existence failures', async () => { }) .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user-a' }) - .resolvesOnce(cognitoUser('user-a', 'a@a.com')) + .resolvesOnce(mockCognitoUser('user-a', 'a@a.com')) .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user-b' }) .rejectsOnce(new UserNotFoundException({ message: '', $metadata: {} })) .on(AdminGetUserCommand, { UserPoolId: USER_POOL_ID, Username: 'user-c' }) - .resolvesOnce(cognitoUser('user-c', 'c@c.com')); + .resolvesOnce(mockCognitoUser('user-c', 'c@c.com')); mockQuicksightClient .callsFake(input => { @@ -347,7 +347,3 @@ test('user and group add errors', async () => { expect(mockCognitoClient.calls()).toHaveLength(6); expect(mockQuicksightClient.calls()).toHaveLength(9); }); - -const cognitoUser = (username: string, email: string): { Username: string; UserAttributes: AttributeType[] } => { - return { Username: username, UserAttributes: [{ Name: 'email', Value: email }] }; -}; diff --git a/src/handlers/quicksight-add-users/handler.ts b/src/handlers/quicksight-add-users/handler.ts index 01892066b..8a74a2c27 100644 --- a/src/handlers/quicksight-add-users/handler.ts +++ b/src/handlers/quicksight-add-users/handler.ts @@ -5,7 +5,7 @@ import { CreateGroupMembershipCommand, RegisterUserCommand } from '@aws-sdk/clie import type { CreateGroupMembershipCommandOutput, RegisterUserCommandOutput } from '@aws-sdk/client-quicksight'; import { AdminCreateUserCommand } from '@aws-sdk/client-cognito-identity-provider'; import type { AdminCreateUserCommandOutput } from '@aws-sdk/client-cognito-identity-provider'; -import { getAccountId, getEnvironmentVariable } from '../../shared/utils/utils'; +import { getAccountId, getEnvironmentVariable, getErrorMessage } from '../../shared/utils/utils'; import { getUserStatus } from '../../shared/quicksight-access/user-status'; const logger = getLogger('lambda/quicksight-add-users'); @@ -16,11 +16,11 @@ interface AddUserRequest { quicksightGroups: string[]; } -interface AddUsersEvent { +export interface AddUsersEvent { requests: AddUserRequest[]; } -interface AddUserResult extends AddUserRequest { +export interface AddUserResult extends AddUserRequest { error?: string; } @@ -58,7 +58,7 @@ const addUser = async (request: AddUserRequest, userPoolId: string, accountId: s } return { ...request }; } catch (error) { - return { ...request, error: error instanceof Error ? error.message : JSON.stringify(error) }; + return { ...request, error: getErrorMessage(error) }; } }; diff --git a/src/handlers/quicksight-sync-users/handler.ts b/src/handlers/quicksight-sync-users/handler.ts index bccae4f7a..055debb36 100644 --- a/src/handlers/quicksight-sync-users/handler.ts +++ b/src/handlers/quicksight-sync-users/handler.ts @@ -2,7 +2,7 @@ import { getLogger } from '../../shared/powertools'; import type { Context } from 'aws-lambda'; import { cognitoClient, quicksightClient } from '../../shared/clients'; import { DeleteUserCommand, RegisterUserCommand } from '@aws-sdk/client-quicksight'; -import { getAccountId, getEnvironmentVariable } from '../../shared/utils/utils'; +import { getAccountId, getEnvironmentVariable, getErrorMessage } from '../../shared/utils/utils'; import { AdminCreateUserCommand, AdminDeleteUserCommand } from '@aws-sdk/client-cognito-identity-provider'; import { getUserStatus } from '../../shared/quicksight-access/user-status'; @@ -55,7 +55,7 @@ const syncUser = async (user: SyncUser, userPoolId: string, accountId: string): await quicksightClient.send(getQuicksightRequest(user, accountId)); return { user }; } catch (error) { - return { user, error: error instanceof Error ? error.message : JSON.stringify(error) }; + return { user, error: getErrorMessage(error) }; } }; diff --git a/src/handlers/test-support/handler.ts b/src/handlers/test-support/handler.ts index 82cce9586..fa98bc749 100644 --- a/src/handlers/test-support/handler.ts +++ b/src/handlers/test-support/handler.ts @@ -1,7 +1,6 @@ import { AWS_ENVIRONMENTS } from '../../shared/constants'; -import { decodeObject, getAccountId, getRequiredParams } from '../../shared/utils/utils'; +import { getAccountId, getRequiredParams, lambdaInvokeResponse } from '../../shared/utils/utils'; import { DescribeLogStreamsCommand, GetLogEventsCommand } from '@aws-sdk/client-cloudwatch-logs'; -import type { InvokeCommandOutput } from '@aws-sdk/client-lambda'; import { InvokeCommand, ListEventSourceMappingsCommand } from '@aws-sdk/client-lambda'; import type { CopyObjectCommandOutput, DeleteObjectCommandOutput, GetObjectCommandOutput } from '@aws-sdk/client-s3'; import { @@ -183,17 +182,6 @@ const s3GetResponse = async (response: GetObjectCommandOutput): Promise> => { - return { - executedVersion: response.ExecutedVersion, - statusCode: response.StatusCode, - functionError: response.FunctionError, - logResult: Buffer.from(response.LogResult ?? '', 'base64').toString('utf-8'), - payload: decodeObject(response.Payload ?? new Uint8Array([0x7b, 0x7d])), - }; -}; - const gzipToString = async (response: GetObjectCommandOutput): Promise => { const bytes = await response.Body?.transformToByteArray(); if (bytes === undefined) { diff --git a/src/shared/utils/test-utils.ts b/src/shared/utils/test-utils.ts index 64d9057d9..9fd443214 100644 --- a/src/shared/utils/test-utils.ts +++ b/src/shared/utils/test-utils.ts @@ -2,6 +2,7 @@ import type { APIGatewayProxyEventV2, SQSEvent } from 'aws-lambda'; import { readFile } from 'fs/promises'; import type { Readable } from 'stream'; import type { SdkStream } from '@aws-sdk/types'; +import type { AttributeType } from '@aws-sdk/client-cognito-identity-provider'; export const mockSQSEvent = (...bodies: unknown[]): SQSEvent => { return { @@ -46,3 +47,12 @@ export const mockApiGatewayEvent = async ( }, }; }; + +interface MockCognitoUser { + Username: string; + UserAttributes: AttributeType[]; +} + +export const mockCognitoUser = (username: string, email: string): MockCognitoUser => { + return { Username: username, UserAttributes: [{ Name: 'email', Value: email }] }; +}; diff --git a/src/shared/utils/utils.ts b/src/shared/utils/utils.ts index 7161caa6b..79b986c78 100644 --- a/src/shared/utils/utils.ts +++ b/src/shared/utils/utils.ts @@ -1,6 +1,7 @@ import type { GetObjectCommandOutput } from '@aws-sdk/client-s3'; import { AWS_ENVIRONMENTS } from '../constants'; import type { Context } from 'aws-lambda'; +import type { InvokeCommandOutput } from '@aws-sdk/client-lambda'; /** * Requires that an object has the specified properties (and they are not null or undefined), throwing an error if not. @@ -87,6 +88,30 @@ export const getAccountId = (context: Context): string => { ); }; +export interface LambdaInvokeResponse { + executedVersion?: string; + statusCode?: number; + functionError?: string; + logResult: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + payload: Record; +} + +// custom response as real response LogResult is base64 encoded and Payload is encoded as a UintArray +export const lambdaInvokeResponse = (response: InvokeCommandOutput): LambdaInvokeResponse => { + return { + executedVersion: response.ExecutedVersion, + statusCode: response.StatusCode, + functionError: response.FunctionError, + logResult: Buffer.from(response.LogResult ?? '', 'base64').toString('utf-8'), + payload: decodeObject(response.Payload ?? new Uint8Array([0x7b, 0x7d])), + }; +}; + +export const getErrorMessage = (error: unknown): string => { + return error instanceof Error ? error.message : JSON.stringify(error); +}; + // see https://stackoverflow.com/a/65666402 const throwExpression = (message: string): never => { throw new Error(message); diff --git a/src/test-resources/user-spreadsheet-data-gds.json b/src/test-resources/user-spreadsheet-data-gds.json new file mode 100644 index 000000000..bc48f5168 --- /dev/null +++ b/src/test-resources/user-spreadsheet-data-gds.json @@ -0,0 +1,189 @@ +{ + "sheets": [ + { + "data": [ + { + "rowData": [ + { + "values": [ + { + "userEnteredValue": { + "stringValue": "Name" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + }, + { + "userEnteredValue": { + "stringValue": "Email" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + }, + { + "userEnteredValue": { + "stringValue": "Type" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User One" + } + }, + { + "userEnteredValue": { + "stringValue": "user.one@digital.cabinet-office.gov.uk" + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Two" + } + }, + { + "userEnteredValue": { + "stringValue": "user.two@digital.cabinet-office.gov.uk" + }, + "userEnteredFormat": { + "textFormat": { + "link": { + "uri": "mailto:user.two@digital.cabinet-office.gov.uk" + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Three" + } + }, + { + "userEnteredValue": { + "stringValue": "user.three@digital.cabinet-office.gov.uk" + }, + "userEnteredFormat": { + "backgroundColor": { + "red": 0.972549, + "green": 0.972549, + "blue": 0.972549 + }, + "horizontalAlignment": "LEFT", + "textFormat": { + "foregroundColor": { + "red": 0.11372549, + "green": 0.10980392, + "blue": 0.11372549 + }, + "fontFamily": "Slack-Lato, Slack-Fractions, appleLogo, sans-serif", + "fontSize": 11, + "foregroundColorStyle": { + "rgbColor": { + "red": 0.11372549, + "green": 0.10980392, + "blue": 0.11372549 + } + }, + "link": { + "uri": "mailto:user.three@digital.cabinet-office.gov.uk" + } + }, + "backgroundColorStyle": { + "rgbColor": { + "red": 0.972549, + "green": 0.972549, + "blue": 0.972549 + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Four" + } + }, + { + "userEnteredValue": { + "stringValue": "user.four@digital.cabinet-office.gov.uk" + }, + "userEnteredFormat": { + "backgroundColor": { + "red": 1, + "green": 1, + "blue": 1 + }, + "horizontalAlignment": "LEFT", + "verticalAlignment": "BOTTOM", + "wrapStrategy": "WRAP", + "textFormat": { + "foregroundColor": {}, + "fontFamily": "Arial", + "fontSize": 10, + "underline": false, + "foregroundColorStyle": { + "rgbColor": {} + } + }, + "backgroundColorStyle": { + "rgbColor": { + "red": 1, + "green": 1, + "blue": 1 + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/test-resources/user-spreadsheet-data-rp.json b/src/test-resources/user-spreadsheet-data-rp.json new file mode 100644 index 000000000..4c5348682 --- /dev/null +++ b/src/test-resources/user-spreadsheet-data-rp.json @@ -0,0 +1,384 @@ +{ + "sheets": [ + { + "data": [ + { + "rowData": [ + { + "values": [ + { + "userEnteredValue": { + "stringValue": "Name" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + }, + { + "userEnteredValue": { + "stringValue": "Email" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + }, + { + "userEnteredValue": { + "stringValue": "Type" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + }, + { + "userEnteredValue": { + "stringValue": "Relying Party" + }, + "userEnteredFormat": { + "textFormat": { + "bold": true + } + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User One" + } + }, + { + "userEnteredValue": { + "stringValue": "user.one@dbs.gov.uk" + }, + "userEnteredFormat": { + "textFormat": { + "fontSize": 10 + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DBS" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Two" + } + }, + { + "userEnteredValue": { + "stringValue": "user.two@dbs.gov.uk" + }, + "userEnteredFormat": { + "horizontalAlignment": "LEFT", + "textFormat": { + "foregroundColor": {}, + "fontSize": 10, + "foregroundColorStyle": { + "rgbColor": {} + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DBS" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Three" + } + }, + { + "userEnteredValue": { + "stringValue": "user.three@dbs.gov.uk" + }, + "userEnteredFormat": { + "backgroundColor": { + "red": 1, + "green": 1, + "blue": 1 + }, + "textFormat": { + "foregroundColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + }, + "fontSize": 10, + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + } + } + }, + "backgroundColorStyle": { + "rgbColor": { + "red": 1, + "green": 1, + "blue": 1 + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DBS" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "-" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Four" + }, + "userEnteredFormat": { + "verticalAlignment": "BOTTOM", + "textFormat": { + "foregroundColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + }, + "fontFamily": "\"Calibri Light\", sans-serif", + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + } + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "user.four@dvsa.gov.uk" + }, + "userEnteredFormat": { + "horizontalAlignment": "LEFT", + "textFormat": { + "foregroundColor": {}, + "fontSize": 10, + "underline": false, + "foregroundColorStyle": { + "rgbColor": {} + } + }, + "hyperlinkDisplayType": "PLAIN_TEXT" + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DVSA" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Five" + }, + "userEnteredFormat": { + "verticalAlignment": "BOTTOM", + "textFormat": { + "foregroundColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + }, + "fontFamily": "\"Calibri Light\", sans-serif", + "strikethrough": true, + "underline": false, + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + } + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "user.five@dvsa.gov.uk" + }, + "userEnteredFormat": { + "backgroundColor": { + "red": 1, + "green": 1, + "blue": 1 + }, + "textFormat": { + "foregroundColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + }, + "fontSize": 10, + "strikethrough": true, + "underline": false, + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + } + } + }, + "hyperlinkDisplayType": "PLAIN_TEXT", + "backgroundColorStyle": { + "rgbColor": { + "red": 1, + "green": 1, + "blue": 1 + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DVSA" + } + } + ] + }, + { + "values": [ + { + "userEnteredValue": { + "stringValue": "User Six" + }, + "userEnteredFormat": { + "verticalAlignment": "BOTTOM", + "textFormat": { + "foregroundColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + }, + "fontFamily": "\"Calibri Light\", sans-serif", + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12941177, + "green": 0.12941177, + "blue": 0.12941177 + } + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "user.six@dvsa.gov.uk" + }, + "userEnteredFormat": { + "backgroundColor": { + "red": 1, + "green": 1, + "blue": 1 + }, + "textFormat": { + "foregroundColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + }, + "fontSize": 10, + "underline": false, + "foregroundColorStyle": { + "rgbColor": { + "red": 0.12156863, + "green": 0.12156863, + "blue": 0.12156863 + } + } + }, + "hyperlinkDisplayType": "PLAIN_TEXT", + "backgroundColorStyle": { + "rgbColor": { + "red": 1, + "green": 1, + "blue": 1 + } + } + } + }, + { + "userEnteredValue": { + "stringValue": "Reader" + } + }, + { + "userEnteredValue": { + "stringValue": "DVSA" + } + } + ] + } + ] + } + ] + } + ] +}