From 27bd0f8577588fd8fc64d99aa8fc468242d641c5 Mon Sep 17 00:00:00 2001 From: Mischa Spiegelmock Date: Mon, 1 Jul 2024 21:52:01 -0700 Subject: [PATCH] DB improvements --- backend/prisma/schema.prisma | 2 +- backend/src/db/client.ts | 38 ++++++++++++++++++----- backend/src/db/runMigrations.ts | 16 +++++++--- stacks/database.ts | 24 +++++++++++---- stacks/iam.ts | 16 ++++++++++ stacks/index.ts | 2 ++ stacks/resources/migrationScript.ts | 30 +++++++++--------- stacks/secrets.ts | 17 +++++----- web/next.config.mjs | 48 +++++++++++++++++++++++++++-- 9 files changed, 149 insertions(+), 44 deletions(-) create mode 100644 stacks/iam.ts diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 1494b6b..0f91605 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -3,7 +3,7 @@ generator client { provider = "prisma-client-js" - previewFeatures = [] + previewFeatures = ["postgresqlExtensions"] binaryTargets = ["native"] } diff --git a/backend/src/db/client.ts b/backend/src/db/client.ts index ba35027..d0ddc42 100644 --- a/backend/src/db/client.ts +++ b/backend/src/db/client.ts @@ -1,9 +1,33 @@ -import type { Prisma, PrismaClient as PrismaClientType } from '@prisma/client'; -import { PrismaClient } from '@prisma/client'; -import { Config } from 'sst/node/config'; -import memoize from 'memoizee'; -import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager'; +import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager' +import type { Prisma, PrismaClient as PrismaClientType } from '@prisma/client' +import { PrismaClient } from '@prisma/client' +import memoize from 'memoizee' +import { Config } from 'sst/node/config' export const getPrisma = memoize(async (opts?: Prisma.PrismaClientOptions): Promise => { - return new PrismaClient(opts); -}); + const databaseUrl = await getDatabaseUrl() + opts = opts || {} + opts.datasourceUrl ||= databaseUrl + + return new PrismaClient(opts) +}) + +export const getDatabaseUrl = async (): Promise => { + let databaseUrl = process.env.DATABASE_URL + if (process.env.USE_DB_CONFIG !== 'true' && databaseUrl) return databaseUrl + + // load database secret + const secretArn = Config.DB_SECRET_ARN + const client = new SecretsManagerClient({}) + const req = new GetSecretValueCommand({ SecretId: secretArn }) + const res = await client.send(req) + if (!res.SecretString) throw new Error(`Missing secretString in ${secretArn}`) + const secrets = JSON.parse(res.SecretString) as any + const { host, username, password, port, dbname } = secrets + if (!host) throw new Error('Missing host in secrets') + + // construct database url + databaseUrl = `postgresql://${username}:${password}@${host}:${port}/${dbname}` + + return databaseUrl +} diff --git a/backend/src/db/runMigrations.ts b/backend/src/db/runMigrations.ts index b6927a7..9c75aa8 100644 --- a/backend/src/db/runMigrations.ts +++ b/backend/src/db/runMigrations.ts @@ -3,13 +3,13 @@ Not really using a public API. */ +import { SecretsManagerClient, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager' +import { isProd } from '@common/env' +import { PrismaClient } from '@prisma/client' import { Migrate } from '@prisma/migrate/dist/Migrate.js' import { ensureDatabaseExists } from '@prisma/migrate/dist/utils/ensureDatabaseExists' import { printFilesFromMigrationIds } from '@prisma/migrate/dist/utils/printFiles' import chalk from 'chalk' -import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager' -import { Prisma, PrismaClient } from '@prisma/client' -import { isProd } from '@common/env' export const handler = async (): Promise => { const schemaPath = '/var/task/backend/prisma/schema.prisma' @@ -66,12 +66,12 @@ ${editedMigrationNames.join('\n')}`, } } -const loadDatabaseUrl = async (): Promise => { +// like getDatabaseUrl from client.ts but doesn't use SST config +const _getDatabaseUrl = async (): Promise => { let databaseUrl = process.env.DATABASE_URL if (process.env.USE_DB_CONFIG !== 'true' && databaseUrl) return databaseUrl // load database secret - // FIXME config const secretArn = process.env.DB_SECRET_ARN const client = new SecretsManagerClient({}) const req = new GetSecretValueCommand({ SecretId: secretArn }) @@ -83,6 +83,12 @@ const loadDatabaseUrl = async (): Promise => { // construct database url databaseUrl = `postgresql://${username}:${password}@${host}:${port}/${dbname}` + + return databaseUrl +} + +const loadDatabaseUrl = async (): Promise => { + const databaseUrl = await _getDatabaseUrl() process.env.DATABASE_URL = databaseUrl return databaseUrl } diff --git a/stacks/database.ts b/stacks/database.ts index 0afc52f..ace951e 100644 --- a/stacks/database.ts +++ b/stacks/database.ts @@ -1,3 +1,5 @@ +import { APP_NAME } from '@common/index' +import { Duration, IAspect, RemovalPolicy } from 'aws-cdk-lib' import { ISecurityGroup, IVpc, Port, SecurityGroup } from 'aws-cdk-lib/aws-ec2' import { CfnFunction } from 'aws-cdk-lib/aws-lambda' import { @@ -9,25 +11,26 @@ import { DatabaseClusterEngine, IServerlessCluster, ParameterGroup, - ServerlessCluster, ServerlessClusterFromSnapshot, ServerlessClusterProps, SnapshotCredentials, } from 'aws-cdk-lib/aws-rds' import { ISecret, Secret } from 'aws-cdk-lib/aws-secretsmanager' import { Construct, IConstruct } from 'constructs' -import { App, Script, Function, Config, Stack, StackContext, use, RDS } from 'sst/constructs' import { config } from 'dotenv' -import { APP_NAME } from '@common/index' -import { Duration, IAspect, RemovalPolicy } from 'aws-cdk-lib' +import { App, Config, Function, Script, Stack, StackContext, use } from 'sst/constructs' import { Network } from 'stacks/network' import { IS_PRODUCTION } from './config' +import { Iam } from './iam' +import { Effect, Policy, PolicyStatement } from 'aws-cdk-lib/aws-iam' // if no parameter group specified, log queries that take at least this long export const logMinDurationStatementDefault = 90 // ms export function Database({ stack, app }: StackContext) { const net = use(Network) + const { defaultLambdaRole } = use(Iam) + const { vpc } = net const defaultDatabaseName = APP_NAME @@ -41,7 +44,8 @@ export function Database({ stack, app }: StackContext) { }) let db: DatabaseWithSecret | undefined = undefined - if (!process.env.CREATE_AURORA_DATABASE) return {} + const createDatabase = process.env.CREATE_AURORA_DATABASE === 'true' && !app.local + if (!createDatabase) return {} // database settings const dbProps: DatabaseProps & Partial = { @@ -147,7 +151,15 @@ export function Database({ stack, app }: StackContext) { }) } - // app.addDefaultFunctionPermissions([dbSecret, 'grantRead']); + if (dbSecret) { + app.addDefaultFunctionPermissions([ + new PolicyStatement({ + effect: Effect.ALLOW, + resources: [dbSecret.secretArn], + actions: ['secretsmanager:GetSecretValue'], + }), + ]) + } return { db, defaultDatabaseName, dbAccessSecurityGroup } } diff --git a/stacks/iam.ts b/stacks/iam.ts new file mode 100644 index 0000000..0a18848 --- /dev/null +++ b/stacks/iam.ts @@ -0,0 +1,16 @@ +import { Function, StackContext } from 'sst/constructs' + +export function Iam({ stack }: StackContext) { + // default role for lambda functions + // so we don't end up with >1000 roles + // HACK to make a role that inherits permissions/config from the app + // by making an empty function + // more info: https://discord.com/channels/983865673656705025/1027663092957581383 + const placeholderFn = new Function(stack, 'IamDefault', { + handler: 'backend/src/api/internalFunctions/empty.handler', + }) + + return { + defaultLambdaRole: placeholderFn.role!, + } +} diff --git a/stacks/index.ts b/stacks/index.ts index 8936366..fedded2 100644 --- a/stacks/index.ts +++ b/stacks/index.ts @@ -12,6 +12,7 @@ import { RestApi } from './restApi' import { Web } from './web' import { Aspects } from 'aws-cdk-lib' import { Secrets } from './secrets' +import { Iam } from './iam' // deal with dynamic imports of node built-ins (e.g. "crypto") // from https://github.com/evanw/esbuild/pull/2067#issuecomment-1073039746 @@ -44,6 +45,7 @@ export default function main(app: sst.App) { app .stack(Network) + .stack(Iam) .stack(Secrets) .stack(Dns) .stack(Layers) diff --git a/stacks/resources/migrationScript.ts b/stacks/resources/migrationScript.ts index 78d994c..b985b58 100644 --- a/stacks/resources/migrationScript.ts +++ b/stacks/resources/migrationScript.ts @@ -1,21 +1,21 @@ -import { RemovalPolicy } from 'aws-cdk-lib'; -import { IVpc } from 'aws-cdk-lib/aws-ec2'; -import { Construct } from 'constructs'; -import { App, Function, Script } from 'sst/constructs'; -import { PRISMA_VERSION } from '../layers'; -import { PrismaLayer } from './prismaLayer'; -import { RUN_DB_MIGRATIONS } from 'stacks/config'; +import { RemovalPolicy } from 'aws-cdk-lib' +import { IVpc } from 'aws-cdk-lib/aws-ec2' +import { Construct } from 'constructs' +import { App, Function, Script } from 'sst/constructs' +import { PRISMA_VERSION } from '../layers' +import { PrismaLayer } from './prismaLayer' +import { RUN_DB_MIGRATIONS } from 'stacks/config' interface DbMigrationScriptProps { - vpc?: IVpc; - dbSecretsArn: string; + vpc?: IVpc + dbSecretsArn: string } export class DbMigrationScript extends Construct { constructor(scope: Construct, id: string, { vpc, dbSecretsArn }: DbMigrationScriptProps) { - super(scope, id); + super(scope, id) - const app = App.of(scope) as App; + const app = App.of(scope) as App // lambda layer for migrations const migrationLayer = new PrismaLayer(this, 'PrismaMigrateLayer', { @@ -28,7 +28,7 @@ export class DbMigrationScript extends Construct { prismaEngines: ['schema-engine'], prismaModules: ['@prisma/engines', '@prisma/internals', '@prisma/client'], binaryTargets: ['linux-arm64-openssl-3.0.x'], - }); + }) const migrationFunction = new Function(this, 'MigrationScriptLambda', { vpc, @@ -45,7 +45,7 @@ export class DbMigrationScript extends Construct { nodejs: { format: 'cjs', - esbuild: { external: migrationLayer.externalModules || [] }, + esbuild: { external: migrationLayer.externalModules || [], target: 'node20' }, }, timeout: '13 minutes', environment: { @@ -54,14 +54,14 @@ export class DbMigrationScript extends Construct { // uncomment in an emergency if you get a lock error on prod // PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK: "true", }, - }); + }) // script to run migrations for us during deployment if (RUN_DB_MIGRATIONS) { new Script(this, 'MigrationScript', { onCreate: migrationFunction, onUpdate: migrationFunction, - }); + }) } } } diff --git a/stacks/secrets.ts b/stacks/secrets.ts index 6432034..135afb0 100644 --- a/stacks/secrets.ts +++ b/stacks/secrets.ts @@ -1,14 +1,15 @@ -import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; -import { Config, StackContext } from 'sst/constructs'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager' +import { Config, StackContext, use } from 'sst/constructs' +import { Iam } from './iam' export function Secrets({ stack }: StackContext) { - const secretsArn = process.env.SECRETS_ARN; + const secretsArn = process.env.SECRETS_ARN // needed for NEXTAUTH_SECRET env var since there is no way to provide it via SST Config - let secrets; + let secrets if (secretsArn) { // import - secrets = Secret.fromSecretCompleteArn(stack, 'Secrets', secretsArn); + secrets = Secret.fromSecretCompleteArn(stack, 'Secrets', secretsArn) } else { secrets = secretsArn ? Secret.fromSecretCompleteArn(stack, 'Secrets', secretsArn) @@ -20,11 +21,11 @@ export function Secrets({ stack }: StackContext) { generateStringKey: 'AUTH_SECRET', excludePunctuation: true, }, - }); + }) } // add more SST secrets here - const SECRET_1 = new Config.Secret(stack, 'SECRET_1'); + const SECRET_1 = new Config.Secret(stack, 'SECRET_1') - return { secrets, SECRET_1 }; + return { secrets, SECRET_1 } } diff --git a/web/next.config.mjs b/web/next.config.mjs index 4678774..ff5cb77 100644 --- a/web/next.config.mjs +++ b/web/next.config.mjs @@ -1,4 +1,48 @@ /** @type {import('next').NextConfig} */ -const nextConfig = {}; +const nextConfig = { + transpilePackages: ['@common'], -export default nextConfig; + // for open-next output + outputFileTracingRoot: path.join(__dirname, '../'), + // don't include dev deps in the deployed bundle + outputFileTracingExcludes: { + '*': [ + './**/.prisma/client/libquery_engine-darwin*', // prisma mac binary + './**/@swc/core-linux-x64-gnu*', + './**/@swc/core-linux-x64-musl*', + './**/@esbuild*', + './**/webpack*', + './**/rollup*', + './**/terser*', + './**/sharp*', + ], + }, + + images: { + remotePatterns: [ + { + protocol: 'https', + hostname: '**', + port: '', + pathname: '**', + }, + { + protocol: 'http', + hostname: 'localhost', + port: '6001', + pathname: '**', + }, + ], + minimumCacheTTL: 86400 * 365, // cache optimized images for a long time + }, + + // https://docs.sst.dev/constructs/NextjsSite#source-maps + webpack: (config, options) => { + if (!options.dev) { + config.devtool = 'source-map' + } + return config + }, +} + +export default nextConfig