diff --git a/docs/upload.md b/docs/upload.md new file mode 100644 index 00000000..61db2d7b --- /dev/null +++ b/docs/upload.md @@ -0,0 +1,179 @@ +# Upload + +The `upload` construct deploys a S3 bucket where you can upload files from the frontend. + +It also creates a Lambda function that will generate a temporary URL to upload to the S3 bucket. + +## Quick start + +```bash +serverless plugin install -n serverless-lift +``` + +```yaml +service: my-app +provider: + name: aws + +functions: + myFunction: + handler: src/index.handler + events: + - httpApi: '*' + +constructs: + upload: + type: upload + +plugins: + - serverless-lift +``` + +On `serverless deploy`, a S3 bucket will be created and the Lambda function will be attached to your API Gateway. + +## How it works + +The `upload` construct creates and configures the S3 bucket for the upload: + +- Files stored in the bucket are automatically encrypted (S3 takes care of encrypting and decrypting data on the fly, without change to our applications). +- Files are stored in a `tmp` folder and files are automatically deleted after 24 hours. +- Cross-Origin Resource Sharing (CORS) is configured to be reachable from a web browser. + +It also creates a Lambda function : + +- It is automatically attached to your API Gateway under the path `/upload-url` +- It requires to be called via a **POST** request containing a JSON body with the fields `fileName` and `contentType` +- It will generate the pre-signed URL that will be valid for 5 minutes +- It will return a JSON containing the `uploadUrl` and the `fileName` which is the path in the S3 bucket where the file will be stored + +**Warning:** because files are deleted from the bucket after 24 hours, your backend code +should move it if it needs to be stored permanently. This is done this way to avoid uploaded files that are never used, +such as a user that uploads a file but never submits the form. + +## How to use it in the browser + +Here is an example of how to use this construct with `fetch` + +```html + +... + +``` + +## Variables + +All upload constructs expose the following variables: + +- `bucketName`: the name of the deployed S3 bucket +- `bucketArn`: the ARN of the deployed S3 bucket + +This can be used to reference the bucket from Lambda functions, for example: + +```yaml +constructs: + upload: + type: upload + +functions: + myFunction: + handler: src/index.handler + environment: + UPLOAD_BUCKET_NAME: ${construct:upload.bucketName} +``` + +_How it works: the `${construct:upload.bucketName}` variable will automatically be replaced with a CloudFormation reference to the S3 bucket._ + +This is useful to process the uploaded files. Remember that the files will be automatically deleted after 24 hours. + +## Permissions + +By default, all the Lambda functions deployed in the same `serverless.yml` file **will be allowed to read/write into the upload bucket**. + +In the example below, there are no IAM permissions to set up: `myFunction` will be allowed to read and write into the `upload` bucket. + +```yaml +constructs: + upload: + type: upload + +functions: + myFunction: + handler: src/index.handler + environment: + UPLOAD_BUCKET_NAME: ${construct:avatars.bucketName} +``` + +Automatic permissions can be disabled: [read more about IAM permissions](permissions.md). + +## Configuration reference + +### API Gateway + +API Gateway provides 2 versions of APIs: + +- v1: REST API +- v2: HTTP API, the fastest and cheapest + +By default, the `upload` construct supports v2 HTTP APIs. + +If your Lambda functions uses `http` events (v1 REST API) instead of `httpApi` events (v2 HTTP API), use the `apiGateway: "rest"` option: + +```yaml +constructs: + upload: + type: upload + apiGateway: 'rest' # either "rest" (v1) or "http" (v2, the default) + +functions: + v1: + handler: foo.handler + events: + - http: 'GET /' # REST API (v1) + v2: + handler: bar.handler + events: + - httpApi: 'GET /' # HTTP API (v2) +``` + +### Encryption + +By default, files are encrypted using [the default S3 encryption mechanism](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html) (free). + +Alternatively, for example to comply with certain policies, it is possible to [use KMS](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html): + +```yaml +constructs: + upload: + # ... + encryption: kms +``` + +### More options + +Looking for more options in the construct configuration? [Open a GitHub issue](https://github.com/getlift/lift/issues/new). diff --git a/package.json b/package.json index 778660d0..735d374d 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "repository": "https://github.com/getlift/lift", "description": "Lift", "dependencies": { + "@aws-cdk/aws-apigateway": "^1.110.1", "@aws-cdk/aws-apigatewayv2": "^1.110.1", "@aws-cdk/aws-apigatewayv2-integrations": "^1.110.1", "@aws-cdk/aws-certificatemanager": "^1.110.1", diff --git a/src/constructs/aws/Storage.ts b/src/constructs/aws/Storage.ts index 276c2ed5..45ae0737 100644 --- a/src/constructs/aws/Storage.ts +++ b/src/constructs/aws/Storage.ts @@ -10,7 +10,6 @@ const STORAGE_DEFINITION = { type: "object", properties: { type: { const: "storage" }, - archive: { type: "number", minimum: 30 }, encryption: { anyOf: [{ const: "s3" }, { const: "kms" }], }, @@ -19,7 +18,6 @@ const STORAGE_DEFINITION = { } as const; const STORAGE_DEFAULTS: Required> = { type: "storage", - archive: 45, encryption: "s3", }; diff --git a/src/constructs/aws/Upload.ts b/src/constructs/aws/Upload.ts new file mode 100644 index 00000000..17e56f09 --- /dev/null +++ b/src/constructs/aws/Upload.ts @@ -0,0 +1,179 @@ +import { BlockPublicAccess, Bucket, BucketEncryption, HttpMethods } from "@aws-cdk/aws-s3"; +import type { Construct as CdkConstruct } from "@aws-cdk/core"; +import { CfnOutput, Duration, Fn, Stack } from "@aws-cdk/core"; +import { Code, Function as LambdaFunction, Runtime } from "@aws-cdk/aws-lambda"; +import type { FromSchema } from "json-schema-to-ts"; +import type { AwsProvider } from "@lift/providers"; +import { AwsConstruct } from "@lift/constructs/abstracts"; +import type { IHttpApi } from "@aws-cdk/aws-apigatewayv2"; +import { HttpApi, HttpMethod, HttpRoute, HttpRouteKey } from "@aws-cdk/aws-apigatewayv2"; +import type { IRestApi, Resource } from "@aws-cdk/aws-apigateway"; +import { LambdaIntegration, RestApi } from "@aws-cdk/aws-apigateway"; +import { LambdaProxyIntegration } from "@aws-cdk/aws-apigatewayv2-integrations"; +import { Role } from "@aws-cdk/aws-iam"; +import { PolicyStatement } from "../../CloudFormation"; + +const UPLOAD_DEFINITION = { + type: "object", + properties: { + type: { const: "upload" }, + apiGateway: { enum: ["http", "rest"] }, + encryption: { + anyOf: [{ const: "s3" }, { const: "kms" }], + }, + }, + additionalProperties: false, +} as const; +const UPLOAD_DEFAULTS: Required> = { + type: "upload", + encryption: "s3", + apiGateway: "http", +}; + +type Configuration = FromSchema; + +export class Upload extends AwsConstruct { + public static type = "upload"; + public static schema = UPLOAD_DEFINITION; + + private readonly bucket: Bucket; + private readonly bucketNameOutput: CfnOutput; + private function: LambdaFunction; + private httpApi: IHttpApi | undefined; + private route: HttpRoute | undefined; + private restApi: RestApi | undefined; + + constructor(scope: CdkConstruct, id: string, configuration: Configuration, private provider: AwsProvider) { + super(scope, id); + + const resolvedConfiguration = Object.assign({}, UPLOAD_DEFAULTS, configuration); + + const encryptionOptions = { + s3: BucketEncryption.S3_MANAGED, + kms: BucketEncryption.KMS_MANAGED, + }; + + this.bucket = new Bucket(this, "Bucket", { + encryption: encryptionOptions[resolvedConfiguration.encryption], + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + cors: [ + { + allowedMethods: [HttpMethods.PUT], + allowedOrigins: ["*"], + allowedHeaders: ["*"], + }, + ], + lifecycleRules: [ + { + expiration: Duration.days(1), + }, + ], + }); + + this.bucketNameOutput = new CfnOutput(this, "BucketName", { + value: this.bucket.bucketName, + }); + + this.function = new LambdaFunction(this, "Function", { + code: Code.fromInline(this.createFunctionCode()), + handler: "index.handler", + runtime: Runtime.NODEJS_12_X, + environment: { + LIFT_UPLOAD_BUCKET_NAME: this.bucket.bucketName, + }, + role: Role.fromRoleArn( + this, + "LambdaRole", + Fn.getAtt(this.provider.naming.getRoleLogicalId(), "Arn").toString() + ), + }); + + if (resolvedConfiguration.apiGateway === "http") { + this.provider.enableHttpApiCors(); + this.httpApi = HttpApi.fromHttpApiAttributes(this, "HttpApi", { + httpApiId: Fn.ref(this.provider.naming.getHttpApiLogicalId()), + }); + + this.route = new HttpRoute(this, "Route", { + httpApi: this.httpApi, + integration: new LambdaProxyIntegration({ + handler: this.function, + }), + routeKey: HttpRouteKey.with("/upload-url", HttpMethod.POST), + }); + } + + if (resolvedConfiguration.apiGateway === "rest") { + this.restApi = RestApi.fromRestApiAttributes(this, "RestApi", { + restApiId: Fn.ref(this.provider.naming.getRestApiLogicalId()), + rootResourceId: Fn.getAtt(this.provider.naming.getRestApiLogicalId(), "RootResourceId").toString(), + }) as RestApi; + + const resource: Resource = this.restApi.root.addResource("upload-url"); + resource.addCorsPreflight({ + allowHeaders: ["*"], + allowMethods: ["POST"], + allowOrigins: ["*"], + }); + resource.addMethod("POST", new LambdaIntegration(this.function)); + } + } + + variables(): Record { + return { + bucketArn: this.bucket.bucketArn, + bucketName: this.bucket.bucketName, + }; + } + + permissions(): PolicyStatement[] { + return [ + new PolicyStatement( + ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], + [this.bucket.bucketArn, Stack.of(this).resolve(Fn.join("/", [this.bucket.bucketArn, "*"]))] + ), + ]; + } + + outputs(): Record Promise> { + return { + bucketName: () => this.getBucketName(), + }; + } + + async getBucketName(): Promise { + return this.provider.getStackOutput(this.bucketNameOutput); + } + + private createFunctionCode(): string { + return ` +const AWS = require('aws-sdk'); +const crypto = require("crypto"); +const s3 = new AWS.S3(); + +exports.handler = async (event) => { + const body = JSON.parse(event.body); + const fileName = \`tmp/\${crypto.randomBytes(5).toString('hex')}-\${body.fileName}\`; + + const url = s3.getSignedUrl('putObject', { + Bucket: process.env.LIFT_UPLOAD_BUCKET_NAME, + Key: fileName, + ContentType: body.contentType, + Expires: 60 * 5, + }); + + return { + body: JSON.stringify({ + fileName: fileName, + uploadUrl: url, + }), + headers: { + "Access-Control-Allow-Origin": event.headers.origin, + }, + statusCode: 200 + }; +} + `; + } +} diff --git a/src/providers/AwsProvider.ts b/src/providers/AwsProvider.ts index bc02d444..a11566a6 100644 --- a/src/providers/AwsProvider.ts +++ b/src/providers/AwsProvider.ts @@ -13,10 +13,12 @@ import { Vpc, Webhook, } from "@lift/constructs/aws"; +import { Upload } from "@lift/constructs/aws/Upload"; import { getStackOutput } from "../CloudFormation"; import type { CloudformationTemplate, Provider as LegacyAwsProvider, Serverless } from "../types/serverless"; import { awsRequest } from "../classes/aws"; import ServerlessError from "../utils/error"; +import { HttpApiCorsConfig } from "../types/serverless"; const AWS_DEFINITION = { type: "object", @@ -63,6 +65,7 @@ export class AwsProvider implements ProviderInterface { getLambdaLogicalId: (functionName: string) => string; getRestApiLogicalId: () => string; getHttpApiLogicalId: () => string; + getRoleLogicalId: () => string; }; constructor(private readonly serverless: Serverless) { @@ -136,6 +139,22 @@ export class AwsProvider implements ProviderInterface { return this.serverless.service.provider.vpc ?? null; } + /** + * This enables HTTP API CORS preflight requests if the user + * didn't do it explicitly. + */ + enableHttpApiCors(): void { + if (this.serverless.service.provider.httpApi === undefined) { + this.serverless.service.provider.httpApi = { + cors: true, + }; + } + + if (this.serverless.service.provider.httpApi.cors === undefined) { + this.serverless.service.provider.httpApi.cors = true; + } + } + /** * Resolves the value of a CloudFormation stack output. */ @@ -172,5 +191,6 @@ AwsProvider.registerConstructs( StaticWebsite, Vpc, DatabaseDynamoDBSingleTable, - ServerSideWebsite + ServerSideWebsite, + Upload ); diff --git a/src/types/serverless.ts b/src/types/serverless.ts index ae6a7a38..86e05d04 100644 --- a/src/types/serverless.ts +++ b/src/types/serverless.ts @@ -27,6 +27,7 @@ export type Provider = { getRestApiLogicalId: () => string; getHttpApiLogicalId: () => string; getCompiledTemplateFileName: () => string; + getRoleLogicalId: () => string; }; getRegion: () => string; /** diff --git a/test/unit/upload.test.ts b/test/unit/upload.test.ts new file mode 100644 index 00000000..19075e30 --- /dev/null +++ b/test/unit/upload.test.ts @@ -0,0 +1,161 @@ +import { get } from "lodash"; +import { baseConfig, runServerless } from "../utils/runServerless"; + +describe("upload", () => { + it("should create all required resources with HTTP API", async () => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + }, + }, + }), + }); + + const bucket = computeLogicalId("upload", "Bucket"); + const bucketPolicy = computeLogicalId("upload", "Bucket", "Policy"); + const uploadFunction = computeLogicalId("upload", "Function"); + const httpApiRoute = computeLogicalId("upload", "Route"); + + expect(Object.keys(cfTemplate.Resources)).toStrictEqual([ + "ServerlessDeploymentBucket", + "ServerlessDeploymentBucketPolicy", + bucket, + bucketPolicy, + uploadFunction, + "uploadRouteuploadRoute2545F0B8PermissionCB079AC2", + "uploadRouteHttpIntegration02104492e88c1940a1c8d0dbac532c8091C83E5A", + httpApiRoute, + ]); + }); + it("should create all required resources with REST API", async () => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + apiGateway: "rest", + }, + }, + }), + }); + + const bucket = computeLogicalId("upload", "Bucket"); + const bucketPolicy = computeLogicalId("upload", "Bucket", "Policy"); + const uploadFunction = computeLogicalId("upload", "Function"); + + expect(Object.keys(cfTemplate.Resources)).toStrictEqual([ + "ServerlessDeploymentBucket", + "ServerlessDeploymentBucketPolicy", + bucket, + bucketPolicy, + uploadFunction, + "uploadRestApiuploadurl2A547A06", + "uploadRestApiuploadurlOPTIONS1BD5E4F2", + "uploadRestApiuploadurlPOSTApiPermissionuploadRestApiC195B6D4POSTuploadurlE1E5BEF5", + "uploadRestApiuploadurlPOSTApiPermissionTestuploadRestApiC195B6D4POSTuploadurl7144EDCF", + "uploadRestApiuploadurlPOST347E9EEB", + ]); + }); + + it("should delete files after 1 day", async () => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + }, + }, + }), + }); + + expect( + get(cfTemplate.Resources[computeLogicalId("upload", "Bucket")].Properties, "LifecycleConfiguration") + ).toStrictEqual({ + Rules: [ + { + ExpirationInDays: 1, + Status: "Enabled", + }, + ], + }); + }); + + it("should enable CORS on the bucket", async () => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + }, + }, + }), + }); + + expect( + get(cfTemplate.Resources[computeLogicalId("upload", "Bucket")].Properties, "CorsConfiguration") + ).toStrictEqual({ + CorsRules: [ + { + AllowedHeaders: ["*"], + AllowedMethods: ["PUT"], + AllowedOrigins: ["*"], + }, + ], + }); + }); + + it("should enable block public access on the bucket", async () => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + }, + }, + }), + }); + + expect( + get(cfTemplate.Resources[computeLogicalId("upload", "Bucket")].Properties, "PublicAccessBlockConfiguration") + ).toStrictEqual({ + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + }); + }); + + test.each([ + ["s3", "AES256"], + ["kms", "aws:kms"], + ])("should allow %p encryption", async (encryption, expectedSSEAlgorithm) => { + const { cfTemplate, computeLogicalId } = await runServerless({ + command: "package", + config: Object.assign(baseConfig, { + constructs: { + upload: { + type: "upload", + encryption: encryption, + }, + }, + }), + }); + + expect(cfTemplate.Resources[computeLogicalId("upload", "Bucket")].Properties).toMatchObject({ + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { SSEAlgorithm: expectedSSEAlgorithm }, + }, + ], + }, + }); + }); +});