From 0dbcc874206d8d87c2c1da1773e5390968dfa527 Mon Sep 17 00:00:00 2001 From: Jack Williams Date: Thu, 12 Dec 2024 15:50:58 +0000 Subject: [PATCH] Experimental `getAsyncCtx()` (#776) ## Summary Adds a new `getAsyncCtx()` function exported from `"inngest/experimental"` that will attempt to retrieve the function input arguments. - Uses `@inngest/test@workspace:^` internally, exclusively for the `inngest` package. This can be a bit weird, as `inngest` is a peer dep of `@inngest/test` so typing can be flaky. - Altered exports of `@inngest/test` to have simpler compilation and publishing. - Added the experimental `getAsyncCtx()` ```ts import { getAsyncCtx } from "inngest/experimental"; const ctx = await getAsyncCtx(); ``` ### Questions - [x] Should we remove the store for a particular execution context when the run completes? No. It will still be garbage collected and info may still be useful. ## Checklist - [ ] ~Added a [docs PR](https://github.com/inngest/website) that references this PR~ N/A Experimental for now - [x] Added unit/integration tests - [x] Added changesets if applicable --- .changeset/cyan-sheep-train.md | 5 + .changeset/dull-students-wink.md | 5 + .changeset/silver-dolls-mix.md | 11 ++ .github/actions/setup-and-build/action.yml | 8 +- packages/inngest/jest.config.js | 2 +- packages/inngest/jsr.json | 3 +- packages/inngest/package.json | 6 + .../src/components/execution/als.test.ts | 155 ++++++++++++++++++ .../inngest/src/components/execution/als.ts | 52 ++++++ .../inngest/src/components/execution/v1.ts | 71 ++++---- packages/inngest/src/experimental.ts | 2 + packages/middleware-validation/package.json | 2 +- packages/test/package.json | 15 +- pnpm-lock.yaml | 13 +- 14 files changed, 300 insertions(+), 50 deletions(-) create mode 100644 .changeset/cyan-sheep-train.md create mode 100644 .changeset/dull-students-wink.md create mode 100644 .changeset/silver-dolls-mix.md create mode 100644 packages/inngest/src/components/execution/als.test.ts create mode 100644 packages/inngest/src/components/execution/als.ts create mode 100644 packages/inngest/src/experimental.ts diff --git a/.changeset/cyan-sheep-train.md b/.changeset/cyan-sheep-train.md new file mode 100644 index 000000000..ca59b095d --- /dev/null +++ b/.changeset/cyan-sheep-train.md @@ -0,0 +1,5 @@ +--- +"inngest": patch +--- + +Use `@inngest/test@workspace:^` internally for testing diff --git a/.changeset/dull-students-wink.md b/.changeset/dull-students-wink.md new file mode 100644 index 000000000..48adece99 --- /dev/null +++ b/.changeset/dull-students-wink.md @@ -0,0 +1,5 @@ +--- +"@inngest/test": patch +--- + +Altered exports to now be namespaced by `./dist/`; if you have directly imported files from `@inngest/test`, you may need to change the imports diff --git a/.changeset/silver-dolls-mix.md b/.changeset/silver-dolls-mix.md new file mode 100644 index 000000000..a12ddcb67 --- /dev/null +++ b/.changeset/silver-dolls-mix.md @@ -0,0 +1,11 @@ +--- +"inngest": minor +--- + +Add experimental `getAsyncCtx()`, allowing the retrieval of a run's input (`event`, `step`, `runId`, etc) from the relevant async chain. + +```ts +import { getAsyncCtx } from "inngest/experimental"; + +const ctx = await getAsyncCtx(); +``` diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml index 8ac71366c..680d2e23b 100644 --- a/.github/actions/setup-and-build/action.yml +++ b/.github/actions/setup-and-build/action.yml @@ -36,7 +36,13 @@ runs: if: ${{ inputs.install-dependencies == 'true' }} run: pnpm install shell: bash - working-directory: ${{ inputs.working-directory }}/packages/inngest + working-directory: ${{ inputs.working-directory }} + + - name: Build test dependencies + if: ${{ inputs.install-dependencies == 'true' }} + run: pnpm run build + shell: bash + working-directory: ${{ inputs.working-directory }}/packages/test - name: Build if: ${{ inputs.build == 'true' }} diff --git a/packages/inngest/jest.config.js b/packages/inngest/jest.config.js index 26469b956..b36cf164c 100644 --- a/packages/inngest/jest.config.js +++ b/packages/inngest/jest.config.js @@ -6,7 +6,7 @@ module.exports = { roots: ["/src"], moduleNameMapper: { "(\\..+)\\.js": "$1", - inngest: "/src", + "^inngest$": "/src", "^@local$": "/src", "^@local/(.*)": "/src/$1", "^@local/(.*)\\.js": "/src/$1", diff --git a/packages/inngest/jsr.json b/packages/inngest/jsr.json index dbffb144e..ca315c3e4 100644 --- a/packages/inngest/jsr.json +++ b/packages/inngest/jsr.json @@ -17,6 +17,7 @@ ], "exports": { ".": "./src/index.ts", + "./experimental": "./src/experimental.ts", "./astro": "./src/astro.ts", "./bun": "./src/bun.ts", "./cloudflare": "./src/cloudflare.ts", @@ -37,4 +38,4 @@ "./nitro": "./src/nitro.ts", "./types": "./src/types.ts" } -} \ No newline at end of file +} diff --git a/packages/inngest/package.json b/packages/inngest/package.json index e4bbf706c..3d33bcdfb 100644 --- a/packages/inngest/package.json +++ b/packages/inngest/package.json @@ -37,6 +37,11 @@ "import": "./index.js", "types": "./index.d.ts" }, + "./experimental": { + "require": "./experimental.js", + "import": "./experimental.js", + "types": "./experimental.d.ts" + }, "./astro": { "require": "./astro.js", "import": "./astro.js", @@ -199,6 +204,7 @@ "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", "@inngest/eslint-plugin-internal": "workspace:^", + "@inngest/test": "workspace:^", "@jest/globals": "^29.5.0", "@shopify/jest-koa-mocks": "^5.1.1", "@sveltejs/kit": "^1.27.3", diff --git a/packages/inngest/src/components/execution/als.test.ts b/packages/inngest/src/components/execution/als.test.ts new file mode 100644 index 000000000..d31af77d1 --- /dev/null +++ b/packages/inngest/src/components/execution/als.test.ts @@ -0,0 +1,155 @@ +import { InngestTestEngine } from "@inngest/test"; +import { type AsyncContext } from "@local/components/execution/als"; + +describe("getAsyncLocalStorage", () => { + const warningSpy = jest.spyOn(console, "warn"); + + afterEach(() => { + jest.unmock("node:async_hooks"); + jest.resetModules(); + }); + + test("should return an `AsyncLocalStorageIsh`", async () => { + const mod = await import("@local/components/execution/als"); + const als = await mod.getAsyncLocalStorage(); + + expect(als).toBeDefined(); + expect(als.getStore).toBeDefined(); + expect(als.run).toBeDefined(); + }); + + test("should return the same instance of `AsyncLocalStorageIsh`", async () => { + const mod = await import("@local/components/execution/als"); + + const als1p = mod.getAsyncLocalStorage(); + const als2p = mod.getAsyncLocalStorage(); + + const als1 = await als1p; + const als2 = await als2p; + + expect(als1).toBe(als2); + }); + + test("should return `undefined` if node:async_hooks is not supported", async () => { + jest.mock("node:async_hooks", () => { + throw new Error("import failed"); + }); + + const mod = await import("@local/components/execution/als"); + const als = await mod.getAsyncLocalStorage(); + + expect(warningSpy).toHaveBeenCalledWith( + expect.stringContaining( + "node:async_hooks is not supported in this runtime" + ) + ); + + expect(als).toBeDefined(); + expect(als.getStore()).toBeUndefined(); + expect(als.run).toBeDefined(); + }); +}); + +describe("getAsyncCtx", () => { + const wait = async () => { + await new Promise((resolve) => setTimeout(resolve)); + await new Promise((resolve) => process.nextTick(resolve)); + }; + + afterEach(() => { + jest.unmock("node:async_hooks"); + jest.resetModules(); + }); + + test("should return `undefined` outside of an Inngest async context", async () => { + const mod = await import("@local/components/execution/als"); + const store = await mod.getAsyncCtx(); + + expect(store).toBeUndefined(); + }); + + test("should return the input context during execution", async () => { + const { Inngest } = await import("@local"); + const mod = await import("@local/experimental"); + + const inngest = new Inngest({ id: "test" }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let resolve: (value: any) => void | PromiseLike; + const externalP = new Promise((r) => { + resolve = r; + }); + + let internalRunId: string | undefined; + + const fn = inngest.createFunction( + { id: "test" }, + { event: "" }, + ({ runId }) => { + internalRunId = runId; + + void wait() + .then(() => mod.getAsyncCtx()) + .then(resolve); + + return "done"; + } + ); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + const t = new InngestTestEngine({ function: fn as any }); + + const { result } = await t.execute(); + + expect(result).toBe("done"); + expect(internalRunId).toBeTruthy(); + + const store = await externalP; + expect(store).toBeDefined(); + expect(store?.ctx.runId).toBe(internalRunId); + }); + + test("should return `undefined` if node:async_hooks is not supported", async () => { + jest.mock("node:async_hooks", () => { + throw new Error("import failed"); + }); + + const { Inngest } = await import("@local"); + const mod = await import("@local/experimental"); + + const inngest = new Inngest({ id: "test" }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let resolve: (value: any) => void | PromiseLike; + const externalP = new Promise((r) => { + resolve = r; + }); + + let internalRunId: string | undefined; + + const fn = inngest.createFunction( + { id: "test" }, + { event: "" }, + ({ runId }) => { + internalRunId = runId; + + void wait() + .then(() => mod.getAsyncCtx()) + .then(resolve); + + return "done"; + } + ); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + const t = new InngestTestEngine({ function: fn as any }); + + const { result } = await t.execute(); + + expect(result).toBe("done"); + expect(internalRunId).toBeTruthy(); + + const store = await externalP; + expect(store).toBeUndefined(); + }); +}); diff --git a/packages/inngest/src/components/execution/als.ts b/packages/inngest/src/components/execution/als.ts new file mode 100644 index 000000000..6476950aa --- /dev/null +++ b/packages/inngest/src/components/execution/als.ts @@ -0,0 +1,52 @@ +import { type Context } from "../../types.js"; + +export interface AsyncContext { + ctx: Context.Any; +} + +/** + * A type that represents a partial, runtime-agnostic interface of + * `AsyncLocalStorage`. + */ +type AsyncLocalStorageIsh = { + getStore: () => AsyncContext | undefined; + run: (store: AsyncContext, fn: () => R) => R; +}; + +/** + * A local-only variable to store the async local storage instance. + */ +let als: Promise | undefined; + +/** + * Retrieve the async context for the current execution. + */ +export const getAsyncCtx = async (): Promise => { + return getAsyncLocalStorage().then((als) => als.getStore()); +}; + +/** + * Get a singleton instance of `AsyncLocalStorage` used to store and retrieve + * async context for the current execution. + */ +export const getAsyncLocalStorage = async (): Promise => { + // eslint-disable-next-line @typescript-eslint/no-misused-promises, no-async-promise-executor + als ??= new Promise(async (resolve) => { + try { + const { AsyncLocalStorage } = await import("node:async_hooks"); + + resolve(new AsyncLocalStorage()); + } catch (err) { + console.warn( + "node:async_hooks is not supported in this runtime. Experimental async context is disabled." + ); + + resolve({ + getStore: () => undefined, + run: (_, fn) => fn(), + }); + } + }); + + return als; +}; diff --git a/packages/inngest/src/components/execution/v1.ts b/packages/inngest/src/components/execution/v1.ts index 145be5145..f0f6ceed8 100644 --- a/packages/inngest/src/components/execution/v1.ts +++ b/packages/inngest/src/components/execution/v1.ts @@ -48,6 +48,7 @@ import { type InngestExecutionOptions, type MemoizedOp, } from "./InngestExecution.js"; +import { getAsyncLocalStorage } from "./als.js"; export const createV1InngestExecution: InngestExecutionFactory = (options) => { return new V1InngestExecution(options); @@ -459,44 +460,48 @@ class V1InngestExecution extends InngestExecution implements IInngestExecution { * and middleware hooks where appropriate. */ private async startExecution(): Promise { - /** - * Mutate input as neccessary based on middleware. - */ - await this.transformInput(); + return getAsyncLocalStorage().then((als) => + als.run({ ctx: this.fnArg }, async (): Promise => { + /** + * Mutate input as neccessary based on middleware. + */ + await this.transformInput(); - /** - * Start the timer to time out the run if needed. - */ - void this.timeout?.start(); + /** + * Start the timer to time out the run if needed. + */ + void this.timeout?.start(); - await this.state.hooks?.beforeMemoization?.(); + await this.state.hooks?.beforeMemoization?.(); - /** - * If we had no state to begin with, immediately end the memoization phase. - */ - if (this.state.allStateUsed()) { - await this.state.hooks?.afterMemoization?.(); - await this.state.hooks?.beforeExecution?.(); - } + /** + * If we had no state to begin with, immediately end the memoization phase. + */ + if (this.state.allStateUsed()) { + await this.state.hooks?.afterMemoization?.(); + await this.state.hooks?.beforeExecution?.(); + } - /** - * Trigger the user's function. - */ - runAsPromise(() => this.userFnToRun(this.fnArg)) - // eslint-disable-next-line @typescript-eslint/no-misused-promises - .finally(async () => { - await this.state.hooks?.afterMemoization?.(); - await this.state.hooks?.beforeExecution?.(); - await this.state.hooks?.afterExecution?.(); - }) - .then((data) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - this.state.setCheckpoint({ type: "function-resolved", data }); + /** + * Trigger the user's function. + */ + runAsPromise(() => this.userFnToRun(this.fnArg)) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .finally(async () => { + await this.state.hooks?.afterMemoization?.(); + await this.state.hooks?.beforeExecution?.(); + await this.state.hooks?.afterExecution?.(); + }) + .then((data) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + this.state.setCheckpoint({ type: "function-resolved", data }); + }) + .catch((error) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + this.state.setCheckpoint({ type: "function-rejected", error }); + }); }) - .catch((error) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - this.state.setCheckpoint({ type: "function-rejected", error }); - }); + ); } /** diff --git a/packages/inngest/src/experimental.ts b/packages/inngest/src/experimental.ts new file mode 100644 index 000000000..32eee2377 --- /dev/null +++ b/packages/inngest/src/experimental.ts @@ -0,0 +1,2 @@ +export { getAsyncCtx } from "./components/execution/als.js"; +export type { AsyncContext } from "./components/execution/als.js"; diff --git a/packages/middleware-validation/package.json b/packages/middleware-validation/package.json index 87c6e815b..50825f422 100644 --- a/packages/middleware-validation/package.json +++ b/packages/middleware-validation/package.json @@ -45,7 +45,7 @@ }, "devDependencies": { "@eslint/js": "^9.7.0", - "@inngest/test": "0.1.1-pr-741.0", + "@inngest/test": "^0.1.3", "@types/eslint__js": "^8.42.3", "@types/jest": "^29.5.14", "eslint": "^8.30.0", diff --git a/packages/test/package.json b/packages/test/package.json index 3646f4b2f..b4ff7d6e6 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -2,27 +2,26 @@ "name": "@inngest/test", "version": "0.1.3", "description": "Tooling for testing Inngest functions.", - "main": "./index.js", - "types": "./index.d.ts", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "publishConfig": { "registry": "https://registry.npmjs.org" }, "scripts": { "test": "jest", - "build": "pnpm run build:clean && pnpm run build:tsc && pnpm run build:copy", + "build": "pnpm run build:clean && pnpm run build:tsc", "build:clean": "rm -rf ./dist", "build:tsc": "tsc --project tsconfig.build.json", - "build:copy": "cp package.json LICENSE.md README.md CHANGELOG.md dist", "pack": "pnpm run build && yarn pack --verbose --frozen-lockfile --filename inngest-test.tgz --cwd dist", - "postversion": "pnpm run build && pnpm run build:copy", + "postversion": "pnpm run build", "release": "DIST_DIR=dist node ../../scripts/release/publish.js && pnpm dlx jsr publish --allow-slow-types --allow-dirty", "release:version": "node ../../scripts/release/jsrVersion.js" }, "exports": { ".": { - "require": "./index.js", - "import": "./index.js", - "types": "./index.d.ts" + "require": "./dist/index.js", + "import": "./dist/index.js", + "types": "./dist/index.d.ts" } }, "keywords": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8ca0966c..85decb382 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -97,6 +97,9 @@ importers: '@inngest/eslint-plugin-internal': specifier: workspace:^ version: link:../eslint-plugin-internal + '@inngest/test': + specifier: workspace:^ + version: link:../test '@jest/globals': specifier: ^29.5.0 version: 29.5.0 @@ -313,8 +316,8 @@ importers: specifier: ^9.7.0 version: 9.7.0 '@inngest/test': - specifier: 0.1.1-pr-741.0 - version: 0.1.1-pr-741.0(@sveltejs/kit@1.27.3(svelte@4.2.5)(vite@4.5.3(@types/node@20.14.8)))(@vercel/node@2.15.9)(aws-lambda@1.0.7)(express@4.19.2)(fastify@4.21.0)(h3@1.8.1)(hono@4.2.7)(koa@2.14.2)(next@13.5.4(@babel/core@7.23.6)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(typescript@5.6.3) + specifier: ^0.1.3 + version: 0.1.3(@sveltejs/kit@1.27.3(svelte@4.2.5)(vite@4.5.3(@types/node@20.14.8)))(@vercel/node@2.15.9)(aws-lambda@1.0.7)(express@4.19.2)(fastify@4.21.0)(h3@1.8.1)(hono@4.2.7)(koa@2.14.2)(next@13.5.4(@babel/core@7.23.6)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(typescript@5.6.3) '@types/eslint__js': specifier: ^8.42.3 version: 8.42.3 @@ -1003,8 +1006,8 @@ packages: resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} deprecated: Use @eslint/object-schema instead - '@inngest/test@0.1.1-pr-741.0': - resolution: {integrity: sha512-qqgGcxjxdFOHeJzfNhuAOVMTd7WQNQ62TB74/WVR4Ul5DlmDFpaC/A7jZp+fWAyD2QIJYl6qFcvevOLKY1I+wQ==} + '@inngest/test@0.1.3': + resolution: {integrity: sha512-3iwhqXs4Z8reWMmbOMObZ9nIfIDzTwpkDPf6L86746hs/rkOy+OZpagKosQZLc0JxxiSFzQhrgCBDtrYJoMIBg==} '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} @@ -5744,7 +5747,7 @@ snapshots: '@humanwhocodes/object-schema@2.0.1': {} - '@inngest/test@0.1.1-pr-741.0(@sveltejs/kit@1.27.3(svelte@4.2.5)(vite@4.5.3(@types/node@20.14.8)))(@vercel/node@2.15.9)(aws-lambda@1.0.7)(express@4.19.2)(fastify@4.21.0)(h3@1.8.1)(hono@4.2.7)(koa@2.14.2)(next@13.5.4(@babel/core@7.23.6)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(typescript@5.6.3)': + '@inngest/test@0.1.3(@sveltejs/kit@1.27.3(svelte@4.2.5)(vite@4.5.3(@types/node@20.14.8)))(@vercel/node@2.15.9)(aws-lambda@1.0.7)(express@4.19.2)(fastify@4.21.0)(h3@1.8.1)(hono@4.2.7)(koa@2.14.2)(next@13.5.4(@babel/core@7.23.6)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(typescript@5.6.3)': dependencies: inngest: 3.25.1(@sveltejs/kit@1.27.3(svelte@4.2.5)(vite@4.5.3(@types/node@20.14.8)))(@vercel/node@2.15.9)(aws-lambda@1.0.7)(express@4.19.2)(fastify@4.21.0)(h3@1.8.1)(hono@4.2.7)(koa@2.14.2)(next@13.5.4(@babel/core@7.23.6)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(typescript@5.6.3) tinyspy: 3.0.2