From a0e0b3bc6f06bdb6122619994370081543af5ebd Mon Sep 17 00:00:00 2001 From: Dmitry Baev Date: Wed, 25 Sep 2024 14:51:46 +0100 Subject: [PATCH] fix(allure-jest): add support for soft assertions (fixes #1152, via #1154) --- .../allure-jest/src/environmentFactory.ts | 135 +++++++++----- packages/allure-jest/test/spec/hooks.test.ts | 165 +++++++++++++++++- .../allure-jest/test/spec/skipped.test.ts | 20 ++- .../allure-jest/test/spec/snapshot.test.ts | 38 ++++ packages/allure-jest/test/spec/todo.test.ts | 13 +- 5 files changed, 317 insertions(+), 54 deletions(-) create mode 100644 packages/allure-jest/test/spec/snapshot.test.ts diff --git a/packages/allure-jest/src/environmentFactory.ts b/packages/allure-jest/src/environmentFactory.ts index 788b13fe8..53c76e89e 100644 --- a/packages/allure-jest/src/environmentFactory.ts +++ b/packages/allure-jest/src/environmentFactory.ts @@ -3,7 +3,7 @@ import type { Circus } from "@jest/types"; import { relative } from "node:path"; import { env } from "node:process"; import * as allure from "allure-js-commons"; -import { Stage, Status } from "allure-js-commons"; +import { Stage, Status, type StatusDetails, type TestResult } from "allure-js-commons"; import type { RuntimeMessage } from "allure-js-commons/sdk"; import { getMessageAndTraceFromError, getStatusFromError } from "allure-js-commons/sdk"; import type { TestPlanV1 } from "allure-js-commons/sdk"; @@ -94,13 +94,10 @@ const createJestEnvironment = (Base: T): T => this.#handleSuiteEnd(); break; case "test_start": - this.#handleTestStart(event.test); - break; - case "test_done": - this.#handleTestDone(); + this.#handleTestScopeStart(); break; - case "test_todo": - this.#handleTestTodo(event.test); + case "test_fn_start": + this.#handleTestStart(event.test); break; case "test_fn_success": this.#handleTestPass(event.test); @@ -108,9 +105,15 @@ const createJestEnvironment = (Base: T): T => case "test_fn_failure": this.#handleTestFail(event.test); break; + case "test_done": + this.#handleTestScopeStop(event.test); + break; case "test_skip": this.#handleTestSkip(event.test); break; + case "test_todo": + this.#handleTestTodo(event.test); + break; case "run_finish": this.#handleRunFinish(); break; @@ -181,7 +184,7 @@ const createJestEnvironment = (Base: T): T => this.runtime.stopFixture(fixtureUuid); } - #startTest(test: Circus.TestEntry) { + #handleTestStart(test: Circus.TestEntry) { const newTestSuitePath = getTestPath(test.parent); const newTestFullName = this.#getTestFullName(test); @@ -191,12 +194,12 @@ const createJestEnvironment = (Base: T): T => return; } - this.#startScope(); const testUuid = this.runtime.startTest( { name: test.name, fullName: newTestFullName, start: test.startedAt ?? undefined, + stage: Stage.RUNNING, labels: [ getLanguageLabel(), getFrameworkLabel("jest"), @@ -215,28 +218,34 @@ const createJestEnvironment = (Base: T): T => return testUuid; } - #stopTest(testUuid: string, duration: number) { - if (!testUuid) { - return; - } - - this.runtime.stopTest(testUuid, { duration }); - this.runtime.writeTest(testUuid); + #handleTestScopeStart() { + this.#startScope(); } - #handleTestStart(test: Circus.TestEntry) { - const testUuid = this.#startTest(test); + #handleTestScopeStop(test: Circus.TestEntry) { + const testUuid = this.runContext.executables.pop(); - if (!testUuid) { - return; + if (testUuid) { + const { details } = this.#statusAndDetails(test.errors); + let tr: TestResult | undefined; + this.runtime.updateTest(testUuid, (result) => { + tr = result; + }); + // hook failure, finish as skipped + if (tr?.status === undefined && tr?.stage === Stage.RUNNING) { + this.runtime.updateTest(testUuid, (result) => { + result.stage = Stage.FINISHED; + result.status = Status.SKIPPED; + result.statusDetails = { + ...result.statusDetails, + ...details, + }; + }); + } + + this.runtime.writeTest(testUuid); } - this.runtime.updateTest(testUuid, (result) => { - result.stage = Stage.RUNNING; - }); - } - - #handleTestDone() { this.#stopScope(); } @@ -247,47 +256,54 @@ const createJestEnvironment = (Base: T): T => } #stopScope() { - const scopeUuid = this.runContext.scopes.pop()!; + const scopeUuid = this.runContext.scopes.pop(); + if (!scopeUuid) { + return; + } this.runtime.writeScope(scopeUuid); } #handleTestPass(test: Circus.TestEntry) { - const testUuid = this.runContext.executables.pop(); + const testUuid = this.#currentExecutable(); if (!testUuid) { return; } + // @ts-ignore + const { suppressedErrors = [] } = this.global.expect.getState(); + const statusAndDetails = this.#statusAndDetails(suppressedErrors as Circus.TestError[]); this.runtime.updateTest(testUuid, (result) => { result.stage = Stage.FINISHED; - result.status = Status.PASSED; + result.status = statusAndDetails.status; + result.statusDetails = { + ...result.statusDetails, + ...statusAndDetails.details, + }; }); - this.#stopTest(testUuid, test.duration ?? 0); + + this.runtime.stopTest(testUuid, { duration: test.duration ?? 0 }); } #handleTestFail(test: Circus.TestEntry) { - const testUuid = this.runContext.executables.pop(); + const testUuid = this.#currentExecutable(); if (!testUuid) { return; } - // jest collects all errors, but we need to report the first one because it's a reason why the test has been failed - const [error] = test.errors; - const hasMultipleErrors = Array.isArray(error); - const firstError: Error = hasMultipleErrors ? error[0] : error; - const details = getMessageAndTraceFromError(firstError); - const status = getStatusFromError(firstError); + const { status, details } = this.#statusAndDetails(test.errors); this.runtime.updateTest(testUuid, (result) => { result.stage = Stage.FINISHED; result.status = status; result.statusDetails = { + ...result.statusDetails, ...details, }; }); - this.#stopTest(testUuid, test.duration ?? 0); + this.runtime.stopTest(testUuid, { duration: test.duration ?? 0 }); } #handleTestSkip(test: Circus.TestEntry) { @@ -297,37 +313,66 @@ const createJestEnvironment = (Base: T): T => return; } - const testUuid = this.runContext.executables.pop(); + // noinspection JSPotentiallyInvalidUsageOfThis + const testUuid = this.#handleTestStart(test); if (!testUuid) { return; } this.runtime.updateTest(testUuid, (result) => { - result.stage = Stage.PENDING; + result.stage = Stage.FINISHED; result.status = Status.SKIPPED; }); - this.#stopTest(testUuid, test.duration ?? 0); + // noinspection JSPotentiallyInvalidUsageOfThis + this.#handleTestScopeStop(test); } #handleTestTodo(test: Circus.TestEntry) { - const testUuid = this.runContext.executables.pop(); - + // noinspection JSPotentiallyInvalidUsageOfThis + const testUuid = this.#handleTestStart(test); if (!testUuid) { return; } this.runtime.updateTest(testUuid, (result) => { - result.stage = Stage.PENDING; + result.stage = Stage.FINISHED; result.status = Status.SKIPPED; + result.statusDetails = { + message: "TODO", + }; }); - this.#stopTest(testUuid, test.duration ?? 0); + // noinspection JSPotentiallyInvalidUsageOfThis + this.#handleTestScopeStop(test); } #handleRunFinish() { this.runtime.writeEnvironmentInfo(); this.runtime.writeCategoriesDefinitions(); } + + #currentExecutable() { + if (this.runContext.executables.length === 0) { + return undefined; + } + return this.runContext.executables[this.runContext.executables.length - 1]; + } + + #statusAndDetails(errors: Circus.TestError[]): { status: Status; details: Partial } { + if (errors.length === 0) { + return { + status: Status.PASSED, + details: {}, + }; + } + // jest collects all errors, but we need to report the first one because it's a reason why the test has been failed + const [error] = errors; + const hasMultipleErrors = Array.isArray(error); + const firstError: Error = hasMultipleErrors ? error[0] : error; + const details = getMessageAndTraceFromError(firstError); + const status = getStatusFromError(firstError); + return { status, details }; + } }; }; diff --git a/packages/allure-jest/test/spec/hooks.test.ts b/packages/allure-jest/test/spec/hooks.test.ts index 1988913d5..57159004b 100644 --- a/packages/allure-jest/test/spec/hooks.test.ts +++ b/packages/allure-jest/test/spec/hooks.test.ts @@ -176,18 +176,43 @@ it("should report beforeAll/afterAll for tests in sub-suites", async () => { ); }); -it("reports failed hooks", async () => { +it("should report failed beforeAll hooks", async () => { const { tests, groups } = await runJestInlineTest({ "sample.test.js": ` beforeAll(() => { throw new Error("foo"); }); - it("passed", () => {}); + it("test 1", () => {}); + it("test 2", () => {}); `, }); - expect(tests).toHaveLength(0); + expect(tests).toHaveLength(2); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "test 1", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + }), + + expect.objectContaining({ + name: "test 2", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + }), + ]), + ); + expect(groups).toHaveLength(1); expect(groups).toEqual( expect.arrayContaining([ @@ -204,6 +229,140 @@ it("reports failed hooks", async () => { }), ]), afters: [], + children: expect.arrayContaining(tests.map((t) => t.uuid)), + }), + ]), + ); +}); + +it("should report failed beforeEach hooks", async () => { + const { tests, groups } = await runJestInlineTest({ + "sample.test.js": ` + beforeEach(() => { + throw new Error("foo"); + }); + + it("sample test", () => {}); + `, + }); + + expect(tests).toHaveLength(1); + const [testResult] = tests; + expect(testResult).toEqual( + expect.objectContaining({ + name: "sample test", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + }), + ); + + expect(groups).toHaveLength(1); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "beforeEach", + befores: expect.arrayContaining([ + expect.objectContaining({ + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + name: "beforeEach", + }), + ]), + afters: [], + children: [testResult.uuid], + }), + ]), + ); +}); + +it("should report failed afterEach hooks", async () => { + const { tests, groups } = await runJestInlineTest({ + "sample.test.js": ` + afterEach(() => { + throw new Error("foo"); + }); + + it("sample test", () => {}); + `, + }); + + expect(tests).toHaveLength(1); + const [testResult] = tests; + expect(testResult).toEqual( + expect.objectContaining({ + name: "sample test", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + ); + + expect(groups).toHaveLength(1); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "afterEach", + afters: expect.arrayContaining([ + expect.objectContaining({ + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + name: "afterEach", + }), + ]), + befores: [], + children: [testResult.uuid], + }), + ]), + ); +}); + +it("should report failed afterAll hooks", async () => { + const { tests, groups } = await runJestInlineTest({ + "sample.test.js": ` + afterAll(() => { + throw new Error("foo"); + }); + + it("sample test", () => {}); + `, + }); + + expect(tests).toHaveLength(1); + const [testResult] = tests; + expect(testResult).toEqual( + expect.objectContaining({ + name: "sample test", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + ); + + expect(groups).toHaveLength(1); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "afterAll", + afters: expect.arrayContaining([ + expect.objectContaining({ + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "foo", + trace: expect.any(String), + }), + name: "afterAll", + }), + ]), + befores: [], + children: [testResult.uuid], }), ]), ); diff --git a/packages/allure-jest/test/spec/skipped.test.ts b/packages/allure-jest/test/spec/skipped.test.ts index 2283b9f03..ea4f9197a 100644 --- a/packages/allure-jest/test/spec/skipped.test.ts +++ b/packages/allure-jest/test/spec/skipped.test.ts @@ -10,8 +10,14 @@ it("skipped test", async () => { }); expect(tests).toHaveLength(1); - expect(tests[0].stage).toBe(Stage.PENDING); - expect(tests[0].status).toBe(Status.SKIPPED); + const [tr] = tests; + + expect(tr).toEqual( + expect.objectContaining({ + stage: Stage.FINISHED, + status: Status.SKIPPED, + }), + ); }); it("test inside skipped suite", async () => { @@ -24,6 +30,12 @@ it("test inside skipped suite", async () => { }); expect(tests).toHaveLength(1); - expect(tests[0].stage).toBe(Stage.PENDING); - expect(tests[0].status).toBe(Status.SKIPPED); + const [tr] = tests; + + expect(tr).toEqual( + expect.objectContaining({ + stage: Stage.FINISHED, + status: Status.SKIPPED, + }), + ); }); diff --git a/packages/allure-jest/test/spec/snapshot.test.ts b/packages/allure-jest/test/spec/snapshot.test.ts new file mode 100644 index 000000000..c37b22e13 --- /dev/null +++ b/packages/allure-jest/test/spec/snapshot.test.ts @@ -0,0 +1,38 @@ +import { expect, it } from "vitest"; +import { Status } from "allure-js-commons"; +import { runJestInlineTest } from "../utils.js"; + +it("should support snapshot testing", async () => { + const { tests } = await runJestInlineTest({ + "sample.spec.js": ` + it("test with snapshot", () => { + expect("some other data").toMatchSnapshot(); + }); + afterEach(() => { + }); + `, + "__snapshots__/sample.spec.js.snap": + "// Jest Snapshot v1, https://goo.gl/fbAQLP\n" + + "\n" + + // prettier-ignore + "exports[`test with snapshot 1`] = `\"some data\"`;\n", + }); + + expect(tests).toHaveLength(1); + const [testResult] = tests; + + expect(testResult).toEqual( + expect.objectContaining({ + name: "test with snapshot", + status: Status.FAILED, + statusDetails: expect.objectContaining({ + message: expect.stringContaining(`expect(received).toMatchSnapshot() + +Snapshot name: \`test with snapshot 1\` + +Snapshot: "some data" +Received: "some other data"`), + }), + }), + ); +}); diff --git a/packages/allure-jest/test/spec/todo.test.ts b/packages/allure-jest/test/spec/todo.test.ts index 1086a8182..9654a61e8 100644 --- a/packages/allure-jest/test/spec/todo.test.ts +++ b/packages/allure-jest/test/spec/todo.test.ts @@ -10,6 +10,15 @@ it("todo", async () => { }); expect(tests).toHaveLength(1); - expect(tests[0].stage).toBe(Stage.PENDING); - expect(tests[0].status).toBe(Status.SKIPPED); + const [tr] = tests; + + expect(tr).toEqual( + expect.objectContaining({ + stage: Stage.FINISHED, + status: Status.SKIPPED, + statusDetails: expect.objectContaining({ + message: "TODO", + }), + }), + ); });