diff --git a/.changeset/swift-ladybugs-cover.md b/.changeset/swift-ladybugs-cover.md new file mode 100644 index 00000000000..7fb4e06195a --- /dev/null +++ b/.changeset/swift-ladybugs-cover.md @@ -0,0 +1,7 @@ +--- +"@nomicfoundation/hardhat-mocha": patch +"@nomicfoundation/hardhat-node-test-runner": patch +"hardhat": patch +--- + +Return typed `Result` from test runners and telemetry tasks ([#8015](https://github.com/NomicFoundation/hardhat/pull/8015)). diff --git a/.peer-bumps.json b/.peer-bumps.json index 837ee1e19e9..ed7c23c2aaf 100644 --- a/.peer-bumps.json +++ b/.peer-bumps.json @@ -10,12 +10,12 @@ { "package": "@nomicfoundation/hardhat-mocha", "peer": "hardhat", - "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks" + "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks, the Result type helpers from hardhat/utils/result, and the TestSummary type from hardhat/types/test" }, { "package": "@nomicfoundation/hardhat-node-test-runner", "peer": "hardhat", - "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks" + "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks, the Result type helpers from hardhat/utils/result, and the TestSummary type from hardhat/types/test" }, { "package": "@nomicfoundation/hardhat-verify", diff --git a/v-next/hardhat-ethers-chai-matchers/test/index.ts b/v-next/hardhat-ethers-chai-matchers/test/index.ts index c00c0702b11..cef315a1aec 100644 --- a/v-next/hardhat-ethers-chai-matchers/test/index.ts +++ b/v-next/hardhat-ethers-chai-matchers/test/index.ts @@ -22,6 +22,9 @@ describe("hardhat-ethers-chai-matchers plugin correctly initialized", () => { noCompile: true, }); - assert.deepEqual(result, { failed: 0, passed: 1 }); + assert.deepEqual(result, { + success: true, + value: { failed: 0, passed: 1, skipped: 0, todo: 0 }, + }); }); }); diff --git a/v-next/hardhat-mocha/src/task-action.ts b/v-next/hardhat-mocha/src/task-action.ts index fd405a42d56..94b671e6244 100644 --- a/v-next/hardhat-mocha/src/task-action.ts +++ b/v-next/hardhat-mocha/src/task-action.ts @@ -1,5 +1,7 @@ import type { HardhatConfig } from "hardhat/types/config"; import type { NewTaskActionFunction } from "hardhat/types/tasks"; +import type { TestSummary } from "hardhat/types/test"; +import type { Result } from "hardhat/types/utils"; import type { MochaOptions } from "mocha"; import { resolve as pathResolve } from "node:path"; @@ -8,6 +10,7 @@ import { HardhatError } from "@nomicfoundation/hardhat-errors"; import { setGlobalOptionsAsEnvVariables } from "@nomicfoundation/hardhat-utils/env"; import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs"; import debug from "debug"; +import { errorResult, successfulResult } from "hardhat/utils/result"; import { createPerformanceTracker } from "./performance.js"; @@ -63,7 +66,7 @@ let testsAlreadyRun = false; const testWithHardhat: NewTaskActionFunction = async ( { testFiles, bail, grep, noCompile }, hre, -) => { +): Promise> => { // Set an environment variable that plugins can use to detect when a process is running tests process.env.HH_TEST = "true"; @@ -92,7 +95,12 @@ const testWithHardhat: NewTaskActionFunction = async ( perf.endPhase("Get test files"); if (files.length === 0) { - return; + return successfulResult({ + failed: 0, + passed: 0, + skipped: 0, + todo: 0, + }); } const unhandledRejectionHookPath = "./unhandled-rejection-mocha-hook.js"; @@ -203,10 +211,6 @@ const testWithHardhat: NewTaskActionFunction = async ( perf.endPhase("Reporting"); - if (testFailures > 0) { - process.exitCode = 1; - } - console.log(); perf.end(); @@ -214,7 +218,14 @@ const testWithHardhat: NewTaskActionFunction = async ( perf.logInto(performanceLog); perf.clear(); - return { failed: testFailures, passed: total - testFailures }; + const summary = { + failed: testFailures, + passed: total - testFailures, + skipped: 0, + todo: 0, + }; + + return testFailures > 0 ? errorResult(summary) : successfulResult(summary); }; export default testWithHardhat; diff --git a/v-next/hardhat-mocha/test/env.ts b/v-next/hardhat-mocha/test/env.ts index 62d7989a136..af2a3ef15fe 100644 --- a/v-next/hardhat-mocha/test/env.ts +++ b/v-next/hardhat-mocha/test/env.ts @@ -13,7 +13,6 @@ describe("Hardhat Mocha env variables", () => { ); const hre = await createHardhatRuntimeEnvironment(hardhatConfig.default); - const exitCode = process.exitCode; const nodeEnv = process.env.NODE_ENV; const hhTest = process.env.HH_TEST; try { @@ -24,7 +23,6 @@ describe("Hardhat Mocha env variables", () => { } finally { process.env.HH_TEST = hhTest; process.env.NODE_ENV = nodeEnv; - process.exitCode = exitCode; } }); }); diff --git a/v-next/hardhat-mocha/test/index.ts b/v-next/hardhat-mocha/test/index.ts index 71c9b040f2a..3ef391fd59f 100644 --- a/v-next/hardhat-mocha/test/index.ts +++ b/v-next/hardhat-mocha/test/index.ts @@ -22,7 +22,10 @@ describe("Hardhat Mocha plugin", () => { const result = await hre.tasks.getTask(["test", "mocha"]).run({}); - assert.deepEqual(result, { failed: 0, passed: 2 }); + assert.deepEqual(result, { + success: true, + value: { failed: 0, passed: 2, skipped: 0, todo: 0 }, + }); }); }); diff --git a/v-next/hardhat-node-test-runner/src/task-action.ts b/v-next/hardhat-node-test-runner/src/task-action.ts index c1a22b76107..aafc8ac1c82 100644 --- a/v-next/hardhat-node-test-runner/src/task-action.ts +++ b/v-next/hardhat-node-test-runner/src/task-action.ts @@ -1,6 +1,7 @@ import type { HardhatConfig } from "hardhat/types/config"; import type { NewTaskActionFunction } from "hardhat/types/tasks"; -import type { LastParameter } from "hardhat/types/utils"; +import type { TestSummary } from "hardhat/types/test"; +import type { LastParameter, Result } from "hardhat/types/utils"; import { pipeline } from "node:stream/promises"; import { run } from "node:test"; @@ -10,6 +11,7 @@ import { hardhatTestReporter } from "@nomicfoundation/hardhat-node-test-reporter import { setGlobalOptionsAsEnvVariables } from "@nomicfoundation/hardhat-utils/env"; import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs"; import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream"; +import { errorResult, successfulResult } from "hardhat/utils/result"; interface TestActionArguments { testFiles: string[]; @@ -56,7 +58,7 @@ async function getTestFiles( const testWithHardhat: NewTaskActionFunction = async ( { testFiles, only, grep, noCompile, testSummaryIndex }, hre, -) => { +): Promise> => { // Set an environment variable that plugins can use to detect when a process is running tests process.env.HH_TEST = "true"; @@ -76,7 +78,12 @@ const testWithHardhat: NewTaskActionFunction = async ( const files = await getTestFiles(testFiles, hre.config); if (files.length === 0) { - return 0; + return successfulResult({ + passed: 0, + failed: 0, + skipped: 0, + todo: 0, + }); } const imports = []; @@ -195,13 +202,11 @@ const testWithHardhat: NewTaskActionFunction = async ( async () => {}, ); - if (testResults.failed > 0) { - process.exitCode = 1; - } - console.log(); - return testResults; + return testResults.failed > 0 + ? errorResult(testResults) + : successfulResult(testResults); }; export default testWithHardhat; diff --git a/v-next/hardhat-node-test-runner/test/index.ts b/v-next/hardhat-node-test-runner/test/index.ts index 2c74e269823..e9c0ad0cbfc 100644 --- a/v-next/hardhat-node-test-runner/test/index.ts +++ b/v-next/hardhat-node-test-runner/test/index.ts @@ -13,7 +13,6 @@ describe("Hardhat Node plugin", () => { ).default; const hre = await createHardhatRuntimeEnvironment(baseHhConfig); - const exitCode = process.exitCode; const nodeEnv = process.env.NODE_ENV; const hhTest = process.env.HH_TEST; try { @@ -24,7 +23,6 @@ describe("Hardhat Node plugin", () => { } finally { process.env.HH_TEST = hhTest; process.env.NODE_ENV = nodeEnv; - process.exitCode = exitCode; } }); @@ -34,7 +32,6 @@ describe("Hardhat Node plugin", () => { ).default; const hre = await createHardhatRuntimeEnvironment(baseHhConfig); - const exitCode = process.exitCode; const nodeEnv = process.env.NODE_ENV; const hhTest = process.env.HH_TEST; try { @@ -45,7 +42,6 @@ describe("Hardhat Node plugin", () => { } finally { process.env.HH_TEST = hhTest; process.env.NODE_ENV = nodeEnv; - process.exitCode = exitCode; } }); }); diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts index 0e0624adf6d..a40ac4a2e1e 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts @@ -1,6 +1,8 @@ import type { RunOptions } from "./runner.js"; import type { TestEvent } from "./types.js"; import type { NewTaskActionFunction } from "../../../types/tasks.js"; +import type { TestSummary } from "../../../types/test.js"; +import type { Result } from "../../../types/utils.js"; import type { Artifact as EdrArtifact, BuildInfoAndOutput, @@ -19,6 +21,7 @@ import { resolveFromRoot } from "@nomicfoundation/hardhat-utils/path"; import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream"; import { getFullyQualifiedName } from "../../../utils/contract-names.js"; +import { errorResult, successfulResult } from "../../../utils/result.js"; import { HardhatRuntimeEnvironmentImplementation } from "../../core/hre.js"; import { isSupportedChainType } from "../../edr/chain-type.js"; import { ArtifactManagerImplementation } from "../artifacts/artifact-manager.js"; @@ -46,7 +49,7 @@ interface TestActionArguments { const runSolidityTests: NewTaskActionFunction = async ( { testFiles, chainType, grep, noCompile, verbosity, testSummaryIndex }, hre, -) => { +): Promise> => { assertHardhatInvariant( hre instanceof HardhatRuntimeEnvironmentImplementation, "Expected HRE to be an instance of HardhatRuntimeEnvironmentImplementation", @@ -279,19 +282,13 @@ const runSolidityTests: NewTaskActionFunction = async ( async () => {}, ); - if (includesFailures || includesErrors) { - process.exitCode = 1; - } - console.log(); - return { - failed, - passed, - skipped, - todo: 0, - failureOutput, - }; + const summary = { failed, passed, skipped, todo: 0, failureOutput }; + + return includesFailures || includesErrors + ? errorResult(summary) + : successfulResult(summary); }; export default runSolidityTests; diff --git a/v-next/hardhat/src/internal/builtin-plugins/telemetry/task-action.ts b/v-next/hardhat/src/internal/builtin-plugins/telemetry/task-action.ts index 8368136e9bb..ac75201d83f 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/telemetry/task-action.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/telemetry/task-action.ts @@ -1,5 +1,6 @@ import type { NewTaskActionFunction } from "../../../types/tasks.js"; +import { errorResult, successfulResult } from "../../../utils/result.js"; import { isTelemetryAllowed, setTelemetryEnabled, @@ -15,8 +16,7 @@ const configureTelemetry: NewTaskActionFunction< > = async ({ enable, disable }) => { if (enable && disable) { console.error("Cannot enable and disable telemetry at the same time"); - process.exitCode = 1; - return; + return errorResult(); } if (enable) { @@ -40,6 +40,8 @@ const configureTelemetry: NewTaskActionFunction< "Telemetry is disabled, to enable it run `npx hardhat telemetry --enable`", ); } + + return successfulResult(); }; export default configureTelemetry; diff --git a/v-next/hardhat/src/internal/builtin-plugins/test/task-action.ts b/v-next/hardhat/src/internal/builtin-plugins/test/task-action.ts index f0f390a6ce4..2ffbacb4d62 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/test/task-action.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/test/task-action.ts @@ -4,13 +4,21 @@ import type { Task, TaskArguments, } from "../../../types/tasks.js"; +import type { TestSummary } from "../../../types/test.js"; +import type { Result } from "../../../types/utils.js"; import { assertHardhatInvariant, HardhatError, } from "@nomicfoundation/hardhat-errors"; +import { isObject } from "@nomicfoundation/hardhat-utils/lang"; import chalk, { type ChalkInstance } from "chalk"; +import { + errorResult, + isResult, + successfulResult, +} from "../../../utils/result.js"; import { HardhatRuntimeEnvironmentImplementation } from "../../core/hre.js"; interface TestActionArguments { @@ -21,10 +29,27 @@ interface TestActionArguments { verbosity: number; } +// Old plugins may only return { failed, passed } without skipped/todo, +// so we accept a partial shape and fill defaults in the coordinator. +interface PartialTestSummary extends Omit { + skipped?: number; + todo?: number; +} + +function isTestSummary(value: unknown): value is PartialTestSummary { + return ( + isObject(value) && + typeof value.failed === "number" && + typeof value.passed === "number" && + (value.skipped === undefined || typeof value.skipped === "number") && + (value.todo === undefined || typeof value.todo === "number") + ); +} + const runAllTests: NewTaskActionFunction = async ( { testFiles, chainType, grep, noCompile, verbosity }, hre, -) => { +): Promise> => { // If this code is executed, it means the user has not specified a test runner. // If file paths are specified, we need to determine which test runner applies to each test file. // If no file paths are specified, each test runner will execute all tests located under its configured path in the Hardhat configuration. @@ -49,18 +74,10 @@ const runAllTests: NewTaskActionFunction = async ( hre._coverage.disableReport(); } - const testSummaries: Record< - string, - { - failed?: number; - passed?: number; - skipped?: number; - todo?: number; - failureOutput?: string; - } - > = {}; + const testSummaries: Record = {}; let failureIndex = 1; + let hasFailures = false; for (const subtask of thisTask.subtasks.values()) { const files = getTestFilesForSubtask(subtask, testFiles, subtasksToFiles); @@ -84,18 +101,45 @@ const runAllTests: NewTaskActionFunction = async ( args.verbosity = verbosity; } - const summaryId = subtask.id[subtask.id.length - 1]; - if (subtask.options.has("testSummaryIndex")) { args.testSummaryIndex = failureIndex; + } + + const subtaskResult = await subtask.run(args); - testSummaries[summaryId] = await subtask.run(args); - failureIndex += testSummaries[summaryId].failed ?? 0; - } else if (summaryId === "mocha") { - // mocha doesn't use the testSummaryIndex, but it does return failure & success counts - testSummaries[summaryId] = await subtask.run(args); - } else { - await subtask.run(args); + const isSubtaskResult = isResult( + subtaskResult, + isTestSummary, + isTestSummary, + ); + const summary = isSubtaskResult + ? subtaskResult.success + ? subtaskResult.value + : subtaskResult.error + : isTestSummary(subtaskResult) + ? subtaskResult + : undefined; + + if (summary !== undefined) { + const summaryId = subtask.id[subtask.id.length - 1]; + testSummaries[summaryId] = { + skipped: 0, + todo: 0, + ...summary, + }; + + if (subtask.options.has("testSummaryIndex")) { + failureIndex += summary.failed; + } + } + + if ( + (isSubtaskResult && !subtaskResult.success) || + // Backwards compatibility: old plugins may not return a Result, so fall + // back to the process exit code to detect failures + (process.exitCode !== undefined && process.exitCode !== 0) + ) { + hasFailures = true; } } @@ -106,19 +150,19 @@ const runAllTests: NewTaskActionFunction = async ( const outputLines: string[] = []; for (const [subtaskName, results] of Object.entries(testSummaries)) { - if (results.passed !== undefined && results.passed > 0) { + if (results.passed > 0) { passed.push([subtaskName, results.passed]); } - if (results.failed !== undefined && results.failed > 0) { + if (results.failed > 0) { failed.push([subtaskName, results.failed]); } - if (results.skipped !== undefined && results.skipped > 0) { + if (results.skipped > 0) { skipped.push([subtaskName, results.skipped]); } - if (results.todo !== undefined && results.todo > 0) { + if (results.todo > 0) { todo.push([subtaskName, results.todo]); } @@ -176,9 +220,11 @@ const runAllTests: NewTaskActionFunction = async ( console.log(); } - if (process.exitCode !== undefined && process.exitCode !== 0) { + if (hasFailures) { console.error("Test run failed"); } + + return hasFailures ? errorResult() : successfulResult(); }; function logSummaryLine( diff --git a/v-next/hardhat/src/types/test.ts b/v-next/hardhat/src/types/test.ts index 442207f7a35..21aedbe7985 100644 --- a/v-next/hardhat/src/types/test.ts +++ b/v-next/hardhat/src/types/test.ts @@ -5,3 +5,15 @@ export interface HardhatTestUserConfig {} /* eslint-disable-next-line @typescript-eslint/no-empty-interface -- Empty interface allow plugins to extend the Test configuration for Hardhat. */ export interface HardhatTestConfig {} + +/** + * Summary of a test run, containing counts of test outcomes and optional + * failure output. + */ +export interface TestSummary { + failed: number; + passed: number; + skipped: number; + todo: number; + failureOutput?: string; +} diff --git a/v-next/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts b/v-next/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts index 4c5f49107d4..491aa7f41c9 100644 --- a/v-next/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts +++ b/v-next/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts @@ -150,7 +150,6 @@ describe("solidity-test/task-action", function () { it("should set the NODE_ENV variable if undefined and HH_TEST always", async () => { hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); - const exitCode = process.exitCode; const nodeEnv = process.env.NODE_ENV; const hhTest = process.env.HH_TEST; try { @@ -161,14 +160,12 @@ describe("solidity-test/task-action", function () { } finally { process.env.HH_TEST = hhTest; process.env.NODE_ENV = nodeEnv; - process.exitCode = exitCode; } }); it("should not set the NODE_ENV variable if defined before", async () => { hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); - const exitCode = process.exitCode; const nodeEnv = process.env.NODE_ENV; const hhTest = process.env.HH_TEST; try { @@ -179,20 +176,31 @@ describe("solidity-test/task-action", function () { } finally { process.env.HH_TEST = hhTest; process.env.NODE_ENV = nodeEnv; - process.exitCode = exitCode; } }); - it("should run all the tests and throw if any of them fail", async () => { + it("should return an error result when any test fails", async () => { hre = await createHardhatRuntimeEnvironment(hardhatConfigFailingTests); - const exitCode = process.exitCode; - try { - await hre.tasks.getTask(["test", "solidity"]).run({ noCompile: true }); - assert.equal(process.exitCode, 1); - } finally { - process.exitCode = exitCode; - } + const result = await hre.tasks + .getTask(["test", "solidity"]) + .run({ noCompile: true }); + assert.deepEqual(result, { + success: false, + error: { failed: 0, passed: 0, skipped: 0, todo: 0, failureOutput: "" }, + }); + }); + + it("should return a success result when all tests pass", async () => { + hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); + + const result = await hre.tasks + .getTask(["test", "solidity"]) + .run({ noCompile: true }); + assert.deepEqual(result, { + success: true, + value: { failed: 0, passed: 0, skipped: 0, todo: 0, failureOutput: "" }, + }); }); describe("when the contracts are in the optimism chain type", () => { @@ -205,19 +213,14 @@ describe("solidity-test/task-action", function () { }); }); - it("should throw because the test is not compatible with the l1 chain type", async () => { + it("should return an error result because the test is not compatible with the l1 chain type", async () => { hre = await createHardhatRuntimeEnvironment(hardhatConfigOpTests); - const exitCode = process.exitCode; - try { - // default chain type is l1 - await hre.tasks.getTask(["test", "solidity"]).run({ - noCompile: true, - }); - assert.equal(process.exitCode, 1); - } finally { - process.exitCode = exitCode; - } + // default chain type is l1 + const result = await hre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + }); + assert.equal(result.success, false); }); }); @@ -254,45 +257,35 @@ describe("solidity-test/task-action", function () { it("Should compile the test files, but not the contracts", async () => { const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); - const exitCode = process.exitCode; - try { - await overriddenHre.tasks.getTask(["test", "solidity"]).run({ - noCompile: true, - }); - - // We only call build once - assert.equal(buildArgs.length, 1); - - const lastArgs = buildArgs[0]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, []); - } finally { - process.exitCode = exitCode; - } + await overriddenHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + }); + + // We only call build once + assert.equal(buildArgs.length, 1); + + const lastArgs = buildArgs[0]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, []); }); it("Should compile only the provided test files, and not the contracts", async () => { const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); - const exitCode = process.exitCode; const testFiles = ["test/contracts/all/Counter-1.t.sol"]; - try { - await overriddenHre.tasks.getTask(["test", "solidity"]).run({ - noCompile: true, - testFiles, - }); - - // We only call build once - assert.equal(buildArgs.length, 1); - - const lastArgs = buildArgs[0]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, testFiles); - } finally { - process.exitCode = exitCode; - } + await overriddenHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + testFiles, + }); + + // We only call build once + assert.equal(buildArgs.length, 1); + + const lastArgs = buildArgs[0]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, testFiles); }); }); @@ -300,50 +293,40 @@ describe("solidity-test/task-action", function () { it("Should compile the contracts and then the test files", async () => { const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); - const exitCode = process.exitCode; - try { - await overriddenHre.tasks.getTask(["test", "solidity"]).run({}); + await overriddenHre.tasks.getTask(["test", "solidity"]).run({}); - assert.equal(buildArgs.length, 2); + assert.equal(buildArgs.length, 2); - const firstArgs = buildArgs[0]; - assert.equal(firstArgs.noContracts, false); - assert.equal(firstArgs.noTests, true); - assert.deepEqual(firstArgs.files, []); + const firstArgs = buildArgs[0]; + assert.equal(firstArgs.noContracts, false); + assert.equal(firstArgs.noTests, true); + assert.deepEqual(firstArgs.files, []); - const lastArgs = buildArgs[1]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, []); - } finally { - process.exitCode = exitCode; - } + const lastArgs = buildArgs[1]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, []); }); it("Should compile the contracts and then the provided test files", async () => { const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); - const exitCode = process.exitCode; const testFiles = ["test/contracts/all/Counter-1.t.sol"]; - try { - await overriddenHre.tasks - .getTask(["test", "solidity"]) - .run({ testFiles }); - - assert.equal(buildArgs.length, 2); - - const firstArgs = buildArgs[0]; - assert.equal(firstArgs.noContracts, false); - assert.equal(firstArgs.noTests, true); - assert.deepEqual(firstArgs.files, []); - - const lastArgs = buildArgs[1]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, testFiles); - } finally { - process.exitCode = exitCode; - } + await overriddenHre.tasks + .getTask(["test", "solidity"]) + .run({ testFiles }); + + assert.equal(buildArgs.length, 2); + + const firstArgs = buildArgs[0]; + assert.equal(firstArgs.noContracts, false); + assert.equal(firstArgs.noTests, true); + assert.deepEqual(firstArgs.files, []); + + const lastArgs = buildArgs[1]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, testFiles); }); }); }); diff --git a/v-next/hardhat/test/internal/builtin-plugins/test/task-action.ts b/v-next/hardhat/test/internal/builtin-plugins/test/task-action.ts new file mode 100644 index 00000000000..f0b8d77fffc --- /dev/null +++ b/v-next/hardhat/test/internal/builtin-plugins/test/task-action.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict"; +import { afterEach, describe, it } from "node:test"; + +import { overrideTask, task } from "../../../../src/config.js"; +import { createHardhatRuntimeEnvironment } from "../../../../src/hre.js"; +import { ArgumentType } from "../../../../src/types/arguments.js"; +import { successfulResult, errorResult } from "../../../../src/utils/result.js"; + +// Override the builtin solidity subtask to be a no-op +const solidityNoOp = overrideTask(["test", "solidity"]) + .setInlineAction(async () => undefined) + .build(); + +function mockRunner(name: string, action: (...args: any[]) => unknown) { + return task(["test", name]) + .addVariadicArgument({ + name: "testFiles", + description: "Test files", + defaultValue: [], + }) + .addOption({ + name: "grep", + description: "Only run tests matching the given string or regexp", + type: ArgumentType.STRING_WITHOUT_DEFAULT, + defaultValue: undefined, + }) + .addFlag({ name: "noCompile" }) + .setInlineAction(action) + .build(); +} + +describe("test/task-action", function () { + afterEach(function () { + process.exitCode = undefined; + }); + + describe("subtask returning Result", function () { + it("should return a successful result when the subtask returns a successful Result", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => + successfulResult({ passed: 3, failed: 0, skipped: 0, todo: 0 }), + ), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: true, value: undefined }); + }); + + it("should return an error result when the subtask returns a failed Result", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => + errorResult({ passed: 1, failed: 2, skipped: 0, todo: 0 }), + ), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + }); + + describe("subtask returning a plain TestSummary (backwards compat)", function () { + it("should return a successful result when the subtask returns a plain TestSummary and exitCode is not set", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => ({ + passed: 5, + failed: 0, + skipped: 0, + todo: 0, + })), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: true, value: undefined }); + }); + + it("should return an error result when the subtask returns a plain TestSummary and exitCode is 1", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => { + process.exitCode = 1; + return { passed: 1, failed: 3, skipped: 0, todo: 0 }; + }), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + }); + + describe("subtask returning undefined with process.exitCode (backwards compat)", function () { + it("should return an error result when the subtask returns undefined and exitCode is 1", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => { + process.exitCode = 1; + }), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + }); + + describe("subtask returning a partial TestSummary (backwards compat)", function () { + it("should return a successful result when the subtask returns only failed and passed", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => ({ + passed: 3, + failed: 0, + })), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: true, value: undefined }); + }); + + it("should return an error result when the subtask returns a partial TestSummary and exitCode is 1", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => { + process.exitCode = 1; + return { passed: 1, failed: 2 }; + }), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + }); + + describe("mixed subtask return types", function () { + it("should return an error result when a Result subtask succeeds but a plain summary subtask fails", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => + successfulResult({ passed: 3, failed: 0, skipped: 0, todo: 0 }), + ), + mockRunner("runner-b", () => { + process.exitCode = 1; + return { passed: 2, failed: 1, skipped: 0, todo: 0 }; + }), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + + it("should return a successful result when all subtasks succeed with different return styles", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => + successfulResult({ passed: 3, failed: 0, skipped: 0, todo: 0 }), + ), + mockRunner("runner-b", () => ({ + passed: 2, + failed: 0, + skipped: 0, + todo: 0, + })), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: true, value: undefined }); + }); + + it("should return an error result when a Result subtask succeeds but another subtask only sets exitCode", async () => { + const hre = await createHardhatRuntimeEnvironment({ + tasks: [ + solidityNoOp, + mockRunner("runner-a", () => + successfulResult({ passed: 3, failed: 0, skipped: 0, todo: 0 }), + ), + mockRunner("runner-b", () => { + process.exitCode = 1; + }), + ], + }); + + const result = await hre.tasks.getTask("test").run({ noCompile: true }); + + assert.deepEqual(result, { success: false, error: undefined }); + }); + }); +}); diff --git a/v-next/hardhat/test/utils/result.ts b/v-next/hardhat/test/utils/result.ts index 9e1a0670c2f..7a160a3d25e 100644 --- a/v-next/hardhat/test/utils/result.ts +++ b/v-next/hardhat/test/utils/result.ts @@ -8,7 +8,7 @@ import { } from "../../src/utils/result.js"; describe("result", function () { - describe("successResult", function () { + describe("successfulResult", function () { it("should create a successful Result with the given value", function () { const result = successfulResult(42); assert.deepEqual(result, { success: true, value: 42 });