feat: return typed Result from test runner and telemetry task actions

.changeset/swift-ladybugs-cover.md

@@ -0,0 +1,7 @@
+---
+"@nomicfoundation/hardhat-mocha": patch
+"@nomicfoundation/hardhat-node-test-runner": patch
+"hardhat": patch
+---
+
+Return typed `Result` from test runners and telemetry tasks ([#8015](https://github.com/NomicFoundation/hardhat/pull/8015)).

.peer-bumps.json

@@ -10,12 +10,12 @@
     {
       "package": "@nomicfoundation/hardhat-mocha",
       "peer": "hardhat",
-      "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks"
+      "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks, the Result type helpers from hardhat/utils/result, and the TestSummary type from hardhat/types/test"
     },
     {
       "package": "@nomicfoundation/hardhat-node-test-runner",
       "peer": "hardhat",
-      "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks"
+      "reason": "It depends on the new TestHooks#onTestRunStart, TestHooks#onTestWorkerDone, and TestHooks#onTestRunDone hooks, the Result type helpers from hardhat/utils/result, and the TestSummary type from hardhat/types/test"
     },
     {
       "package": "@nomicfoundation/hardhat-verify",

v-next/hardhat-ethers-chai-matchers/test/index.ts

@@ -22,6 +22,9 @@ describe("hardhat-ethers-chai-matchers plugin correctly initialized", () => {
       noCompile: true,
     });
 
-    assert.deepEqual(result, { failed: 0, passed: 1 });
+    assert.deepEqual(result, {
+      success: true,
+      value: { failed: 0, passed: 1, skipped: 0, todo: 0 },
+    });
   });
 });

v-next/hardhat-mocha/src/task-action.ts

@@ -1,5 +1,7 @@
 import type { HardhatConfig } from "hardhat/types/config";
 import type { NewTaskActionFunction } from "hardhat/types/tasks";
+import type { TestSummary } from "hardhat/types/test";
+import type { Result } from "hardhat/types/utils";
 import type { MochaOptions } from "mocha";
 
 import { resolve as pathResolve } from "node:path";
@@ -8,6 +10,7 @@ import { HardhatError } from "@nomicfoundation/hardhat-errors";
 import { setGlobalOptionsAsEnvVariables } from "@nomicfoundation/hardhat-utils/env";
 import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs";
 import debug from "debug";
+import { errorResult, successfulResult } from "hardhat/utils/result";
 
 import { createPerformanceTracker } from "./performance.js";
 
@@ -63,7 +66,7 @@ let testsAlreadyRun = false;
 const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
   { testFiles, bail, grep, noCompile },
   hre,
-) => {
+): Promise<Result<TestSummary, TestSummary>> => {
   // Set an environment variable that plugins can use to detect when a process is running tests
   process.env.HH_TEST = "true";
 
@@ -92,7 +95,12 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
   perf.endPhase("Get test files");
 
   if (files.length === 0) {
-    return;
+    return successfulResult({
+      failed: 0,
+      passed: 0,
+      skipped: 0,
+      todo: 0,
+    });
   }
 
   const unhandledRejectionHookPath = "./unhandled-rejection-mocha-hook.js";
@@ -203,18 +211,21 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
 
   perf.endPhase("Reporting");
 
-  if (testFailures > 0) {
-    process.exitCode = 1;
-  }
-
   console.log();
 
   perf.end();
 
   perf.logInto(performanceLog);
   perf.clear();
 
-  return { failed: testFailures, passed: total - testFailures };
+  const summary = {
+    failed: testFailures,
+    passed: total - testFailures,
+    skipped: 0,
+    todo: 0,
+  };
+
+  return testFailures > 0 ? errorResult(summary) : successfulResult(summary);
 };
 
 export default testWithHardhat;

v-next/hardhat-mocha/test/env.ts

@@ -13,7 +13,6 @@ describe("Hardhat Mocha env variables", () => {
     );
     const hre = await createHardhatRuntimeEnvironment(hardhatConfig.default);
 
-    const exitCode = process.exitCode;
     const nodeEnv = process.env.NODE_ENV;
     const hhTest = process.env.HH_TEST;
     try {
@@ -24,7 +23,6 @@ describe("Hardhat Mocha env variables", () => {
     } finally {
       process.env.HH_TEST = hhTest;
       process.env.NODE_ENV = nodeEnv;
-      process.exitCode = exitCode;
     }
   });
 });

v-next/hardhat-mocha/test/index.ts

@@ -22,7 +22,10 @@ describe("Hardhat Mocha plugin", () => {
 
       const result = await hre.tasks.getTask(["test", "mocha"]).run({});
 
-      assert.deepEqual(result, { failed: 0, passed: 2 });
+      assert.deepEqual(result, {
+        success: true,
+        value: { failed: 0, passed: 2, skipped: 0, todo: 0 },
+      });
     });
   });
 

v-next/hardhat-node-test-runner/src/task-action.ts

@@ -1,6 +1,7 @@
 import type { HardhatConfig } from "hardhat/types/config";
 import type { NewTaskActionFunction } from "hardhat/types/tasks";
-import type { LastParameter } from "hardhat/types/utils";
+import type { TestSummary } from "hardhat/types/test";
+import type { LastParameter, Result } from "hardhat/types/utils";
 
 import { pipeline } from "node:stream/promises";
 import { run } from "node:test";
@@ -10,6 +11,7 @@ import { hardhatTestReporter } from "@nomicfoundation/hardhat-node-test-reporter
 import { setGlobalOptionsAsEnvVariables } from "@nomicfoundation/hardhat-utils/env";
 import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs";
 import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream";
+import { errorResult, successfulResult } from "hardhat/utils/result";
 
 interface TestActionArguments {
   testFiles: string[];
@@ -56,7 +58,7 @@ async function getTestFiles(
 const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
   { testFiles, only, grep, noCompile, testSummaryIndex },
   hre,
-) => {
+): Promise<Result<TestSummary, TestSummary>> => {
   // Set an environment variable that plugins can use to detect when a process is running tests
   process.env.HH_TEST = "true";
 
@@ -76,7 +78,12 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
   const files = await getTestFiles(testFiles, hre.config);
 
   if (files.length === 0) {
-    return 0;
+    return successfulResult({
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      todo: 0,
+    });
   }
 
   const imports = [];
@@ -195,13 +202,11 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
     async () => {},
   );
 
-  if (testResults.failed > 0) {
-    process.exitCode = 1;
-  }
-
   console.log();
 
-  return testResults;
+  return testResults.failed > 0
+    ? errorResult(testResults)
+    : successfulResult(testResults);
 };
 
 export default testWithHardhat;

v-next/hardhat-node-test-runner/test/index.ts

@@ -13,7 +13,6 @@ describe("Hardhat Node plugin", () => {
     ).default;
     const hre = await createHardhatRuntimeEnvironment(baseHhConfig);
 
-    const exitCode = process.exitCode;
     const nodeEnv = process.env.NODE_ENV;
     const hhTest = process.env.HH_TEST;
     try {
@@ -24,7 +23,6 @@ describe("Hardhat Node plugin", () => {
     } finally {
       process.env.HH_TEST = hhTest;
       process.env.NODE_ENV = nodeEnv;
-      process.exitCode = exitCode;
     }
   });
 
@@ -34,7 +32,6 @@ describe("Hardhat Node plugin", () => {
     ).default;
     const hre = await createHardhatRuntimeEnvironment(baseHhConfig);
 
-    const exitCode = process.exitCode;
     const nodeEnv = process.env.NODE_ENV;
     const hhTest = process.env.HH_TEST;
     try {
@@ -45,7 +42,6 @@ describe("Hardhat Node plugin", () => {
     } finally {
       process.env.HH_TEST = hhTest;
       process.env.NODE_ENV = nodeEnv;
-      process.exitCode = exitCode;
     }
   });
 });

v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts

@@ -1,6 +1,8 @@
 import type { RunOptions } from "./runner.js";
 import type { TestEvent } from "./types.js";
 import type { NewTaskActionFunction } from "../../../types/tasks.js";
+import type { TestSummary } from "../../../types/test.js";
+import type { Result } from "../../../types/utils.js";
 import type {
   Artifact as EdrArtifact,
   BuildInfoAndOutput,
@@ -19,6 +21,7 @@ import { resolveFromRoot } from "@nomicfoundation/hardhat-utils/path";
 import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream";
 
 import { getFullyQualifiedName } from "../../../utils/contract-names.js";
+import { errorResult, successfulResult } from "../../../utils/result.js";
 import { HardhatRuntimeEnvironmentImplementation } from "../../core/hre.js";
 import { isSupportedChainType } from "../../edr/chain-type.js";
 import { ArtifactManagerImplementation } from "../artifacts/artifact-manager.js";
@@ -46,7 +49,7 @@ interface TestActionArguments {
 const runSolidityTests: NewTaskActionFunction<TestActionArguments> = async (
   { testFiles, chainType, grep, noCompile, verbosity, testSummaryIndex },
   hre,
-) => {
+): Promise<Result<TestSummary, TestSummary>> => {
   assertHardhatInvariant(
     hre instanceof HardhatRuntimeEnvironmentImplementation,
     "Expected HRE to be an instance of HardhatRuntimeEnvironmentImplementation",
@@ -279,19 +282,13 @@ const runSolidityTests: NewTaskActionFunction<TestActionArguments> = async (
     async () => {},
   );
 
-  if (includesFailures || includesErrors) {
-    process.exitCode = 1;
-  }
-
   console.log();
 
-  return {
-    failed,
-    passed,
-    skipped,
-    todo: 0,
-    failureOutput,
-  };
+  const summary = { failed, passed, skipped, todo: 0, failureOutput };
+
+  return includesFailures || includesErrors
+    ? errorResult(summary)
+    : successfulResult(summary);
 };
 
 export default runSolidityTests;

v-next/hardhat/src/internal/builtin-plugins/telemetry/task-action.ts

@@ -1,5 +1,6 @@
 import type { NewTaskActionFunction } from "../../../types/tasks.js";
 
+import { errorResult, successfulResult } from "../../../utils/result.js";
 import {
   isTelemetryAllowed,
   setTelemetryEnabled,
@@ -15,8 +16,7 @@ const configureTelemetry: NewTaskActionFunction<
 > = async ({ enable, disable }) => {
   if (enable && disable) {
     console.error("Cannot enable and disable telemetry at the same time");
-    process.exitCode = 1;
-    return;
+    return errorResult();
   }
 
   if (enable) {
@@ -40,6 +40,8 @@ const configureTelemetry: NewTaskActionFunction<
       "Telemetry is disabled, to enable it run `npx hardhat telemetry --enable`",
     );
   }
+
+  return successfulResult();
 };
 
 export default configureTelemetry;