diff --git a/SPLIT_TESTS_COMPILATION_SPEC.md b/SPLIT_TESTS_COMPILATION_SPEC.md index 63f696a2dd5..a7ad3daa40b 100644 --- a/SPLIT_TESTS_COMPILATION_SPEC.md +++ b/SPLIT_TESTS_COMPILATION_SPEC.md @@ -1,3 +1,5 @@ + + # Spec: `splitTestsCompilation` Config Field ## Overview @@ -264,25 +266,21 @@ When `splitTestsCompilation === true`: ### Solidity Test Runner (`hardhat test solidity`) +Before branching on `splitTestsCompilation`, the runner validates that all provided `testFiles` are classified as tests by `getScope()`, throwing `SELECTED_FILES_ARENT_SOLIDITY_TESTS` if any are not. This validation runs in both modes. + When `splitTestsCompilation === false`: - `noCompile === true` skips compilation entirely -- `noCompile !== true` performs one full unified build -- `testFiles` only controls which tests are executed -- partial Solidity test runs may still compile all Solidity tests as a temporary limitation -- the runner must compute the selected test roots independently from the build return value +- `noCompile !== true` calls `build({ files: testFiles })` once, without `noTests` or `noContracts` — a full build when `testFiles` is empty, a partial build of the specified files otherwise +- `testFiles` controls both which files are compiled and which tests are executed +- the runner uses `testRootPaths` from the build return value to determine which tests to run - when `noCompile === true`, selected test roots must still be validated against the compiled artifacts available on disk -- if a selected Solidity test file exists but has not been compiled, the task throws a `HardhatError` +- if a selected Solidity test file exists but has not been compiled, the task throws `SELECTED_TEST_FILES_NOT_COMPILED` - only the selected test roots are used for: - deciding which suites to execute - deprecated-test warnings - artifacts and build info are read from a single directory: `getArtifactsDirectory("contracts")` -Important distinction in unified mode: - -- compiled test roots: all test roots produced by the unified build -- executed test roots: the tests requested by the user, or all test roots when no specific `testFiles` are provided - When `splitTestsCompilation === true`, current behavior is preserved: - the first build (contracts) is guarded by `noCompile` @@ -570,6 +568,7 @@ Rewrite the high-level build task to implement the new unified-mode semantics. ### Changes 1. `packages/hardhat/src/internal/builtin-plugins/solidity/tasks/build.ts` + - Add mode-independent validation before the `splitTestsCompilation` branch: - if `--no-contracts` and any explicit file is a contract, throw `INCOMPATIBLE_FILES_WITH_BUILD_FLAGS` - if `--no-tests` and any explicit file is a test, throw `INCOMPATIBLE_FILES_WITH_BUILD_FLAGS` @@ -591,7 +590,7 @@ Rewrite the high-level build task to implement the new unified-mode semantics. 2. `packages/hardhat-errors/src/descriptors.ts` - Replace `FILES_WITH_SCOPE_FILTERS_NOT_SUPPORTED` with `INCOMPATIBLE_FILES_WITH_BUILD_FLAGS` (same error number 917) - - `UNRECOGNIZED_FILES_NOT_COMPILED` (915) is no longer used in build.ts (still used by the solidity-test runner, addressed in Phase 6) + - `UNRECOGNIZED_FILES_NOT_COMPILED` (915) is no longer used in build.ts. After Phase 6, it is no longer used anywhere — the solidity-test runner replaces it with `SELECTED_FILES_ARENT_SOLIDITY_TESTS` (815) ### Validation @@ -625,7 +624,7 @@ Rewrite the high-level build task to implement the new unified-mode semantics. - split mode: explicit test files only (no flags) skips the contracts scope entirely - other split-mode regressions for current behavior - Run `pnpm test` in `packages/hardhat` -- **Known failures after Phase 4:** 2 tests fail because the solidity-test runner calls `build({ files: testFiles, noContracts: true })` with a file that `getScope()` classifies as a contract (not in the configured test path). The mode-independent validation catches this as an incompatible combination and throws `INCOMPATIBLE_FILES_WITH_BUILD_FLAGS` (917), but the tests expect the old `UNRECOGNIZED_FILES_NOT_COMPILED` (915). Both originate from `solidity-test/task-action.ts` and are fixed in Phase 6 when the solidity-test runner is updated. +- **Known failures after Phase 4:** 2 tests fail because the solidity-test runner calls `build({ files: testFiles, noContracts: true })` with a file that `getScope()` classifies as a contract (not in the configured test path). The mode-independent validation catches this as an incompatible combination and throws `INCOMPATIBLE_FILES_WITH_BUILD_FLAGS` (917), but the tests expect the old `UNRECOGNIZED_FILES_NOT_COMPILED` (915). Both originate from `solidity-test/task-action.ts`. Phase 6 resolves this by adding an early `getScope()`-based validation in the solidity-test runner that throws `SELECTED_FILES_ARENT_SOLIDITY_TESTS` (815) before the build call, so neither 917 nor 915 fires for this case. ## Phase 5: Other Built-In Task Callers @@ -660,35 +659,41 @@ Update the built-in tasks that currently call `build({ noTests: true })`. While ## Phase 6: Solidity Test Runner -Update the Solidity test runner for unified builds while preserving selected test execution. Note: the solidity-test runner currently uses `build({ files, noContracts: true })`, which is valid after Phase 4 (it produces a partial test-only build). However, in unified mode the runner should perform a full build instead, so this change is still needed for the correct full-build semantics. +Update the Solidity test runner for unified builds while preserving selected test execution. Note: the solidity-test runner currently uses `build({ files, noContracts: true })`, which is valid after Phase 4 (it produces a partial test-only build). In unified mode the runner drops `noContracts` and passes `files: testFiles` to build, performing selective compilation without scope flags. ### Changes 1. `packages/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts` + + - Before branching on `splitTestsCompilation`, validate that all provided `testFiles` are classified as tests by `getScope()`, throwing `SELECTED_FILES_ARENT_SOLIDITY_TESTS` if not - Branch on `hre.config.solidity.splitTestsCompilation` - Unified mode: - - if `noCompile !== true`, call `build()` once without `noTests` or `noContracts` - - compute selected test roots independently from the build return value + - if `noCompile !== true`, call `build({ files: testFiles })` once, without `noTests` or `noContracts` + - use `testRootPaths` from the build return value to determine which tests to run - when `noCompile === true`, validate that every selected Solidity test root has compiled artifacts available - - throw a `HardhatError` if a selected Solidity test file exists but was not compiled + - throw `SELECTED_TEST_FILES_NOT_COMPILED` if a selected Solidity test file exists but was not compiled - use selected test roots for suite execution and deprecated-test warnings - read artifacts and build info from the main artifacts directory only - - accept the temporary limitation that selected runs may still compile all Solidity tests - Split mode: - preserve the current two-build behavior +2. `packages/hardhat-errors/src/descriptors.ts` + - Add `SELECTED_TEST_FILES_NOT_COMPILED` (814) — thrown when `noCompile` is set and selected test files have not been compiled + - Add `SELECTED_FILES_ARENT_SOLIDITY_TESTS` (815) — thrown when non-test files are passed as test files + ### Validation - Run `pnpm lint` in `packages/hardhat` - Run `pnpm build` in `packages/hardhat` - Run existing Solidity test runner tests: `packages/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts` - Add tests for: - - unified mode performs one build + - early validation throws `SELECTED_FILES_ARENT_SOLIDITY_TESTS` for non-test files in both modes + - unified mode performs one build via `build({ files: testFiles })` - unified mode reads artifacts from a single directory - unified mode executes only the selected test files - - a non-selected failing test may be compiled but is not executed + - unified mode compiles only the selected test files (selective compilation) - deprecated-test warnings are emitted only for selected tests - - unified `noCompile === true` throws a `HardhatError` when a selected Solidity test file exists but has not been compiled + - unified `noCompile === true` throws `SELECTED_TEST_FILES_NOT_COMPILED` when a selected Solidity test file exists but has not been compiled - `noCompile === true` works in both modes - split-mode behavior remains unchanged - Run `pnpm test` in `packages/hardhat` diff --git a/packages/hardhat-errors/src/descriptors.ts b/packages/hardhat-errors/src/descriptors.ts index 9e0690c753c..bdb54ea1b46 100644 --- a/packages/hardhat-errors/src/descriptors.ts +++ b/packages/hardhat-errors/src/descriptors.ts @@ -1192,6 +1192,36 @@ Remaining test suites: {suites}`, websiteDescription: "An inline config key was used that does not apply to the type of test function it was attached to. Fuzz test functions (test*) only accept fuzz.* keys and top-level keys, while invariant test functions (invariant*) only accept invariant.* keys and top-level keys.", }, + SELECTED_TEST_FILES_NOT_COMPILED: { + number: 814, + messageTemplate: `The following Solidity test files have not been compiled: + +{files} + +Run \`hardhat build\` to compile your project before running tests with \`--no-compile\`.`, + websiteTitle: "Selected Solidity test files not compiled", + websiteDescription: `You ran Solidity tests with \`--no-compile\`, but some of the selected test files have not been compiled yet. Run \`hardhat build\` first, or remove the \`--no-compile\` flag.`, + }, + SELECTED_FILES_ARE_NOT_SOLIDITY_TESTS: { + number: 815, + messageTemplate: `Trying to run these files as Solidity tests, but they aren't: + +{files} + +Double-check the files that you are providing to the \`test solidity\` task`, + websiteTitle: "Invalid Solidity test files", + websiteDescription: `You ran the \`test solidity\` task with files that aren't classified as Solidity tests.`, + }, + SELECTED_TEST_FILES_DO_NOT_EXIST: { + number: 816, + messageTemplate: `The following Solidity test files do not exist: + +{files} + +Double-check the paths you are providing to the \`test solidity\` task.`, + websiteTitle: "Selected Solidity test files do not exist", + websiteDescription: `You ran the \`test solidity\` task with files that do not exist on disk.`, + }, }, SOLIDITY: { PROJECT_ROOT_RESOLUTION_ERROR: { diff --git a/packages/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts b/packages/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts index e5f41302735..5c38ce7e56e 100644 --- a/packages/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts +++ b/packages/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts @@ -31,7 +31,7 @@ import { formatArtifactId } from "./formatters.js"; * Despite the changes, the signature of the function should still be considered * a draft that may change in the future. * - * TODO: Once the signature is finalized, give feedback to the EDR team. + * Important TODO: Transform this into an AsyncGenerator */ export function run( chainType: ChainType, @@ -41,67 +41,75 @@ export function run( tracingConfig: TracingConfigWithBuffers, sourceNameToUserSourceName: Map, ): TestsStream { - const stream = new ReadableStream({ - async start(controller) { - if (testSuiteIds.length === 0) { - controller.close(); - return; - } - let runCompleted = false; + const stream = new Readable({ + objectMode: true, + read() {}, + }); - const remainingSuites = new Set( - testSuiteIds.map((id) => - formatArtifactId(id, sourceNameToUserSourceName), - ), - ); + if (testSuiteIds.length === 0) { + stream.push(null); + return stream; + } - // TODO: Add support for predeploys once EDR supports them. - try { - const edrContext = await getGlobalEdrContext(); - const solidityTestResult = await edrContext.runSolidityTests( - hardhatChainTypeToEdrChainType(chainType), - artifacts, - testSuiteIds, - testRunnerConfig, - tracingConfig, - (suiteResult) => { - controller.enqueue({ - type: "suite:done", - data: suiteResult, - }); - remainingSuites.delete( - formatArtifactId(suiteResult.id, sourceNameToUserSourceName), - ); - if (remainingSuites.size === 0) { - if (runCompleted) { - controller.close(); - } - } - }, - ); - controller.enqueue({ - type: "run:done", - data: solidityTestResult, - }); - runCompleted = true; + let runCompleted = false; + + const remainingSuites = new Set( + testSuiteIds.map((id) => formatArtifactId(id, sourceNameToUserSourceName)), + ); - if (remainingSuites.size === 0) { - controller.close(); - } - } catch (error) { - ensureError(error); + // Start the async work immediately. The read() callback is a no-op + // because we push data proactively from the EDR suite-completion + // callback. Using a native Readable (instead of a web ReadableStream + // wrapped with Readable.from) avoids a race where Node.js stream + // cleanup cancels the web reader while the async start callback still + // has pending work — push() on a destroyed Readable is a safe no-op. + // TODO: Add support for predeploys once EDR supports them. + void (async () => { + try { + const edrContext = await getGlobalEdrContext(); + const solidityTestResult = await edrContext.runSolidityTests( + hardhatChainTypeToEdrChainType(chainType), + artifacts, + testSuiteIds, + testRunnerConfig, + tracingConfig, + (suiteResult) => { + stream.push({ + type: "suite:done", + data: suiteResult, + } satisfies TestEvent); + remainingSuites.delete( + formatArtifactId(suiteResult.id, sourceNameToUserSourceName), + ); + if (remainingSuites.size === 0) { + if (runCompleted) { + stream.push(null); + } + } + }, + ); + stream.push({ + type: "run:done", + data: solidityTestResult, + } satisfies TestEvent); + runCompleted = true; - controller.error( - new HardhatError( - HardhatError.ERRORS.CORE.SOLIDITY_TESTS.UNHANDLED_EDR_ERROR_SOLIDITY_TESTS, - { - error: error.message, - }, - ), - ); + if (remainingSuites.size === 0) { + stream.push(null); } - }, - }); + } catch (error) { + ensureError(error); + + stream.destroy( + new HardhatError( + HardhatError.ERRORS.CORE.SOLIDITY_TESTS.UNHANDLED_EDR_ERROR_SOLIDITY_TESTS, + { + error: error.message, + }, + ), + ); + } + })(); - return Readable.from(stream); + return stream; } diff --git a/packages/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts b/packages/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts index b8bb7c8b2a2..856084d606b 100644 --- a/packages/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts +++ b/packages/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts @@ -3,6 +3,7 @@ import type { EdrArtifactWithMetadata, } from "./edr-artifacts.js"; import type { TestEvent } from "./types.js"; +import type { SolidityBuildSystem } from "../../../types/solidity.js"; import type { NewTaskActionFunction } from "../../../types/tasks.js"; import type { TestRunResult } from "../../../types/test.js"; import type { Result } from "../../../types/utils.js"; @@ -15,6 +16,7 @@ import type { import { finished } from "node:stream/promises"; import { HardhatError } from "@nomicfoundation/hardhat-errors"; +import { exists } from "@nomicfoundation/hardhat-utils/fs"; import { resolveFromRoot } from "@nomicfoundation/hardhat-utils/path"; import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream"; @@ -59,6 +61,15 @@ const runSolidityTests: NewTaskActionFunction = async ( process.env.HH_TEST = "true"; const verbosity = hre.globalOptions.verbosity; + const resolvedTestFilesArgument = testFiles.map((f) => + resolveFromRoot(process.cwd(), f), + ); + + await validateThatProvidedFilesAreTests( + hre.solidity, + testFiles, + resolvedTestFilesArgument, + ); // Sets the NODE_ENV environment variable to "test" so the code can detect that tests are running // This is done by other JS/TS test frameworks like vitest @@ -75,34 +86,79 @@ const runSolidityTests: NewTaskActionFunction = async ( ); } - // Run the build task for contract files if needed - if (noCompile !== true) { - await hre.tasks.getTask("build").run({ - noTests: true, - }); - } + let testRootPathsToRun: string[]; + let edrArtifactsWithMetadata: EdrArtifactWithMetadata[]; + let allBuildInfosAndOutputs: BuildInfoAndOutput[]; - // Run the build task for test files - const { testRootPaths }: { testRootPaths: string[] } = await hre.tasks - .getTask("build") - .run({ - files: testFiles, - noContracts: true, - }); - console.log(); + if (hre.config.solidity.splitTestsCompilation) { + if (noCompile !== true) { + await hre.tasks.getTask("build").run({ + noTests: true, + }); + } - // EDR needs all artifacts (contracts + tests) - const edrArtifactsWithMetadata: EdrArtifactWithMetadata[] = []; - const allBuildInfosAndOutputs: BuildInfoAndOutput[] = []; - for (const scope of ["contracts", "tests"] as const) { - const artifactsDir = await hre.solidity.getArtifactsDirectory(scope); - const artifactManager = new ArtifactManagerImplementation(artifactsDir); - edrArtifactsWithMetadata.push( - ...(await buildEdrArtifactsWithMetadata(artifactManager)), - ); - allBuildInfosAndOutputs.push( - ...(await getBuildInfosAndOutputs(artifactManager)), - ); + ({ testRootPaths: testRootPathsToRun } = await hre.tasks + .getTask("build") + .run({ + files: testFiles, + noContracts: true, + })); + console.log(); + + ({ edrArtifactsWithMetadata, allBuildInfosAndOutputs } = + await loadArtifacts(hre.solidity, ["contracts", "tests"])); + } else { + if (noCompile !== true) { + ({ testRootPaths: testRootPathsToRun } = await hre.tasks + .getTask("build") + .run({ + files: testFiles, + })); + } else { + if (resolvedTestFilesArgument.length > 0) { + testRootPathsToRun = resolvedTestFilesArgument; + } else { + testRootPathsToRun = []; + const allRoots = await hre.solidity.getRootFilePaths({ + scope: "contracts", + }); + + for (const root of allRoots) { + if ((await hre.solidity.getScope(root)) === "tests") { + testRootPathsToRun.push(root); + } + } + } + } + console.log(); + + ({ edrArtifactsWithMetadata, allBuildInfosAndOutputs } = + await loadArtifacts(hre.solidity, ["contracts"])); + + // When noCompile, validate selected test roots have compiled artifacts + if (noCompile === true) { + const compiledSources = new Set( + edrArtifactsWithMetadata.map(({ userSourceName }) => + resolveFromRoot(hre.config.paths.root, userSourceName), + ), + ); + + const notCompiledFiles: string[] = []; + for (const root of testRootPathsToRun) { + if (!compiledSources.has(root)) { + notCompiledFiles.push(root); + } + } + + if (notCompiledFiles.length > 0) { + throw new HardhatError( + HardhatError.ERRORS.CORE.SOLIDITY_TESTS.SELECTED_TEST_FILES_NOT_COMPILED, + { + files: notCompiledFiles.map((f) => `- ${f}`).join("\n"), + }, + ); + } + } } const sourceNameToUserSourceName = new Map( @@ -112,25 +168,19 @@ const runSolidityTests: NewTaskActionFunction = async ( ]), ); - edrArtifactsWithMetadata.forEach(({ userSourceName, edrArtifact }) => { - if ( - testRootPaths.includes( - resolveFromRoot(hre.config.paths.root, userSourceName), - ) && - isTestSuiteArtifact(edrArtifact) - ) { - warnDeprecatedTestFail(edrArtifact, sourceNameToUserSourceName); - } - }); - + const testRootPathsSet = new Set(testRootPathsToRun); const testSuiteArtifacts = edrArtifactsWithMetadata .filter(({ userSourceName }) => - testRootPaths.includes( + testRootPathsSet.has( resolveFromRoot(hre.config.paths.root, userSourceName), ), ) .filter(({ edrArtifact }) => isTestSuiteArtifact(edrArtifact)); + for (const { edrArtifact } of testSuiteArtifacts) { + warnDeprecatedTestFail(edrArtifact, sourceNameToUserSourceName); + } + const testSuiteIds = testSuiteArtifacts.map( ({ edrArtifact }) => edrArtifact.id, ); @@ -310,4 +360,70 @@ const runSolidityTests: NewTaskActionFunction = async ( : successfulResult(result); }; +/** + * Validates that the test files provided by the user, resolved in this case, + * are actually test files. + * + * @param solidity The solidity build system + * @param testFiles The test files, as provided by the user + * @param resolvedTestFilesArgument The resolved testFiles + */ +async function validateThatProvidedFilesAreTests( + solidity: SolidityBuildSystem, + testFiles: string[], + resolvedTestFilesArgument: string[], +) { + const existsResults = await Promise.all( + resolvedTestFilesArgument.map((rootPath) => exists(rootPath)), + ); + + const missing: string[] = testFiles.filter((_, i) => !existsResults[i]); + + if (missing.length > 0) { + throw new HardhatError( + HardhatError.ERRORS.CORE.SOLIDITY_TESTS.SELECTED_TEST_FILES_DO_NOT_EXIST, + { + files: missing.map((f) => `- ${f}`).join("\n"), + }, + ); + } + + const scopes = await Promise.all( + resolvedTestFilesArgument.map((rootPath) => solidity.getScope(rootPath)), + ); + + const nonTests: string[] = testFiles.filter((_, i) => scopes[i] !== "tests"); + + if (nonTests.length > 0) { + throw new HardhatError( + HardhatError.ERRORS.CORE.SOLIDITY_TESTS.SELECTED_FILES_ARE_NOT_SOLIDITY_TESTS, + { + files: nonTests.map((f) => `- ${f}`).join("\n"), + }, + ); + } +} + +async function loadArtifacts( + solidity: SolidityBuildSystem, + scopes: Array<"contracts" | "tests">, +): Promise<{ + edrArtifactsWithMetadata: EdrArtifactWithMetadata[]; + allBuildInfosAndOutputs: BuildInfoAndOutput[]; +}> { + const edrArtifactsWithMetadata: EdrArtifactWithMetadata[] = []; + const allBuildInfosAndOutputs: BuildInfoAndOutput[] = []; + for (const scope of scopes) { + const artifactsDir = await solidity.getArtifactsDirectory(scope); + const artifactManager = new ArtifactManagerImplementation(artifactsDir); + edrArtifactsWithMetadata.push( + ...(await buildEdrArtifactsWithMetadata(artifactManager)), + ); + allBuildInfosAndOutputs.push( + ...(await getBuildInfosAndOutputs(artifactManager)), + ); + } + return { edrArtifactsWithMetadata, allBuildInfosAndOutputs }; +} + export default runSolidityTests; diff --git a/packages/hardhat/src/internal/builtin-plugins/solidity/tasks/build.ts b/packages/hardhat/src/internal/builtin-plugins/solidity/tasks/build.ts index 15b0bc85c12..80227e00ce6 100644 --- a/packages/hardhat/src/internal/builtin-plugins/solidity/tasks/build.ts +++ b/packages/hardhat/src/internal/builtin-plugins/solidity/tasks/build.ts @@ -400,15 +400,13 @@ async function partitionRootPathsByScope( * If it's a relative path it's resolved from the CWD. */ function normalizedRootPaths(files: string[]): string[] { - const normalizedPaths = files.map((f) => { + return files.map((f) => { if (isNpmRootPath(f)) { return f; } return resolveFromRoot(process.cwd(), f); }); - - return normalizedPaths; } export default buildAction; diff --git a/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/DeprecatedTestFail.t.sol b/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/DeprecatedTestFail.t.sol new file mode 100644 index 00000000000..92fc84a776d --- /dev/null +++ b/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/DeprecatedTestFail.t.sol @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract DeprecatedTestFailTest { + function testFailDeprecated() public pure { + // Intentionally uses the deprecated testFail* prefix + } +} diff --git a/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/NormalTest.t.sol b/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/NormalTest.t.sol new file mode 100644 index 00000000000..e73be899b5d --- /dev/null +++ b/packages/hardhat/test/fixture-projects/solidity-test/test/contracts/deprecated/NormalTest.t.sol @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import "../../../contracts/Counter.sol"; + +contract NormalTest { + function testNormalPassing() public pure { + // A normal passing test + } +} diff --git a/packages/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts b/packages/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts index e10e681d070..5caeff90bf0 100644 --- a/packages/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts +++ b/packages/hardhat/test/internal/builtin-plugins/solidity-test/task-action.ts @@ -23,6 +23,13 @@ import hardhatConfig from "../../../fixture-projects/solidity-test/hardhat.confi * If it fails, unintended files were executed. */ +// Covers all test subdirectories so a single build produces artifacts +// for every test file in the fixture project. +const hardhatConfigAllTestDirs = { + ...hardhatConfig, + paths: { tests: { solidity: "test/contracts" } }, +}; + const hardhatConfigAllTests = { ...hardhatConfig, paths: { tests: { solidity: "test/contracts/all" } }, @@ -54,9 +61,14 @@ describe("solidity-test/task-action", function () { useFixtureProject("solidity-test"); before(async function () { - hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); + // Build with a config that covers all test subdirectories so that + // noCompile: true tests find pre-compiled artifacts on disk. + const buildHre = await createHardhatRuntimeEnvironment( + hardhatConfigAllTestDirs, + ); + await buildHre.tasks.getTask(["build"]).run({}); - await hre.tasks.getTask(["build"]).run({}); + hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); }); describe("when the solidity task test runner is specified", () => { @@ -83,7 +95,8 @@ describe("solidity-test/task-action", function () { noCompile: true, testFiles: ["./test/not-in-test-path.t.sol"], }), - HardhatError.ERRORS.CORE.SOLIDITY.UNRECOGNIZED_FILES_NOT_COMPILED, + HardhatError.ERRORS.CORE.SOLIDITY_TESTS + .SELECTED_FILES_ARE_NOT_SOLIDITY_TESTS, { files: "- ./test/not-in-test-path.t.sol" }, ); }); @@ -112,7 +125,8 @@ describe("solidity-test/task-action", function () { noCompile: true, testFiles: ["./test/not-in-test-path.t.sol"], }), - HardhatError.ERRORS.CORE.SOLIDITY.UNRECOGNIZED_FILES_NOT_COMPILED, + HardhatError.ERRORS.CORE.SOLIDITY_TESTS + .SELECTED_FILES_ARE_NOT_SOLIDITY_TESTS, { files: "- ./test/not-in-test-path.t.sol" }, ); }); @@ -243,14 +257,17 @@ describe("solidity-test/task-action", function () { describe("building contracts and tests", () => { /** * Returns an HRE that accumulates the args to `build` in the array it - * returns + * returns. */ - async function getHreWithOverriddenBuild(): Promise< - [hre: HardhatRuntimeEnvironment, buildArgs: any[]] - > { + async function getHreWithOverriddenBuild( + splitTestsCompilation: boolean, + ): Promise<[hre: HardhatRuntimeEnvironment, buildArgs: any[]]> { const buildArgs: any[] = []; const overriddenHre = await createHardhatRuntimeEnvironment({ ...hardhatConfigAllTests, + ...(splitTestsCompilation + ? { solidity: { version: "0.8.28", splitTestsCompilation: true } } + : {}), tasks: [ overrideTask("build") .setAction(async () => { @@ -269,85 +286,218 @@ describe("solidity-test/task-action", function () { return [overriddenHre, buildArgs]; } - describe("When noCompile is provided", () => { - it("Should compile the test files, but not the contracts", async () => { - const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); + describe("when splitTestsCompilation is true", () => { + describe("When noCompile is provided", () => { + it("Should compile the test files, but not the contracts", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(true); - await overriddenHre.tasks.getTask(["test", "solidity"]).run({ - noCompile: true, + await overriddenHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + }); + + // We only call build once + assert.equal(buildArgs.length, 1); + + const lastArgs = buildArgs[0]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, []); }); - // We only call build once - assert.equal(buildArgs.length, 1); + it("Should compile only the provided test files, and not the contracts", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(true); + + const testFiles = ["test/contracts/all/Counter-1.t.sol"]; + await overriddenHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + testFiles, + }); - const lastArgs = buildArgs[0]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, []); + // We only call build once + assert.equal(buildArgs.length, 1); + + const lastArgs = buildArgs[0]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, testFiles); + }); }); - it("Should compile only the provided test files, and not the contracts", async () => { - const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); + describe("When noCompile is not provided", () => { + it("Should compile the contracts and then the test files", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(true); - const testFiles = ["test/contracts/all/Counter-1.t.sol"]; - await overriddenHre.tasks.getTask(["test", "solidity"]).run({ - noCompile: true, - testFiles, + await overriddenHre.tasks.getTask(["test", "solidity"]).run({}); + + assert.equal(buildArgs.length, 2); + + const firstArgs = buildArgs[0]; + assert.equal(firstArgs.noContracts, false); + assert.equal(firstArgs.noTests, true); + assert.deepEqual(firstArgs.files, []); + + const lastArgs = buildArgs[1]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, []); }); - // We only call build once - assert.equal(buildArgs.length, 1); + it("Should compile the contracts and then the provided test files", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(true); + + const testFiles = ["test/contracts/all/Counter-1.t.sol"]; + await overriddenHre.tasks + .getTask(["test", "solidity"]) + .run({ testFiles }); + + assert.equal(buildArgs.length, 2); - const lastArgs = buildArgs[0]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, testFiles); + const firstArgs = buildArgs[0]; + assert.equal(firstArgs.noContracts, false); + assert.equal(firstArgs.noTests, true); + assert.deepEqual(firstArgs.files, []); + + const lastArgs = buildArgs[1]; + assert.equal(lastArgs.noContracts, true); + assert.equal(lastArgs.noTests, false); + assert.deepEqual(lastArgs.files, testFiles); + }); }); }); - describe("When noCompile is not provided", () => { - it("Should compile the contracts and then the test files", async () => { - const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); + describe("when splitTestsCompilation is false", () => { + it("should perform one build when noCompile is not provided", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(false); await overriddenHre.tasks.getTask(["test", "solidity"]).run({}); - assert.equal(buildArgs.length, 2); - - const firstArgs = buildArgs[0]; - assert.equal(firstArgs.noContracts, false); - assert.equal(firstArgs.noTests, true); - assert.deepEqual(firstArgs.files, []); + assert.equal(buildArgs.length, 1); - const lastArgs = buildArgs[1]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, []); + const args = buildArgs[0]; + assert.equal(args.noTests, false); + assert.equal(args.noContracts, false); }); - it("Should compile the contracts and then the provided test files", async () => { - const [overriddenHre, buildArgs] = await getHreWithOverriddenBuild(); + it("should perform one build with selected test files", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(false); const testFiles = ["test/contracts/all/Counter-1.t.sol"]; await overriddenHre.tasks .getTask(["test", "solidity"]) .run({ testFiles }); - assert.equal(buildArgs.length, 2); + assert.equal(buildArgs.length, 1); + + const args = buildArgs[0]; + assert.equal(args.noTests, false); + assert.equal(args.noContracts, false); + }); + + it("should not call build when noCompile is provided", async () => { + const [overriddenHre, buildArgs] = + await getHreWithOverriddenBuild(false); - const firstArgs = buildArgs[0]; - assert.equal(firstArgs.noContracts, false); - assert.equal(firstArgs.noTests, true); - assert.deepEqual(firstArgs.files, []); + await overriddenHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + }); - const lastArgs = buildArgs[1]; - assert.equal(lastArgs.noContracts, true); - assert.equal(lastArgs.noTests, false); - assert.deepEqual(lastArgs.files, testFiles); + assert.equal(buildArgs.length, 0); }); }); }); }); + describe("when splitTestsCompilation is false", () => { + it("should execute only the selected test files", async () => { + hre = await createHardhatRuntimeEnvironment(hardhatConfigPartialTests); + + const result = await hre.tasks.getTask(["test", "solidity"]).run({ + testFiles: ["./test/contracts/partial/Counter-1.sol"], + }); + assert.equal(result.success, true); + }); + + it("should read artifacts from a single directory", async () => { + hre = await createHardhatRuntimeEnvironment(hardhatConfigAllTests); + + const result = await hre.tasks.getTask(["test", "solidity"]).run({}); + assert.equal(result.success, true); + }); + + it("should only emit deprecated-test warnings for selected tests", async () => { + const deprecatedConfig = { + ...hardhatConfig, + paths: { tests: { solidity: "test/contracts/deprecated" } }, + }; + const deprecatedHre = + await createHardhatRuntimeEnvironment(deprecatedConfig); + + const warnings: string[] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + await deprecatedHre.tasks.getTask(["test", "solidity"]).run({ + testFiles: ["./test/contracts/deprecated/NormalTest.t.sol"], + }); + } finally { + console.warn = originalWarn; + } + + assert.equal( + warnings.filter((w) => w.includes("testFail")).length, + 0, + "No testFail deprecation warning should be emitted for non-selected tests", + ); + }); + + it("should throw when a selected test file exists but has not been compiled", async () => { + const notBuildConfig = { + ...hardhatConfig, + paths: { tests: { solidity: "test" } }, + }; + const notBuiltHre = await createHardhatRuntimeEnvironment(notBuildConfig); + + try { + await notBuiltHre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + testFiles: ["./test/not-in-test-path.t.sol"], + }); + assert.fail("Expected HardhatError to be thrown"); + } catch (error) { + assert.ok( + HardhatError.isHardhatError(error), + "Expected a HardhatError", + ); + assert.equal( + error.number, + HardhatError.ERRORS.CORE.SOLIDITY_TESTS + .SELECTED_TEST_FILES_NOT_COMPILED.number, + ); + } + }); + + it("should throw when a selected test file does not exist on disk", async () => { + hre = await createHardhatRuntimeEnvironment(hardhatConfigPartialTests); + await assertRejectsWithHardhatError( + hre.tasks.getTask(["test", "solidity"]).run({ + noCompile: true, + testFiles: ["./test/contracts/partial/DoesNotExist.t.sol"], + }), + HardhatError.ERRORS.CORE.SOLIDITY_TESTS + .SELECTED_TEST_FILES_DO_NOT_EXIST, + { files: "- ./test/contracts/partial/DoesNotExist.t.sol" }, + ); + }); + }); + it("should support EIP-7212 precompile at address 0x100", async () => { hre = await createHardhatRuntimeEnvironment(hardhatConfigHardforkTests);