fix(evals): keep expected answers hidden and preserve each case’s settings

.github/workflows/node-ci.yml

@@ -411,9 +411,14 @@ jobs:
           path: reports/python.xml
           if-no-files-found: warn
           retention-days: 14
-      - name: Build SDK and type-check eval tooling
+      - name: Set up Node.js for triage evals
         if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
-        run: pnpm --dir sdk/typescript run build:evals
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "24.15.0"
+      - name: Set up triage eval dependencies and host runtime
+        if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+        run: pnpm --dir evals/triage-finding run setup
       - name: Test deterministic triage eval contracts
         if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
         working-directory: evals/triage-finding

.github/workflows/test-quality.yml

@@ -32,24 +32,28 @@ jobs:
     if: ${{ !cancelled() && (inputs.native-artifacts-ready || needs.native.result == 'success') }}
     name: ${{ matrix.os }} / ${{ matrix.mode }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: ${{ matrix.os == 'windows-latest' && (matrix.mode == 'baseline' && 60 || matrix.mode == 'parallel' && 45) || 30 }}
+    timeout-minutes: ${{ matrix.os == 'windows-latest' && (matrix.mode == 'baseline' && 75 || matrix.mode == 'parallel' && 45) || 30 }}
     env:
       CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        mode: [baseline, isolated, parallel]
-        exclude:
-          - os: windows-latest
-            mode: isolated
+        mode: [baseline, parallel]
         include:
           - mode: baseline
             args: ""
-          - mode: isolated
-            args: --isolate
           - mode: parallel
             args: --parallel=2
+          - os: ubuntu-latest
+            mode: isolated-1
+            args: --isolate --shard=1/3
+          - os: ubuntu-latest
+            mode: isolated-2
+            args: --isolate --shard=2/3
+          - os: ubuntu-latest
+            mode: isolated-3
+            args: --isolate --shard=3/3
           - os: windows-latest
             mode: isolated-1
             args: --isolate --shard=1/7

evals/secret-discovery/runtime.mts

@@ -44,7 +44,7 @@ export async function withEvalState(
   let root;
   let home;
   try {
-    root = await mkdtemp(join(tmpdir(), "source-audit-"));
+    root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));
     // Share saved login updates without importing the caller's configuration.
     let auth;
     try {

evals/secret-discovery/test_runtime.mts

@@ -775,3 +775,40 @@ async ({ root, home, signal }) => {
     },
   );
 }
+
+test(
+  "fixture roots are canonical when the temporary directory is a symlink",
+  unixOnly,
+  async (t) => {
+    const directory = await mkdtemp(join(tmpdir(), "eval-aliased-temp-"));
+    t.after(() => rm(directory, { recursive: true, force: true }));
+    const actual = join(directory, "actual");
+    const alias = join(directory, "alias");
+    await mkdir(actual);
+    await symlink(actual, alias);
+    childProcess.execFileSync(
+      process.execPath,
+      [
+        "--experimental-strip-types",
+        "--input-type=module",
+        "--eval",
+        `
+    import assert from "node:assert/strict";
+    import { mkdtemp, realpath } from "node:fs/promises";
+    import { tmpdir } from "node:os";
+    import { join } from "node:path";
+    import { withEvalState } from ${JSON.stringify(runtimeUrl)};
+    import { gradeResult } from ${JSON.stringify(new URL("./grade.mts", import.meta.url).href)};
+    await withEvalState(async () => await mkdtemp(join(tmpdir(), "home-")), join(tmpdir(), "missing-synthetic-home"), async ({root}) => {
+      assert.equal(root, await realpath(root));
+      const result = { findings: [], coverage: { completeness: "complete", explicitExclusions: [{ pattern: await realpath(root) }] } };
+      const grade = gradeResult(result, { positives: [], files: { "fixture.txt": "synthetic" } }, root);
+      assert.ok(grade.errors.includes("incomplete coverage"));
+    });
+  `,
+      ],
+      { env: { ...process.env, TMPDIR: alias }, stdio: "pipe" },
+    );
+    assert.deepEqual(await readdir(actual), []);
+  },
+);

evals/triage-finding/README.md

@@ -4,7 +4,7 @@ This Promptfoo suite verifies that `$codex-security:triage-finding` accepts the
 JSON result shape. It also covers bare skill invocation with no supplied finding,
 which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub intake cases cover repository-source selection, REST endpoints, and explicit Connector requests without querying live GitHub during the eval.
 
-The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. Codex runs from the checkout's plugin directory and reads its triage skill directly, with ambient plugins and memory disabled. The default suite can also read the synthetic fixtures; calibration cases can read their own hydrated checkout. Dataset labels, assertions, and calibration Git history stay outside those readable roots. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.
+The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. The common runner stages the checkout's triage skill and policy helper in a throwaway directory, with ambient plugins and memory disabled. The default suite can also read the synthetic fixtures; calibration cases can read their own hydrated checkout. Dataset labels, assertions, and calibration Git history stay outside those readable roots. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.
 
 Use Node.js 22.22.0 or newer for the eval runner. Run these commands from the repository root to install dependencies under this eval directory.
 
@@ -18,15 +18,16 @@ pnpm --dir evals/triage-finding run setup
 
 After editing the tooling, rerun `pnpm --dir sdk/typescript run build:evals`. It prepares the SDK and type-checks the eval sources. Node and Promptfoo execute the TypeScript sources directly; package commands enable type stripping for Node 22.13.
 
-Before running model evals, build the checkout's policy helper (requires the [native build prerequisites](../../plugins/codex-security/native/README.md)):
+Before running the eval commands, prepare the native policy helper (requires the [native build prerequisites](../../plugins/codex-security/native/README.md)):
 
 ```bash
 pnpm --dir plugins/codex-security/mcp-app install --frozen-lockfile
 node plugins/codex-security/mcp-app/scripts/build_native.mjs
-node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host
 ```
 
-Config validation and deterministic tests do not need this helper build.
+The common runner builds the staged helper for validation, evaluation, and replay, then removes its temporary runtime after Promptfoo exits. The runtime extension runs after Promptfoo loads the native Codex provider, preserving its authentication precedence. A pinned Promptfoo patch runs saved `:beforeAll` startup hooks during viewer replay. Replay does not restore grading metadata, so grading hooks remain disabled.
+
+Saved runs whose provider ID is `file://.../triage-provider.mts` need to be rerun with the current configuration before retry, resume, or replay. Their saved results remain intact. Native Codex provider IDs are unchanged.
 
 Validate the config:
 

evals/triage-finding/assertions/github-rest-intake.mts

@@ -13,8 +13,12 @@ function endpointPattern(path: string, queryParts: string[] = []) {
     const repository = repositoryName(context.vars.target_repo as string);
     const paths = [path, path.replace("{owner}/{repo}", repository)];
     return (
-      paths.some((candidate) => escapedLiteralPattern(candidate).test(text)) &&
-      queryParts.every((part) => escapedLiteralPattern(part).test(text))
+      paths.some((candidate) =>
+        new RegExp(
+          escapedLiteralPattern(candidate).source + "\\b(?![/-]|\\.\\w)",
+          "i",
+        ).test(text),
+      ) && queryParts.every((part) => escapedLiteralPattern(part).test(text))
     );
   };
 }

evals/triage-finding/assertions/triage-io.mts

@@ -2,24 +2,9 @@ import type { AssertionContext } from "../types.ts";
 import { extractTriageResult, parseExpected } from "./output.mts";
 
 function expectedScanbenchMapping(context: AssertionContext) {
-  const caseId = String(context.vars.case_id || "");
   const label = String(context.vars.expected_binary_label || "");
   const mappings = [];
 
-  if (caseId.endsWith("-vulnerable")) {
-    mappings.push({
-      expectedVerdict: "confirmed",
-      expectedBinaryLabel: "positive",
-      label: "vulnerable scanbench case",
-    });
-  } else if (caseId.endsWith("-fixed")) {
-    mappings.push({
-      expectedVerdict: "not_actionable",
-      expectedBinaryLabel: "negative",
-      label: "fixed scanbench case",
-    });
-  }
-
   if (label === "positive") {
     mappings.push({
       expectedVerdict: "confirmed",
@@ -106,16 +91,15 @@ export default (output: unknown, context: AssertionContext) => {
           `${label}: not_actionable finding must use null rank and rank_queue`,
         );
       }
+    } else if (rankQueue !== finding.verdict) {
+      failures.push(
+        `${label}: rank_queue must match verdict ${finding.verdict}`,
+      );
     } else if (!Number.isInteger(rank) || rank! < 1) {
       failures.push(
         `${label}: actionable or unresolved finding must use a positive integer rank`,
       );
     } else {
-      if (rankQueue !== finding.verdict) {
-        failures.push(
-          `${label}: rank_queue must match verdict ${finding.verdict}, got ${rankQueue}`,
-        );
-      }
       const ranks = ranksByQueue.get(rankQueue) || [];
       ranks.push({ label, rank: rank! });
       ranksByQueue.set(rankQueue, ranks);

evals/triage-finding/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "description": "Promptfoo evals for the Codex Security triage-finding skill",
   "scripts": {
-    "setup": "pnpm install --frozen-lockfile && pnpm rebuild better-sqlite3 && pnpm --dir ../../sdk/typescript run build:evals",
+    "setup": "pnpm install --frozen-lockfile && pnpm rebuild better-sqlite3 && pnpm --dir ../../plugins/codex-security/mcp-app install --frozen-lockfile && pnpm --dir ../../sdk/typescript run build:evals && node ../../plugins/codex-security/mcp-app/scripts/build_native.mjs",
     "calibration:generate": "node --experimental-strip-types scripts/generate-calibration-tests.mts",
     "calibration:hydrate": "node --experimental-strip-types scripts/hydrate-calibration-repos.mts",
     "sastbench:install": "node --experimental-strip-types sastbench/scripts/install-sastbench.mts",
@@ -14,20 +14,20 @@
     "test:calibration-generate": "node --experimental-strip-types scripts/test-generate-calibration-tests.mts",
     "test:calibration-hydrate": "node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts",
     "test:dataset": "node --experimental-strip-types scripts/test-calibration-dataset.mts",
-    "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",
+    "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-promptfoo-runtime.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts && node --experimental-strip-types --test scripts/test-promptfoo-runner.mts scripts/test-triage-provider.mts",
     "test:instructions": "node --experimental-strip-types scripts/test-app-surface-instructions.mts",
     "test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",
-    "validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.yaml",
-    "validate:calibration": "pnpm run calibration:generate && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.calibration.yaml",
-    "validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./sastbench/promptfooconfig.sastbench.yaml",
-    "validate:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./sastbench/promptfooconfig.sastbench-sample.yaml",
-    "eval": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval.json --no-cache --no-share",
-    "eval:smoke": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval-smoke.json --no-cache --no-share --filter-range 0:1",
-    "eval:calibration": "pnpm run calibration:generate && pnpm run calibration:hydrate && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.calibration.yaml -o ./artifacts/triage-finding-calibration-eval.json --no-cache --no-share",
-    "eval:calibration:smoke": "node --experimental-strip-types scripts/generate-calibration-tests.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable --output artifacts/calibration-smoke.yaml && node --experimental-strip-types scripts/hydrate-calibration-repos.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./promptfooconfig.calibration-smoke.yaml -o ./artifacts/triage-finding-calibration-smoke.json --no-cache --no-share",
-    "eval:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./sastbench/promptfooconfig.sastbench.yaml --no-cache --no-share",
-    "eval:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo eval -c ./sastbench/promptfooconfig.sastbench-sample.yaml --no-cache --no-share --max-concurrency 32",
-    "pf:view": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo view"
+    "validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.yaml",
+    "validate:calibration": "pnpm run calibration:generate && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./promptfooconfig.calibration.yaml",
+    "validate:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench.yaml",
+    "validate:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts validate config -c ./sastbench/promptfooconfig.sastbench-sample.yaml",
+    "eval": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval.json --no-cache --no-share",
+    "eval:smoke": "mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.yaml -o ./artifacts/triage-finding-eval-smoke.json --no-cache --no-share --filter-range 0:1",
+    "eval:calibration": "pnpm run calibration:generate && pnpm run calibration:hydrate && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.calibration.yaml -o ./artifacts/triage-finding-calibration-eval.json --no-cache --no-share",
+    "eval:calibration:smoke": "node --experimental-strip-types scripts/generate-calibration-tests.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable --output artifacts/calibration-smoke.yaml && node --experimental-strip-types scripts/hydrate-calibration-repos.mts --case oss-dompurify-ghsa-v8jm-5vwx-cfxm --variant vulnerable && mkdir -p artifacts && PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./promptfooconfig.calibration-smoke.yaml -o ./artifacts/triage-finding-calibration-smoke.json --no-cache --no-share",
+    "eval:sastbench": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./sastbench/promptfooconfig.sastbench.yaml --no-cache --no-share",
+    "eval:sastbench:sample": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts eval -c ./sastbench/promptfooconfig.sastbench-sample.yaml --no-cache --no-share --max-concurrency 32",
+    "pf:view": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true node --experimental-strip-types scripts/run-promptfoo.mts view"
   },
   "dependencies": {
     "@openai/codex-sdk": "0.162.0-alpha.20",

evals/triage-finding/patches/promptfoo@0.123.1.patch

@@ -0,0 +1,24 @@
+diff --git a/dist/src/main.js b/dist/src/main.js
+index 63b3515f0a43d58b4bfa0407e2749b8803f53399..1ac5c28580050c90ba60d3fc2fad0bdee03ac9e8 100755
+--- a/dist/src/main.js
++++ b/dist/src/main.js
+@@ -7143,6 +7143,7 @@ evalRouter.post("/replay", async (req, res) => {
+ 				label: "Replay"
+ 			}],
+ 			providers: [providerConfig],
++			extensions: eval_.config.extensions?.filter((extension) => extension.endsWith(":beforeAll")),
+ 			tests: [{ vars: variables || {} }]
+ 		}, {
+ 			maxConcurrency: 1,
+diff --git a/dist/src/server/index.js b/dist/src/server/index.js
+index 4d3710840e68d5570ef35011a1ff4cc105e617ee..7681d249b4d9bee02c0347b438fdf971e6909282 100644
+--- a/dist/src/server/index.js
++++ b/dist/src/server/index.js
+@@ -61896,6 +61896,7 @@ evalRouter.post("/replay", async (req, res) => {
+ 				label: "Replay"
+ 			}],
+ 			providers: [providerConfig],
++			extensions: eval_.config.extensions?.filter((extension) => extension.endsWith(":beforeAll")),
+ 			tests: [{ vars: variables || {} }]
+ 		}, {
+ 			maxConcurrency: 1,