fix(evals): keep expected answers hidden and preserve each case’s settings

.github/scripts/invoice-desk-source.mjs

@@ -1,5 +1,11 @@
 import { execFileSync } from "node:child_process";
-import { closeSync, mkdirSync, openSync } from "node:fs";
+import {
+  existsSync,
+  realpathSync,
+  closeSync,
+  mkdirSync,
+  openSync,
+} from "node:fs";
 import { dirname, resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 
@@ -45,7 +51,9 @@ export function extractApplication(repository, sha, destination) {
 
 if (
   process.argv[1] &&
-  import.meta.url === pathToFileURL(process.argv[1]).href
+  process.argv[1] !== "-" &&
+  existsSync(process.argv[1]) &&
+  import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href
 ) {
   const count = extractApplication(
     process.env.SOURCE_REPOSITORY,

.github/scripts/invoice-desk-source.test.mjs

@@ -108,3 +108,40 @@ test("rejects lossy filename decoding before any source files can collide", (t)
   );
   assert.equal(existsSync(destination), false);
 });
+
+test(
+  "the source CLI runs through a symlink",
+  { skip: process.platform === "win32" },
+  async (t) => {
+    const { symlinkSync } = await import("node:fs");
+    const { fileURLToPath } = await import("node:url");
+    const { root, repository, git, destination } = fixture(t);
+    const script = join(root, "source.mjs");
+    symlinkSync(
+      fileURLToPath(new URL("./invoice-desk-source.mjs", import.meta.url)),
+      script,
+    );
+    execFileSync(process.execPath, [script], {
+      env: {
+        ...process.env,
+        SOURCE_REPOSITORY: repository,
+        SOURCE_SHA: git("write-tree"),
+        SCAN_SOURCE: destination,
+      },
+      stdio: "pipe",
+    });
+    assert.equal(
+      readFileSync(join(destination, "server.mjs"), "utf8"),
+      'export const version = "$Format:%H$";\n',
+    );
+  },
+);
+
+test("invoice helpers can be imported from a Node stdin module", () => {
+  const source = new URL("./invoice-desk-source.mjs", import.meta.url).href;
+  const target = new URL("./invoice-desk-target.mjs", import.meta.url).href;
+  execFileSync(process.execPath, ["--input-type=module", "-"], {
+    input: `await import(${JSON.stringify(source)}); await import(${JSON.stringify(target)});`,
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+});

.github/scripts/invoice-desk-target.mjs

@@ -1,5 +1,10 @@
 import { execFileSync } from "node:child_process";
-import { appendFileSync, readFileSync } from "node:fs";
+import {
+  existsSync,
+  realpathSync,
+  appendFileSync,
+  readFileSync,
+} from "node:fs";
 import { pathToFileURL } from "node:url";
 
 function eligible(pr, repository, sha) {
@@ -63,7 +68,9 @@ export async function resolveScanTargets(
 
 if (
   process.argv[1] &&
-  import.meta.url === pathToFileURL(process.argv[1]).href
+  process.argv[1] !== "-" &&
+  existsSync(process.argv[1]) &&
+  import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href
 ) {
   const repository = process.env.GITHUB_REPOSITORY;
   const targets = await resolveScanTargets(

.github/scripts/invoice-desk-target.test.mjs

@@ -294,3 +294,35 @@ test("fallback CLI dispatches a keyed run once and leaves inference to that run"
     `targets=${JSON.stringify([{ pr: 7, sha: sourceSha }])}`,
   );
 });
+
+test(
+  "the target CLI runs through a symlink",
+  { skip: process.platform === "win32" },
+  async (t) => {
+    const { symlinkSync } = await import("node:fs");
+    const root = mkdtempSync(join(tmpdir(), "invoice-desk-link-"));
+    t.after(() => rmSync(root, { recursive: true, force: true }));
+    const script = join(root, "target.mjs");
+    symlinkSync(
+      fileURLToPath(new URL("./invoice-desk-target.mjs", import.meta.url)),
+      script,
+    );
+    const event = join(root, "event.json");
+    const output = join(root, "output.txt");
+    writeFileSync(event, "{}");
+    execFileSync(process.execPath, [script], {
+      env: {
+        ...process.env,
+        GITHUB_EVENT_NAME: "workflow_dispatch",
+        GITHUB_EVENT_PATH: event,
+        GITHUB_SHA: mainSha,
+        GITHUB_OUTPUT: output,
+      },
+      stdio: "pipe",
+    });
+    assert.equal(
+      readFileSync(output, "utf8"),
+      `targets=${JSON.stringify([{ pr: 0, sha: mainSha }])}\n`,
+    );
+  },
+);

.github/workflows/node-ci.yml

@@ -474,8 +474,12 @@ jobs:
           path: reports/python.xml
           if-no-files-found: warn
           retention-days: 14
-      - name: Build SDK and type-check eval tooling
-        run: pnpm --dir sdk/typescript run build:evals
+      - name: Set up Node.js for triage evals
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "24.15.0"
+      - name: Set up triage eval dependencies and host runtime
+        run: pnpm --dir evals/triage-finding run setup
       - name: Test deterministic triage eval contracts
         working-directory: evals/triage-finding
         run: node --run test:deterministic

evals/secret-discovery/runtime.mts

@@ -44,7 +44,7 @@ export async function withEvalState(
   let root;
   let home;
   try {
-    root = await mkdtemp(join(tmpdir(), "source-audit-"));
+    root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));
     // Share saved login updates without importing the caller's configuration.
     let auth;
     try {

evals/secret-discovery/test_runtime.mts

@@ -774,3 +774,40 @@ async ({ root, home, signal }) => {
     },
   );
 }
+
+test(
+  "fixture roots are canonical when the temporary directory is a symlink",
+  unixOnly,
+  async (t) => {
+    const directory = await mkdtemp(join(tmpdir(), "eval-aliased-temp-"));
+    t.after(() => rm(directory, { recursive: true, force: true }));
+    const actual = join(directory, "actual");
+    const alias = join(directory, "alias");
+    await mkdir(actual);
+    await symlink(actual, alias);
+    childProcess.execFileSync(
+      process.execPath,
+      [
+        "--experimental-strip-types",
+        "--input-type=module",
+        "--eval",
+        `
+    import assert from "node:assert/strict";
+    import { mkdtemp, realpath } from "node:fs/promises";
+    import { tmpdir } from "node:os";
+    import { join } from "node:path";
+    import { withEvalState } from ${JSON.stringify(runtimeUrl)};
+    import { gradeResult } from ${JSON.stringify(new URL("./grade.mts", import.meta.url).href)};
+    await withEvalState(async () => await mkdtemp(join(tmpdir(), "home-")), join(tmpdir(), "missing-synthetic-home"), async ({root}) => {
+      assert.equal(root, await realpath(root));
+      const result = { findings: [], coverage: { completeness: "complete", explicitExclusions: [{ pattern: await realpath(root) }] } };
+      const grade = gradeResult(result, { positives: [], files: { "fixture.txt": "synthetic" } }, root);
+      assert.ok(grade.errors.includes("incomplete coverage"));
+    });
+  `,
+      ],
+      { env: { ...process.env, TMPDIR: alias }, stdio: "pipe" },
+    );
+    assert.deepEqual(await readdir(actual), []);
+  },
+);

evals/triage-finding/README.md

@@ -2,15 +2,15 @@
 
 This Promptfoo suite verifies that `$codex-security:triage-finding` accepts the current supported input source types and returns the expected `triage-finding/v0`
 JSON result shape. It also covers bare skill invocation with no supplied finding,
-which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub REST intake cases cover the repository-source control flow, endpoint selection, and connector auth-only rule without querying live GitHub during the eval.
+which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub REST intake cases cover the repository-source control flow, endpoint selection, and selected-connector behavior without querying live GitHub during the eval.
 
 The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.
 
 Use Node.js 22.22.0 or newer for the eval runner. Run these commands from the repository root to install dependencies under this eval directory.
 
 The eval directory has its own `pnpm-workspace.yaml` for its Promptfoo dependencies. Tooling type checks use the SDK’s TypeScript dependencies.
 
-Install the SDK and MCP app dependencies using the repository’s normal setup, then install the local eval runner and check the TypeScript tooling:
+Install the SDK and MCP app dependencies using the repository’s normal setup, then install the local eval runner, check the TypeScript tooling, and build the host native helpers with the pinned Rust toolchain from [the native build guide](../../plugins/codex-security/native/README.md); rerun setup after changing native source or its toolchain:
 
 ```bash
 pnpm --dir evals/triage-finding run setup
@@ -42,6 +42,8 @@ Run one case while iterating:
 pnpm --dir evals/triage-finding run eval --filter-metadata case_id=sarif-redirects
 ```
 
+The runner stages this checkout’s triage skill, shared references, schemas, and synthetic fixtures in a temporary directory, and builds the current MCP helper source into that runtime so the skill can resolve SECURITY.md policies. Codex reads the staged skill explicitly; installed plugins and memories are disabled. The read-only permission profile allows the staged runtime and, for calibration, only the selected repository checkout. Dataset labels and sibling checkouts remain outside those roots. The runner removes its temporary directory after normal completion or an interrupt. The provider binds to the current runtime when called, so Promptfoo resume and retry operations use a fresh runtime instead of a deleted directory from the saved configuration.
+
 The eval target is `fixtures/repo`, a small synthetic Express app with both true positive and false positive/review cases. Assertions are deterministic:
 
 - `contains-json` validates the fenced `triage-finding/v0` JSON block against `schemas/triage-result-v0.schema.json`.
@@ -51,11 +53,11 @@ The eval target is `fixtures/repo`, a small synthetic Express app with both true
 - `assertions/missing-input.mts` checks that bare invocation asks for a finding,
   names supported input formats, and does not emit triage result JSON.
 - `tests/github-rest-intake.yaml` opts out of default JSON assertions for GitHub repository-source routing cases.
-- `assertions/github-rest-intake.mts` checks GitHub source selection, REST endpoint selection, Codex project repository inference, advisory/private-report handling, connector auth-only behavior, and explicit-only GitHub Issue handling.
+- `assertions/github-rest-intake.mts` checks GitHub source selection, REST endpoint selection, Codex project repository inference, advisory/private-report handling, and explicit-only GitHub Issue handling. The selected-connector case grades a JSON workflow decision with the built-in schema assertion: retrieval transport, read-only access, missing capabilities, and approval before REST fallback.
 
 ## Calibration Dataset
 
-`datasets/triage-calibration-seed.json` is the first OSS-only calibration dataset for scaling beyond the synthetic fixture app. It contains public OSS vulnerable/fixed commit pairs. Each dataset variant becomes one Promptfoo test case in `tests/calibration-oss.yaml`, and each test points Codex at a pinned local checkout under `artifacts/calibration-repos/`.
+`datasets/triage-calibration-seed.json` is the first OSS-only calibration dataset for scaling beyond the synthetic fixture app. It contains public OSS vulnerable/fixed commit pairs. Each dataset variant becomes one Promptfoo test case in `tests/calibration-oss.yaml`, and each test points Codex at a pinned local checkout under `artifacts/calibration-repos/`. Case IDs, input IDs, and checkout directory names are opaque hashes of the dataset case and commit; vulnerable/fixed labels stay in grader metadata. Regenerate and rehydrate after upgrading from the older variant-named directories; those old ignored directories are no longer used.
 
 ELI5: the dataset says "this exact old commit should be affected" and "this exact fixed commit should not be affected." The generator turns those rows into Promptfoo test prompts. The hydrator downloads the exact repo commits so Codex can inspect real code instead of synthetic snippets.
 

evals/triage-finding/assertions/github-rest-intake.mts

@@ -1,11 +1,16 @@
 import { outputText as textFor, hasTriageJson } from "./output.mts";
 import type { AssertionContext } from "../types.ts";
+function containsAll(text: string, patterns: RegExp[]) {
+  return patterns.every((pattern) => pattern.test(text));
+}
+
 function endpointPattern(path: string, queryParts: string[] = []) {
-  const patterns = [
-    escapedLiteralPattern(path),
-    ...queryParts.map((part) => new RegExp(part, "i")),
-  ];
-  return (text: string) => patterns.every((pattern) => pattern.test(text));
+  const escapedPath = path
+    .replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+    .replace(/\\\{(?:owner|repo)\\\}/g, "[^/\\s?`]+");
+  const queryPatterns = queryParts.map((part) => new RegExp(part, "i"));
+  return (text: string) =>
+    new RegExp(escapedPath, "i").test(text) && containsAll(text, queryPatterns);
 }
 
 function escapedLiteralPattern(value: unknown) {
@@ -77,7 +82,9 @@ const checks = {
             "must use Dependabot alerts endpoint with classification=malware, state=open, and per_page=100",
           ]
         : []),
-      ...(!/source_type:\s*`?advisory`?|normalize as `?advisory`?/i.test(text)
+      ...(!/source_type["']?\s*:\s*[`"']?advisory\b|normalize as [`"']?advisory\b/i.test(
+        text,
+      )
         ? ["must say Dependabot malware normalizes as advisory"]
         : []),
     ];
@@ -89,7 +96,9 @@ const checks = {
       ["state=open", "per_page=100"],
     )(text);
     const hasInstances =
-      /code-scanning\/alerts\/\{alert_number\}\/instances/i.test(text);
+      /code-scanning\/alerts\/(?:\{alert_number\}|[0-9]+)\/instances/i.test(
+        text,
+      );
     return [
       ...(!hasAlerts
         ? [
@@ -99,7 +108,9 @@ const checks = {
       ...(!hasInstances
         ? ["must fetch code scanning alert instances per alert"]
         : []),
-      ...(!/source_type:\s*`?sarif`?|normalize as `?sarif`?/i.test(text)
+      ...(!/source_type["']?\s*:\s*[`"']?sarif\b|normalize as [`"']?sarif\b/i.test(
+        text,
+      )
         ? ["must say code scanning normalizes as sarif"]
         : []),
     ];
@@ -132,26 +143,10 @@ const checks = {
       )
         ? ["must identify state=triage as private vulnerability reports"]
         : []),
-      ...(!/source_type:\s*`?advisory`?|normalize as `?advisory`?/i.test(text)
-        ? ["must say advisories/private reports normalize as advisory"]
-        : []),
-    ];
-  },
-
-  connector_rest_only: (text: string) => {
-    return [
-      ...(!/GitHub Connector.*token|connector.*auth token|token.*GitHub Connector/is.test(
-        text,
-      )
-        ? ["must allow GitHub Connector only as an auth token source"]
-        : []),
-      ...(!/REST/i.test(text)
-        ? ["must state that finding retrieval uses REST"]
-        : []),
-      ...(!/do not use.*GitHub Connector.*(fetch|retrieve|data|findings)|GitHub Connector.*not.*(fetch|retrieve|data|findings)/is.test(
+      ...(!/source_type["']?\s*:\s*[`"']?advisory\b|normalize as [`"']?advisory\b/i.test(
         text,
       )
-        ? ["must say not to use the GitHub Connector for finding retrieval"]
+        ? ["must say advisories/private reports normalize as advisory"]
         : []),
     ];
   },
@@ -170,7 +165,9 @@ const checks = {
             "must say GitHub Issues are not included in all/default source selection",
           ]
         : []),
-      ...(!/source_type:\s*`?freeform`?|normalize as `?freeform`?/i.test(text)
+      ...(!/source_type["']?\s*:\s*[`"']?freeform\b|normalize as [`"']?freeform\b/i.test(
+        text,
+      )
         ? ["must say explicit GitHub Issues normalize as freeform"]
         : []),
     ];

evals/triage-finding/assertions/missing-input.mts

@@ -1,4 +1,4 @@
-import { outputText } from "./output.mts";
+import { outputText, hasTriageJson } from "./output.mts";
 
 function matchedFormatNames(text: string) {
   const supportedFormats: [string, RegExp][] = [
@@ -40,25 +40,12 @@ export default (output: unknown) => {
     );
   }
 
-  if (/```(?:json)?\s*[\s\S]*?```/i.test(text)) {
+  if (hasTriageJson(text)) {
     failures.push(
-      "response must not return a fenced JSON block when no finding was supplied",
+      "response must not emit a triage result JSON object when no finding was supplied",
     );
   }
 
-  for (const pattern of [
-    /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i,
-    /["']findings["']\s*:/i,
-    /["']verdict["']\s*:/i,
-  ]) {
-    if (pattern.test(text)) {
-      failures.push(
-        "response must not emit a triage result JSON object when no finding was supplied",
-      );
-      break;
-    }
-  }
-
   return {
     pass: failures.length === 0,
     score: failures.length === 0 ? 1 : 0,

evals/triage-finding/assertions/output.mts

@@ -6,7 +6,6 @@ export function outputText(output: unknown) {
 
 export function hasTriageJson(text: string) {
   return (
-    /```(?:json)?\s*[\s\S]*?```/i.test(text) ||
     /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||
     /["']findings["']\s*:/i.test(text) ||
     /["']verdict["']\s*:/i.test(text)

evals/triage-finding/assertions/ticket-intake.mts

@@ -27,7 +27,7 @@ const expectedSubissuePatterns: Record<string, RegExp[]> = {
   direct_confirmation: [
     /SEC-294/i,
     /SEC-295/i,
-    /2\s+(?:direct\s+)?sub-issues|two\s+(?:direct\s+)?sub-issues/i,
+    /2\s+(?:direct\s+)?(?:sub-issues|children)|two\s+(?:direct\s+)?(?:sub-issues|children)/i,
     /include|import/i,
     /ask|would you|do you want/i,
   ],
@@ -57,6 +57,12 @@ export default (output: unknown, context: AssertionContext) => {
   const patterns = expectedPatterns[behavior] || [];
   const subissuePatterns = expectedSubissuePatterns[subissueBehavior] || [];
   const failures = [];
+  if (behavior && !expectedPatterns[behavior])
+    failures.push(`unknown expected_ticket_failure: ${behavior}`);
+  if (subissueBehavior && !expectedSubissuePatterns[subissueBehavior])
+    failures.push(`unknown expected_linear_subissues: ${subissueBehavior}`);
+  if (!behavior && !subissueBehavior)
+    failures.push("missing expected ticket behavior");
 
   for (const pattern of patterns) {
     if (!pattern.test(text))