[Infra] feat: update build-validator job and README for Node.js version requirement

.github/workflows/evaluation.yml

@@ -160,9 +160,40 @@ jobs:
           }
         shell: pwsh
 
-  evaluate:
+  build-validator:
+    runs-on: ubuntu-latest
     needs: discover
     if: needs.discover.outputs.has_entries == 'true'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+          cache-dependency-path: eng/skill-validator/package-lock.json
+
+      - name: Build skill-validator
+        run: cd eng/skill-validator && npm ci && npm run build
+
+      - name: Prune dev dependencies
+        run: cd eng/skill-validator && npm prune --omit=dev
+
+      - name: Upload built validator
+        uses: actions/upload-artifact@v4
+        with:
+          name: skill-validator-dist
+          path: |
+            eng/skill-validator/dist/
+            eng/skill-validator/node_modules/
+            eng/skill-validator/package.json
+          retention-days: 1
+
+  evaluate:
+    needs: [discover, build-validator]
+    if: needs.discover.outputs.has_entries == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 30
     name: evaluate (${{ matrix.entry.name }})
@@ -185,103 +216,97 @@ jobs:
         with:
           dotnet-version: ${{ env.DOTNET_VERSION }}
 
-      - name: Build skill-validator
-        run: cd eng/skill-validator && npm ci && npm run build
+      - name: Download built validator
+        uses: actions/download-artifact@v4
+        with:
+          name: skill-validator-dist
+          path: artifacts/skill-validator/
 
       - name: Run skill-validator
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
         continue-on-error: true
         env:
           GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN_2 }}
+          RESULTS_PATH: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}
         run: |
           ARGS="--strict --require-evals"
-          ARGS="$ARGS --reporter console --reporter json:.skill-validator-results/results.json"
+          ARGS="$ARGS --results-dir $RESULTS_PATH --reporter console --reporter json --reporter markdown"
+          ARGS="$ARGS --model ${{ github.event.inputs.model || env.DEFAULT_MODEL }}"
+          ARGS="$ARGS --judge-model ${{ github.event.inputs.judge-model || env.DEFAULT_JUDGE_MODEL }}"
+          ARGS="$ARGS --runs ${{ github.event.inputs.runs || env.DEFAULT_RUNS }}"
+          ARGS="$ARGS --parallel-skills ${{ github.event.inputs.parallel-skills || env.DEFAULT_PARALLEL_SKILLS }}"
+          ARGS="$ARGS --parallel-scenarios ${{ github.event.inputs.parallel-scenarios || env.DEFAULT_PARALLEL_SCENARIOS }}"
+          ARGS="$ARGS --parallel-runs ${{ github.event.inputs.parallel-runs || env.DEFAULT_PARALLEL_RUNS }}"
 
-          MODEL="${{ github.event.inputs.model || env.DEFAULT_MODEL }}"
-          if [ -n "$MODEL" ]; then
-            ARGS="$ARGS --model $MODEL"
-          fi
-          RUNS="${{ github.event.inputs.runs || env.DEFAULT_RUNS }}"
-          if [ -n "$RUNS" ]; then
-            ARGS="$ARGS --runs $RUNS"
-          fi
-          PARALLEL_SKILLS="${{ github.event.inputs.parallel-skills || env.DEFAULT_PARALLEL_SKILLS }}"
-          if [ -n "$PARALLEL_SKILLS" ]; then
-            ARGS="$ARGS --parallel-skills $PARALLEL_SKILLS"
-          fi
-          PARALLEL_SCENARIOS="${{ github.event.inputs.parallel-scenarios || env.DEFAULT_PARALLEL_SCENARIOS }}"
-          if [ -n "$PARALLEL_SCENARIOS" ]; then
-            ARGS="$ARGS --parallel-scenarios $PARALLEL_SCENARIOS"
-          fi
-          PARALLEL_RUNS="${{ github.event.inputs.parallel-runs || env.DEFAULT_PARALLEL_RUNS }}"
-          if [ -n "$PARALLEL_RUNS" ]; then
-            ARGS="$ARGS --parallel-runs $PARALLEL_RUNS"
-          fi
           if [ "${{ github.event.inputs.verbose }}" = "true" ]; then
             ARGS="$ARGS --verbose"
           fi
-          JUDGE_MODEL="${{ github.event.inputs.judge-model || env.DEFAULT_JUDGE_MODEL }}"
-          if [ -n "$JUDGE_MODEL" ]; then
-            ARGS="$ARGS --judge-model $JUDGE_MODEL"
-          fi
 
-          node eng/skill-validator/dist/index.js $ARGS --tests-dir ./src/${{ matrix.entry.component }}/tests ./${{ matrix.entry.skills_path }}
+          node artifacts/skill-validator/dist/index.js $ARGS --tests-dir ./src/${{ matrix.entry.component }}/tests ./${{ matrix.entry.skills_path }}
 
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: skill-validator-results-${{ matrix.entry.name }}
-          path: .skill-validator-results/
+          path: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}/
           include-hidden-files: true
           retention-days: 30
 
-      - name: Generate Summary
-        if: always()
+  comment-on-pr:
+    needs: [discover, evaluate]
+    if: always() && needs.discover.outputs.has_entries == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all result artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: skill-validator-results-*
+          path: all-results/
+          merge-multiple: false
+
+      - name: Consolidate summaries
         run: |
-          RESULTS_DIR=$(ls -d .skill-validator-results/run-* 2>/dev/null | head -1)
-          if [ -z "$RESULTS_DIR" ]; then
-            echo "## Skill Validation Results — ${{ matrix.entry.name }}" >> $GITHUB_STEP_SUMMARY
-            echo "No results found." >> $GITHUB_STEP_SUMMARY
-            exit 0
+          COMMENT_MARKER="<!-- skill-validator-results -->"
+          BODY="$COMMENT_MARKER"$'\n'
+          FOUND=false
+
+          for COMPONENT_DIR in all-results/skill-validator-results-*/; do
+            SUMMARY="$COMPONENT_DIR/summary.md"
+            if [ -f "$SUMMARY" ]; then
+              BODY+=$(cat "$SUMMARY")$'\n\n'
+              FOUND=true
+            fi
+          done
+
+          if [ "$FOUND" = "false" ]; then
+            BODY+="## Skill Validation Results"$'\n'"No results were produced."$'\n'
           fi
 
-          RESULTS_FILE="$RESULTS_DIR/results.json"
-          if [ ! -f "$RESULTS_FILE" ]; then
-            echo "## Skill Validation Results — ${{ matrix.entry.name }}" >> $GITHUB_STEP_SUMMARY
-            echo "No results.json found." >> $GITHUB_STEP_SUMMARY
-            exit 0
-          fi
+          BODY+=$'\n'"[Full results](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"$'\n'
 
-          # Generate markdown summary from results
-          node -e "
-          const fs = require('fs');
-          const results = JSON.parse(fs.readFileSync('$RESULTS_FILE', 'utf8'));
-          let md = '## Skill Validation Results — ${{ matrix.entry.name }}\n\n';
-          md += '| Skill | Test | Baseline | With Skill | Δ | Verdict |\n';
-          md += '|-------|----------|----------|------------|---|--------|\n';
-          for (const v of results.verdicts) {
-            for (const s of v.scenarios) {
-              const base = s.baseline?.judgeResult?.overallScore?.toFixed(1) ?? '—';
-              const skill = s.withSkill?.judgeResult?.overallScore?.toFixed(1) ?? '—';
-              const delta = (s.withSkill?.judgeResult?.overallScore - s.baseline?.judgeResult?.overallScore)?.toFixed(1);
-              const deltaStr = delta > 0 ? '+' + delta : delta;
-              const icon = v.passed ? '✅' : '❌';
-              md += '| ' + v.skillName + ' | ' + s.scenarioName + ' | ' + base + '/5 | ' + skill + '/5 | ' + deltaStr + ' | ' + icon + ' |\n';
-            }
-          }
-          md += '\nModel: ' + results.model + ' | Judge: ' + results.judgeModel + '\n';
-          md += '\n[Full results](' + '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + ')\n';
-          fs.writeFileSync('.skill-validator-results/summary.md', md);
-          process.stdout.write(md);
-          " >> $GITHUB_STEP_SUMMARY
-
-      - name: Comment on PR
-        if: always() && github.event_name == 'pull_request'
+          echo "$BODY" > consolidated-comment.md
+          cat consolidated-comment.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Post or update PR comment
+        if: github.event_name == 'pull_request'
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh pr comment ${{ github.event.pull_request.number }} --body-file .skill-validator-results/summary.md
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          MARKER="<!-- skill-validator-results -->"
+
+          # Find existing comment with our marker
+          COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
+            --paginate --jq ".[] | select(.body | startswith(\"$MARKER\")) | .id" | head -1)
+
+          if [ -n "$COMMENT_ID" ]; then
+            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
+              -X PATCH -F "body=@consolidated-comment.md"
+          else
+            gh pr comment "$PR_NUMBER" --body-file consolidated-comment.md
+          fi
 
   publish-benchmark:
     needs: [discover, evaluate]

CONTRIBUTING.md

@@ -213,21 +213,19 @@ Prerequisites: Node.js >= 20 and `gh auth login`.
 
 ```bash
 # Build the validator
-cd eng/skill-validator
-npm ci
-npm run build
+cd eng/skill-validator && npm ci && npm run build && cd ../..
 
 # Run tests for a single component
-node dist/index.js --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills
+node eng/skill-validator/dist/index.js --tests-dir src/dotnet-msbuild/tests src/dotnet-msbuild/skills
 
 # Run tests for a single skill (pass the skill directory directly)
-node dist/index.js --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills/common-build-errors
+node eng/skill-validator/dist/index.js --tests-dir src/dotnet-msbuild/tests src/dotnet-msbuild/skills/common-build-errors
 
 # Fewer runs for faster iteration (default is 5)
-node dist/index.js --runs 3 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills
+node eng/skill-validator/dist/index.js --runs 1 --tests-dir src/dotnet-msbuild/tests src/dotnet-msbuild/skills
 
 # Use a specific model
-node dist/index.js --model claude-opus-4.6 --tests-dir ./src/dotnet-msbuild/tests ./src/dotnet-msbuild/skills
+node eng/skill-validator/dist/index.js --model claude-sonnet-4.5 --tests-dir src/dotnet-msbuild/tests src/dotnet-msbuild/skills
 ```
 
 > [!WARNING]  

eng/skill-validator/README.md

@@ -19,7 +19,7 @@ Plugging into your CI, it ensures every new skill adds real value, and existing
 
 ## Prerequisites
 
-- Node.js >= 20
+- Node.js >= 22
 - Authenticated with GitHub via `gh auth login` (the SDK picks up your credentials automatically)
 
 ## Install
@@ -51,15 +51,14 @@ skill-validator --model gpt-5.3-codex --judge-model claude-opus-4.6-fast ./skill
 # Multiple runs for stability
 skill-validator --runs 5 ./skills/
 
-# Output as JSON or JUnit XML
-skill-validator --reporter json:results.json ./skills/
-skill-validator --reporter junit:results.xml ./skills/
+# Override the default results directory (.skill-validator-results)
+skill-validator --results-dir ./my-results ./skills/
+
+# File reporters can also be specified explicitly.
+skill-validator --reporter junit ./skills/
 
 # Strict mode (require all skills to have evals)
 skill-validator --strict ./skills/
-
-# Custom results directory
-skill-validator --results-dir ./my-results ./skills/
 ```
 
 ## Writing eval files
@@ -230,18 +229,18 @@ The default of 5 runs provides sufficient precision for significance testing (va
 | `--require-evals` | `false` | Fail if skill has no tests/eval.yaml |
 | `--strict` | `false` | Enable --require-evals and strict checking |
 | `--verbose` | `false` | Show tool calls and agent events during runs |
-| `--reporter <spec>` | `console` | Output format: `console`, `json:path`, `junit:path` |
-| `--results-dir <path>` | `.skill-validator-results` | Directory for saved run results |
-| `--no-save-results` | | Disable saving run results to disk |
+| `--reporter <spec>` | `console`, `json`, `markdown` | Output format: `console`, `json`, `junit`, `markdown`. |
+| `--results-dir <path>` | `.skill-validator-results` | Directory for file reporter output. |
 
 Models are validated on startup — invalid model names fail fast with a list of available models.
 
 ## Output
 
-Results are displayed in the console with color-coded scores and metric deltas. Run results are also auto-saved to `.skill-validator-results/run-{timestamp}/` containing:
+Results are displayed in the console with color-coded scores and metric deltas. By default, `json` and `markdown` reporters are enabled and write to `.skill-validator-results/` (override with `--results-dir`). File reporters write to that directory:
 
-- `results.json` — full results with model, timestamp, and all verdicts
-- Per-skill directories with `verdict.json` and per-scenario markdown files
+- `json` — `results.json` with model, timestamp, and all verdicts
+- `junit` — `results.xml` with JUnit XML test results
+- `markdown` — `summary.md` with a results table, plus per-skill directories with per-scenario judge reports
 
 ## CI integration
 

eng/skill-validator/src/cli.ts

@@ -2,12 +2,12 @@ import { Command } from "commander";
 import chalk from "chalk";
 import pLimit from "p-limit";
 import { discoverSkills } from "./discovery.js";
-import { runAgent, stopSharedClient, getSharedClient } from "./runner.js";
+import { runAgent, stopSharedClient, getSharedClient, cleanupWorkDirs } from "./runner.js";
 import { evaluateAssertions, evaluateConstraints } from "./assertions.js";
 import { judgeRun } from "./judge.js";
 import { pairwiseJudge } from "./pairwise-judge.js";
 import { compareScenario, computeVerdict } from "./comparator.js";
-import { reportResults, saveRunResults } from "./reporter.js";
+import { reportResults } from "./reporter.js";
 import { analyzeSkill, formatProfileLine, formatProfileWarnings } from "./skill-profile.js";
 import type {
   ValidatorConfig,
@@ -86,11 +86,11 @@ class Spinner {
 }
 
 function parseReporter(value: string): ReporterSpec {
-  const [type, outputPath] = value.split(":");
-  if (type !== "console" && type !== "json" && type !== "junit") {
+  const type = value;
+  if (type !== "console" && type !== "json" && type !== "junit" && type !== "markdown") {
     throw new Error(`Unknown reporter type: ${type}`);
   }
-  return { type, outputPath };
+  return { type };
 }
 
 export function createProgram(): Command {
@@ -123,7 +123,7 @@ export function createProgram(): Command {
     .option("--confidence-level <number>", "Confidence level for statistical intervals (0-1)", "0.95")
     .option(
       "--results-dir <path>",
-      "Directory to save run results",
+      "Directory to save results to (default: .skill-validator-results). Used by file-based reporters (json, junit, markdown).",
       ".skill-validator-results"
     )
     .option(
@@ -132,11 +132,24 @@ export function createProgram(): Command {
     )
     .option(
       "--reporter <spec>",
-      "Reporter (console, json:path, junit:path). Can be repeated.",
+      "Reporter (console, json, junit, markdown). Can be repeated.",
       (val: string, prev: ReporterSpec[]) => [...prev, parseReporter(val)],
       [] as ReporterSpec[]
     )
     .action(async (paths: string[], opts) => {
+      const reporters: ReporterSpec[] =
+        opts.reporter.length > 0
+          ? opts.reporter
+          : [{ type: "console" as const }, { type: "json" as const }, { type: "markdown" as const }];
+
+      const fileReporters = reporters.filter((r) => r.type !== "console");
+      if (fileReporters.length > 0 && !opts.resultsDir) {
+        const names = fileReporters.map((r) => r.type).join(", ");
+        throw new Error(
+          `--results-dir is required when using file-based reporters: ${names}`
+        );
+      }
+
       const config: ValidatorConfig = {
         minImprovement: parseFloat(opts.minImprovement),
         requireCompletion: opts.requireCompletion,
@@ -152,12 +165,8 @@ export function createProgram(): Command {
         parallelRuns: Math.max(1, parseInt(opts.parallelRuns, 10) || 1),
         judgeTimeout: parseInt(opts.judgeTimeout, 10) * 1000,
         confidenceLevel: parseFloat(opts.confidenceLevel || "0.95"),
-        reporters:
-          opts.reporter.length > 0
-            ? opts.reporter
-            : [{ type: "console" as const }],
+        reporters,
         skillPaths: paths,
-        saveResults: opts.saveResults !== false,
         resultsDir: opts.resultsDir,
         testsDir: opts.testsDir,
       };
@@ -471,14 +480,14 @@ export async function run(config: ValidatorConfig): Promise<number> {
     }
   }
 
-  await reportResults(verdicts, config.reporters, config.verbose);
-
-  if (config.saveResults) {
-    const runDir = await saveRunResults(verdicts, config.resultsDir, config.model, config.judgeModel);
-    console.log(chalk.dim(`Run results saved to ${runDir}`));
-  }
+  await reportResults(verdicts, config.reporters, config.verbose, {
+    model: config.model,
+    judgeModel: config.judgeModel,
+    resultsDir: config.resultsDir,
+  });
 
   await stopSharedClient();
+  await cleanupWorkDirs();
 
   const allPassed = verdicts.every((v) => v.passed);
   return allPassed ? 0 : 1;