feat: add benchmark CI scaffold

.github/workflows/benchmark-ci.yml

@@ -0,0 +1,150 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Benchmark CI
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Commit SHA, branch, or tag to benchmark"
+        required: true
+        default: "main"
+      config:
+        description: "Benchmark config path"
+        required: true
+        default: "benchmarks/configs/smoke.json"
+      num_records:
+        description: "Rows for each configured preview experiment"
+        required: true
+        default: "1"
+      experiments:
+        description: "Experiments to run"
+        required: true
+        type: choice
+        options:
+          - "all"
+          - "rewrite"
+          - "redact"
+        default: "all"
+      sync_async_records:
+        description: "Rows for the mock sync/async benchmark"
+        required: true
+        default: "20"
+      sync_async_iterations:
+        description: "Iterations for the mock sync/async benchmark"
+        required: true
+        default: "1"
+      run_sync_async:
+        description: "Run the optional mock sync/async benchmark"
+        required: true
+        type: choice
+        options:
+          - "false"
+          - "true"
+        default: "false"
+      dd_trace:
+        description: "Capture DataDesigner LLM traces"
+        required: true
+        type: choice
+        options:
+          - "none"
+          - "last_message"
+          - "all_messages"
+        default: "none"
+
+permissions:
+  contents: read
+
+env:
+  NEMO_TELEMETRY_ENABLED: "false"
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: [self-hosted, anonymizer-evals]
+    timeout-minutes: 60
+    env:
+      BENCHMARK_REF: ${{ inputs.ref }}
+      BENCHMARK_CONFIG: ${{ inputs.config }}
+      BENCHMARK_NUM_RECORDS: ${{ inputs.num_records }}
+      BENCHMARK_EXPERIMENTS: ${{ inputs.experiments }}
+      BENCHMARK_SYNC_ASYNC_RECORDS: ${{ inputs.sync_async_records }}
+      BENCHMARK_SYNC_ASYNC_ITERATIONS: ${{ inputs.sync_async_iterations }}
+      BENCHMARK_RUN_SYNC_ASYNC: ${{ inputs.run_sync_async }}
+      BENCHMARK_DD_TRACE: ${{ inputs.dd_trace }}
+
+    steps:
+      - name: Checkout benchmark harness
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+
+      - name: Checkout benchmark target
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.BENCHMARK_REF }}
+          path: .benchmark-sut
+          fetch-depth: "0"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install benchmark target
+        run: |
+          uv venv .venv
+          uv pip install --python .venv/bin/python ./.benchmark-sut
+
+      - name: Resolve benchmark target commit
+        id: target
+        working-directory: .benchmark-sut
+        run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Run benchmark scaffold
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+        run: |
+          SYNC_ASYNC_ARGS=()
+          if [ "$BENCHMARK_RUN_SYNC_ASYNC" = "true" ]; then
+            SYNC_ASYNC_ARGS+=(--run-sync-async)
+          fi
+
+          TRACE_ARGS=(--dd-trace "$BENCHMARK_DD_TRACE")
+          if [ "$BENCHMARK_DD_TRACE" != "none" ]; then
+            TRACE_ARGS+=(--trace-dir benchmark-results/traces)
+          fi
+
+          .venv/bin/python scripts/benchmark_ci.py \
+            --require-api-key \
+            --config "$BENCHMARK_CONFIG" \
+            --sut-ref "$BENCHMARK_REF" \
+            --sut-commit "${{ steps.target.outputs.commit }}" \
+            --num-records "$BENCHMARK_NUM_RECORDS" \
+            --experiments "$BENCHMARK_EXPERIMENTS" \
+            --sync-async-records "$BENCHMARK_SYNC_ASYNC_RECORDS" \
+            --sync-async-iterations "$BENCHMARK_SYNC_ASYNC_ITERATIONS" \
+            --out benchmark-results/results.json \
+            --summary-out benchmark-results/summary.md \
+            "${TRACE_ARGS[@]}" \
+            "${SYNC_ASYNC_ARGS[@]}"
+
+      - name: Add summary
+        if: always()
+        run: |
+          if [ -f benchmark-results/summary.md ]; then
+            cat benchmark-results/summary.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: benchmark-results/

.gitignore

@@ -108,6 +108,7 @@ ai/tmp/
 
 # Anonymizer execution artifacts
 .anonymizer-artifacts/
+benchmark-results/
 docs/notebook_source/data/synth_bios_sample10_anonymized.csv
 
 # TLS certs and keys (if any)

benchmarks/README.md

@@ -0,0 +1,28 @@
+# Benchmark configs
+
+`scripts/benchmark_ci.py` reads benchmark experiments from JSON config files. The smoke config is intentionally small and runs one public-safe dataset through the preview pipelines.
+
+Add a benchmark by adding a dataset and experiment to a config:
+
+```json
+{
+  "datasets": {
+    "my_dataset": {
+      "path": "docs/data/example.csv",
+      "sha256": "<sha256>",
+      "text_column": "text",
+      "data_summary": "Short dataset description"
+    }
+  },
+  "experiments": [
+    {
+      "name": "redact_my_dataset",
+      "pipeline": "redact",
+      "dataset": "my_dataset",
+      "num_records": 10
+    }
+  ]
+}
+```
+
+Datasets can use either `path` or `url`. URL datasets must include `sha256`; the runner downloads them into `.benchmark-data-cache` and verifies the hash before running.

benchmarks/configs/smoke.json

@@ -0,0 +1,26 @@
+{
+  "name": "smoke",
+  "description": "Small real-provider benchmark smoke test.",
+  "datasets": {
+    "synthetic_bios": {
+      "path": "docs/data/NVIDIA_synthetic_biographies.csv",
+      "sha256": "407efcb6fdc840562b75875862a4b1231b1e44c832f9ed2913f2243314b002ee",
+      "text_column": "biography",
+      "data_summary": "Biographical profiles"
+    }
+  },
+  "experiments": [
+    {
+      "name": "rewrite_preview",
+      "pipeline": "rewrite",
+      "dataset": "synthetic_bios",
+      "num_records": 1
+    },
+    {
+      "name": "redact_preview",
+      "pipeline": "redact",
+      "dataset": "synthetic_bios",
+      "num_records": 1
+    }
+  ]
+}