NVIDIA-NeMo · nicot · Jul 14, 2026 · Jul 14, 2026 · Jul 14, 2026 · Jul 14, 2026
@@ -0,0 +1,10 @@
+name: insights-intake-stack
+description: Self-contained Intake (ClickHouse + auth,entities,intake) on :8080 over $RUNNER_TEMP/state
+runs:
+  using: composite
+  steps:
+    - name: Start ClickHouse + platform services
+      shell: bash
+      run: >-
+        uv run --no-project python
+        nemo-platform/plugins/nemo-insights/testbed/eval/stack.py
@@ -0,0 +1,23 @@
+name: insights-testbed-prep
+description: uv + dependency sync + tau2 data check + judge model repoint
+inputs:
+  install-tau2:
+    description: Also uv-sync tau2-bench and verify its data (needed to RUN tau2, not to analyze)
+    default: "true"
+  sync-insights:
+    description: uv-sync the optional insights group; disable only for stack-only jobs
+    default: "true"
+runs:
+  using: composite
+  steps:
+    - name: Install uv
+      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      with:
+        working-directory: nemo-platform
+    - name: Sync + judge repoint
+      shell: bash
+      run: >-
+        uv run --no-project python
+        nemo-platform/plugins/nemo-insights/testbed/eval/prep.py
+        --sync-insights=${{ inputs.sync-insights }}
+        --install-tau2=${{ inputs.install-tau2 }}
@@ -0,0 +1,224 @@
+name: insights-testbed
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "What to run"
+        type: choice
+        options: [produce, analyze, stack-check]
+        default: analyze
+      subjects:
+        description: "Comma-separated testbed subjects"
+        type: string
+        default: tau2-airline
+      state:
+        description: "Empty = each subject's state.lock pin"
+        type: string
+        default: ""
+      num_tasks:
+        description: "Override subject num_tasks (empty = testbeds.toml value)"
+        type: string
+        default: ""
+      num_trials:
+        description: "Override subject num_trials (empty = testbeds.toml value)"
+        type: string
+        default: ""
+      publish_state:
+        description: "produce only: upload the new state version to the testbed-state release"
+        type: boolean
+        default: true
+      reason:
+        description: "produce only: why this fixture exists (one line for the release catalog)"
+        type: string
+        default: ""
+  pull_request:
+    types: [opened, synchronize, labeled]
+    paths:
+      - "plugins/nemo-insights/**"
+      - ".github/workflows/insights-testbed.yml"
+      - ".github/actions/insights-intake-stack/**"
+      - ".github/actions/insights-testbed-prep/**"
+
+env:
+  OPENAI_API_BASE: ${{ vars.OPENAI_API_BASE || 'https://inference-api.nvidia.com' }}
+  TAU2_JUDGE_LLM: ${{ vars.TAU2_JUDGE_LLM || 'openai/nvidia/nvidia/evals-nemotron-ultra' }}
+  TAU2_BENCH_REF: 8ebb7499622fc2be9b9d510d6f7a7653461f4f29
+
+jobs:
+  plugin-tests:
+    name: Insights plugin tests
+    runs-on: ubuntu-latest
+    permissions: { contents: read }
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with: { path: nemo-platform }
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          working-directory: nemo-platform
+      - name: Run plugin tests
+        working-directory: nemo-platform
+        run: uv run --frozen --group insights pytest -q plugins/nemo-insights/tests
+
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      mode: ${{ steps.resolve.outputs.mode }}
+      subjects: ${{ steps.resolve.outputs.subjects }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with: { path: nemo-platform }
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          working-directory: nemo-platform
+      - id: resolve
+        env:
+          INPUT_MODE: ${{ inputs.mode }}
+          INPUT_SUBJECTS: ${{ inputs.subjects }}
+        run: uv run --no-project python nemo-platform/plugins/nemo-insights/testbed/eval/plan.py
+
+  stack-check:
+    needs: plan
+    if: needs.plan.outputs.mode == 'stack-check'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with: { path: nemo-platform }
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          repository: sierra-research/tau2-bench
+          ref: ${{ env.TAU2_BENCH_REF }}
+          path: tau2-bench
+      - uses: ./nemo-platform/.github/actions/insights-testbed-prep
+        with: { install-tau2: "false", sync-insights: "false" }
+      - uses: ./nemo-platform/.github/actions/insights-intake-stack
+      - name: Verify stack end-to-end
+        run: uv run --no-project python nemo-platform/plugins/nemo-insights/testbed/eval/stack.py --verify
+      - name: Platform log on failure
+        if: failure()
+        run: tail -100 "$RUNNER_TEMP/platform.log" || true
+
+  produce:
+    needs: plan
+    # produce is EXPLICIT-ONLY: a human dispatches it (e.g. after a tau2-bench or agent
+    # change warrants a new fixture). It must never fire from PRs, pushes, or any
+    # future automatic trigger — hence the hard event gate on top of the mode check.
+    if: needs.plan.outputs.mode == 'produce' && github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    concurrency: { group: testbed-state-produce, cancel-in-progress: false }
+    permissions: { contents: write }   # release asset upload
+    env:
+      GH_TOKEN: ${{ github.token }}
+      INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY }}
+      # Same gateway key; litellm/tau2 read it under the OpenAI-conventional name.
+      OPENAI_API_KEY: ${{ secrets.INFERENCE_API_KEY }}
+    steps:
+      - name: Require secrets
+        run: |
+          [ -n "$INFERENCE_API_KEY" ] || { echo "secret INFERENCE_API_KEY is not set"; exit 1; }
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with: { path: nemo-platform }
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          repository: sierra-research/tau2-bench
+          ref: ${{ env.TAU2_BENCH_REF }}
+          path: tau2-bench
+      - uses: ./nemo-platform/.github/actions/insights-testbed-prep
+      - uses: ./nemo-platform/.github/actions/insights-intake-stack
+      - name: Run subjects (tau2 -> ingest -> insights)
+        working-directory: nemo-platform/plugins/nemo-insights
+        env:
+          SUBJECTS: ${{ needs.plan.outputs.subjects }}
+          NUM_TASKS: ${{ inputs.num_tasks }}
+          NUM_TRIALS: ${{ inputs.num_trials }}
+        run: uv run --project ../.. python testbed/eval/run_subjects.py
+      - name: Snapshot state
+        if: always()
+        working-directory: nemo-platform/plugins/nemo-insights
+        env:
+          SUBJECTS_JSON: ${{ needs.plan.outputs.subjects }}
+          # NUM_TASKS / NUM_TRIALS / REASON feed the manifest's CI-lineage fields.
+          NUM_TASKS: ${{ inputs.num_tasks }}
+          NUM_TRIALS: ${{ inputs.num_trials }}
+          REASON: ${{ inputs.reason }}
+        run: uv run --project ../.. python -m testbed snapshot --subjects-json "$SUBJECTS_JSON" --base http://localhost:8080 -o "$RUNNER_TEMP/bundles/candidate.tar.zst"
+      - name: Round-trip fidelity guard
+        working-directory: nemo-platform/plugins/nemo-insights
+        # Re-ingest the candidate into scratch workspaces on the in-job stack, re-export,
+        # doc-diff; any mismatch fails the job before the candidate can be published.
+        # platform-root auto-resolves to the containing nemo-platform checkout.
+        run: uv run --project ../.. python -m testbed roundtrip "$RUNNER_TEMP/bundles/candidate.tar.zst" --base http://localhost:8080
+      - name: Upload state artifact
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          # run_attempt suffix: upload-artifact@v4 409s on duplicate names within a
+          # run, which would fail this always() step on a re-run and block publish.
+          name: state-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/bundles/candidate.tar.zst
+          if-no-files-found: warn
+      - name: Publish to testbed-state release
+        if: success() && inputs.publish_state
+        working-directory: nemo-platform/plugins/nemo-insights
+        env:
+          REASON: ${{ inputs.reason }}
+        # --no-verify: the round-trip fidelity guard already ran as its own step above.
+        run: uv run --project ../.. python -m testbed publish "$RUNNER_TEMP/bundles/candidate.tar.zst" --reason "$REASON" --no-verify
+      - name: Platform log on failure
+        if: failure()
+        run: tail -200 "$RUNNER_TEMP/platform.log" || true
+
+  analyze:
+    needs: plan
+    # PR gate: on `pull_request` events, only run with the `run-insights` label. Every
+    # other trigger (workflow_dispatch, any future dispatch-like event) is gated on
+    # mode alone, matching stack-check/produce above.
+    if: >-
+      needs.plan.outputs.mode == 'analyze' &&
+      (github.event_name != 'pull_request' ||
+       (github.event.pull_request.head.repo.full_name == github.repository &&
+        contains(github.event.pull_request.labels.*.name, 'run-insights')))
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions: { contents: read }
+    strategy:
+      fail-fast: false
+      matrix:
+        subject: ${{ fromJSON(needs.plan.outputs.subjects) }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+      INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY }}
+    steps:
+      - name: Require secrets
+        run: '[ -n "$INFERENCE_API_KEY" ] || { echo "secret INFERENCE_API_KEY is not set"; exit 1; }'
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with: { path: nemo-platform }
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          repository: sierra-research/tau2-bench
+          ref: ${{ env.TAU2_BENCH_REF }}
+          path: tau2-bench
+      - uses: ./nemo-platform/.github/actions/insights-testbed-prep
+        with: { install-tau2: "false" }
+      - uses: ./nemo-platform/.github/actions/insights-intake-stack
+      - name: Generate insights
+        working-directory: nemo-platform/plugins/nemo-insights
+        env:
+          SUBJECT: ${{ matrix.subject }}
+          STATE: ${{ inputs.state }}
+        # Empty STATE = bare analyze = the subject's state.lock pin (the
+        # reproducible default); a non-empty ref overrides it via --state.
+        run: |
+          uv run --project ../.. python -m testbed analyze "$SUBJECT" ${STATE:+--state "$STATE"} --summary-md "$GITHUB_STEP_SUMMARY"
+          cat "testbed/tmp/insights_${SUBJECT}.yaml"
+      - name: Upload insights
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: insights-${{ matrix.subject }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: nemo-platform/plugins/nemo-insights/testbed/tmp/insights_${{ matrix.subject }}.yaml
+      - name: Platform log on failure
+        if: failure()
+        run: tail -200 "$RUNNER_TEMP/platform.log" || true
@@ -0,0 +1,61 @@
+# NeMo Insights
+
+Optional NeMo Platform plugin for analyzing agent telemetry and persisting actionable insights.
+
+## Install from the monorepo
+
+```bash
+uv sync --group insights
+```
+
+The plugin is intentionally not part of `enabled-plugins`.
+
+## CLI
+
+```bash
+uv run nemo insights analyze \
+  --agent research-agent \
+  --workspace default \
+  --base-url http://localhost:8080
+
+uv run nemo insights analysis enable --agent research-agent
+uv run nemo insights analysis status
+uv run nemo insights analysis disable --agent research-agent
+```
+
+`--base-url` defaults to `NMP_BASE_URL`, then `http://localhost:8080`.
+
+## API and SDK
+
+The service is mounted under:
+
+```text
+/apis/insights/v2/workspaces/{workspace}
+```
+
+The plugin SDK is available as `client.insights`, including:
+
+- `client.insights.insights`
+- `client.insights.analysis_configs`
+- `client.insights.analysis_run_statuses`
+
+## Configuration
+
+Periodic analysis settings use the `NEMO_INSIGHTS_` environment prefix. For example:
+
+```bash
+export NEMO_INSIGHTS_ANALYST_FREQUENCY=daily
+export NEMO_INSIGHTS_ANALYST_TIMEZONE=America/Denver
+```
+
+## Development
+
+```bash
+uv run --group insights pytest plugins/nemo-insights/tests
+uv run ruff check plugins/nemo-insights
+```
+
+## Testbed
+
+The analyst-only testbed is in [`testbed/`](testbed/). It can replay pinned
+Intake traces or run Tau2 benchmarks before invoking `nemo insights analyze`.
@@ -0,0 +1,76 @@
+insights:
+- id: insight-b3adbc1b909843cf83dfd35a4bfa5e3d
+  workspace: default
+  title: Spurious refusal on benign terminal-bench tasks
+  agent: codex
+  description: 'Failure mode: codex aborts a task by emitting the canned refusal "I''m
+    sorry, but I cannot assist with that request." as its final assistant message,
+    scoring 0 on the task verifier. Affected component: the final LLM/assistant
+    turn (model aws/anthropic/bedrock-claude-opus-4-6 under the codex harness) — the
+    refusal appears AFTER the agent has already begun work (2–5 shell_command/update_plan
+    calls), not at the outset. Trigger conditions: entirely benign coding/ops prompts
+    — constraint scheduling, generating a [regex, replacement] JSON file, implementing
+    a headless terminal interface. The same task ids pass in other trials, so the
+    refusal is non-deterministic. Hypothesis: an over-aggressive safety/instruction-following
+    layer mis-classifies these benign tasks as disallowed and short-circuits an in-progress
+    trajectory.'
+  status: open
+  trace_refs:
+  - constraints-scheduling-xhsKy3g
+  - regex-chess-HfzhtaZ
+  - headless-terminal-DzYqtUy
+  created_at: '2026-06-05T22:28:00.691135+00:00'
+  updated_at: '2026-06-05T22:28:00.691135+00:00'
+- id: insight-c0bbadce52c34656a0e61b215142f291
+  workspace: default
+  title: Declares success against self-authored tests that don't meet the real spec
+  agent: codex
+  description: 'Failure mode: codex finishes with a confident completion claim — "The
+    implementation is complete and all tests pass", "I have successfully implemented...",
+    "passes the required tests" — yet the task verifier scores 0. Affected component:
+    the agent''s self-verification step. In the trajectories it writes and runs its
+    OWN tests (e.g. `Rscript -e ''source("ars.R"); test()''`) and treats their passing
+    as task completion, instead of validating against the task''s stated acceptance
+    criteria. Trigger conditions: open-ended implementation tasks where the agent
+    can author its own tests (adaptive-rejection-sampler fails 5/6 trials this way;
+    also SAM cell-segmentation, regex-chess, Raman fitting, mailman setup). Hypothesis:
+    the agent anchors on self-invented success criteria and never reconciles them
+    with the real spec, producing overconfident but incorrect completions.'
+  status: open
+  trace_refs:
+  - adaptive-rejection-sampler-sCQXPuA
+  - adaptive-rejection-sampler-qYLBFwo
+  - adaptive-rejection-sampler-Fgeag3m
+  - sam-cell-seg-XVe4VSi
+  - regex-chess-mbr83he
+  - raman-fitting-4TkuEBg
+  - mailman-Aw7evCn
+  created_at: '2026-06-05T22:28:00.691135+00:00'
+  updated_at: '2026-06-05T22:28:00.691135+00:00'
+- id: insight-78d54712121d416192b9aa04bc78d375
+  workspace: default
+  title: Trajectory truncated mid-debugging on long environment-heavy tasks
+  agent: codex
+  description: 'Failure mode: on long-horizon, environment-heavy tasks the codex trajectory
+    terminates with its last recorded action being a shell_command and no closing
+    completion summary; the task verifier scores 0. The final tool output frequently
+    still shows a live failure (segfault exit 139, gcc/compile errors, exit 1 tracebacks,
+    or ''No SSH banner after 60s''), i.e. the solution was never finalized. Affected
+    component: the agent loop / step-budget — this end-on-tool-call-without-summary
+    pattern occurs in ~56% of failures vs ~11% of passes, and the affected runs cluster
+    at high tool-call counts (50+: qemu, path-tracing, make-mips, protein-assembly).
+    Trigger conditions: tasks involving builds, emulation, ML training/setup, or long
+    iterative debugging (qemu+ssh boot, custom compressor, MIPS interpreter, Caffe
+    CIFAR-10 build, path-tracing reverse-engineering). Hypothesis: the agent exhausts
+    its turn/step budget (or is cut off) before converging, leaving a broken solution
+    and never delivering or verifying a result.'
+  status: open
+  trace_refs:
+  - qemu-alpine-ssh-Z9EtcT7
+  - write-compressor-U9gsqTF
+  - path-tracing-reverse-XKeZn2P
+  - make-mips-interpreter-vgGbWQh
+  - caffe-cifar-10-mZcR43P
+  - polyglot-c-py-jcceToH
+  created_at: '2026-06-05T22:28:00.691135+00:00'
+  updated_at: '2026-06-05T22:28:00.691135+00:00'