Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/brev-nightly-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ name: E2E / Brev Nightly
# main E2E workflow remains isolated from Brev platform flakiness.
#
# Suites:
# all credential-sanitization + telegram-injection
# telegram-injection Telegram command-injection regression coverage
# messaging-providers Telegram + Discord provider/L7 proxy validation
# messaging-compatible-endpoint local compatible-endpoint Telegram validation
# full install/onboard/inference/CLI path
Expand Down Expand Up @@ -43,7 +43,7 @@ jobs:
strategy:
fail-fast: false
matrix:
test_suite: [all, messaging-providers, messaging-compatible-endpoint, full]
test_suite: [telegram-injection, messaging-providers, messaging-compatible-endpoint, full]
uses: ./.github/workflows/e2e-branch-validation.yaml
with:
# Bind tested code to the ref that supplied this reviewed workflow. Do
Expand Down
12 changes: 2 additions & 10 deletions .github/workflows/e2e-branch-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: E2E / Branch Validation
# WHEN TO USE:
# - Before merging a PR that touches onboard, sandbox, security, or infra
# - To validate a branch works end-to-end on a clean Linux environment
# - To run security regression suites (credential-sanitization, injection)
# - To run the Telegram command-injection regression suite
# - Manually via workflow_dispatch, or called from other workflows
#
# NOTE: This does NOT test a published Brev Launchable image. It bootstraps a
Expand All @@ -29,10 +29,6 @@ name: E2E / Branch Validation
# full — Install → onboard → sandbox verify → live inference
# against NVIDIA Endpoints → CLI operations. Tests the
# complete user journey. (~10 min, destroys sandbox)
# credential-sanitization — 24 tests validating PR #743: credential stripping from
# migration snapshots, auth-profiles.json deletion, blueprint
# digest verification, symlink traversal protection, and
# runtime sandbox credential checks. Requires running sandbox.
# telegram-injection — 18 tests validating PR #584: command injection prevention
# through $(cmd), backticks, quote breakout, ${VAR} expansion,
# process table leak checks, and SANDBOX_NAME validation.
Expand All @@ -47,8 +43,6 @@ name: E2E / Branch Validation
# dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0.
# gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E
# sandbox proof suite from source. (~45 min)
# all — Runs credential-sanitization + telegram-injection, each with
# its own sandbox lifecycle (NOT the independent full journey).
#
# Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY
# Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance)
Expand All @@ -67,13 +61,11 @@ on:
type: choice
options:
- full
- credential-sanitization
- telegram-injection
- messaging-providers
- messaging-compatible-endpoint
- dashboard-remote-bind
- gpu
- all
keep_alive:
description: "Keep Brev instance alive after tests (for SSH debugging)"
required: false
Expand Down Expand Up @@ -179,7 +171,7 @@ jobs:
TEST_SUITE: ${{ inputs.test_suite }}
run: |
case "$TEST_SUITE" in
full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;;
full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu) ;;
*)
echo "::error::test_suite is not one of the supported Brev E2E suites"
exit 1
Expand Down
150 changes: 1 addition & 149 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ jobs:

retired-selector-compatibility:
needs: generate-matrix
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -1982,103 +1982,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

credential-sanitization:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.targets), ',credential-sanitization,') }}
runs-on: ubuntu-latest
timeout-minutes: 65
env:
E2E_JOB: "1"
E2E_TARGET_ID: "credential-sanitization"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-sanitization
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-credential-sanitization"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run credential sanitization live test
# Preserves the
# same ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running
# install.sh, onboarding a real sandbox, and probing sandbox state from
# Vitest while fixture redaction owns evidence logs.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-sanitization.test.ts

- name: Upload credential sanitization artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

credential-migration:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.targets), ',credential-migration,') }}
runs-on: ubuntu-latest
timeout-minutes: 50
env:
E2E_JOB: "1"
E2E_TARGET_ID: "credential-migration"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-migration
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-cred-migration"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run credential migration live test
# This live test
# stages NVIDIA_INFERENCE_API_KEY through legacy credentials.json as the
# custom provider's COMPATIBLE_API_KEY. The hosted service behind this
# repo-scoped secret is inference-api.nvidia.com, not Build/NVIDIA
# Endpoints, so the test must exercise the compatible-provider route.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
NEMOCLAW_PROVIDER: custom
NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1
NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_PREFERRED_API: openai-completions
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-migration.test.ts

- name: Upload credential migration artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

sessions-agents-cli:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sessions-agents-cli,') || contains(format(',{0},', inputs.targets), ',sessions-agents-cli,') }}
Expand Down Expand Up @@ -4724,54 +4627,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

diagnostics:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.targets), ',diagnostics,') }}
runs-on: ubuntu-latest
timeout-minutes: 60
env:
E2E_JOB: "1"
E2E_TARGET_ID: "diagnostics"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/diagnostics
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-diag"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run diagnostics live test
# This preserves the
# ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running
# debug archives, install.sh/onboard, sandbox exec/status, and
# gateway-backed credentials list/reset from Vitest.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/diagnostics.test.ts

- name: Upload diagnostics artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

# Scheduled release-baseline coverage for #6108. This runs independently of
# the current lifecycle lane so the exact v0.0.71 build does not extend that
# lane's critical path.
Expand Down Expand Up @@ -5883,8 +5738,6 @@ jobs:
brave-search,
ollama-auth-proxy,
cron-preflight-inference-local,
credential-sanitization,
credential-migration,
sessions-agents-cli,
hermes-e2e,
hermes-gpu-startup,
Expand Down Expand Up @@ -5920,7 +5773,6 @@ jobs:
model-router-provider-routed-inference,
sandbox-operations,
sandbox-survival,
diagnostics,
snapshot-commands,
openclaw-plugin-runtime-exdev-release,
openclaw-plugin-runtime-exdev,
Expand Down
5 changes: 0 additions & 5 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -246,11 +246,6 @@
"test": "rejects credential-backed provider smokes in the PR-safe inference-routing job",
"category": "security"
},
{
"file": "test/e2e/support/e2e-workflow.test.ts",
"test": "rejects diagnostics workflow-boundary drift for secret and Docker auth handling",
"category": "security"
},
{
"file": "test/e2e/support/e2e-workflow.test.ts",
"test": "rejects duplicate unguarded Docker Hub auth in messaging-compatible-endpoint",
Expand Down
6 changes: 6 additions & 0 deletions src/lib/security/credential-filter-secret-patterns.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ describe("isCredentialField", () => {
expect(isCredentialField("clientSecret")).toBe(true);
expect(isCredentialField("bearerToken")).toBe(true);
expect(isCredentialField("privateKey")).toBe(true);
expect(isCredentialField("signingKey")).toBe(true);
expect(isCredentialField("sessionToken")).toBe(true);
expect(isCredentialField("sessionKey")).toBe(true);
expect(isCredentialField("authKey")).toBe(true);
// OpenClaw channel token fields (#5027).
expect(isCredentialField("botToken")).toBe(true);
expect(isCredentialField("appToken")).toBe(true);
Expand Down Expand Up @@ -103,6 +105,10 @@ describe("isCredentialField", () => {
expect(isCredentialField("PATH")).toBe(false);
expect(isCredentialField("tokenizer")).toBe(false);
expect(isCredentialField("maxTokens")).toBe(false);
expect(isCredentialField("displayName")).toBe(false);
expect(isCredentialField("sortKey")).toBe(false);
expect(isCredentialField("sessionId")).toBe(false);
expect(isCredentialField("accessLevel")).toBe(false);
expect(isCredentialField("X-Request-Id")).toBe(false);
expect(isCredentialField("author")).toBe(false);
});
Expand Down
2 changes: 1 addition & 1 deletion test/brev-nightly-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ describe("Brev nightly workflow contract", () => {

expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}");
expect(validation?.run).toContain(
"full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all",
"full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu",
);
expect(validation?.run).toContain("exit 1");
expect(steps.indexOf(validation as NonNullable<typeof validation>)).toBeLessThan(
Expand Down
5 changes: 1 addition & 4 deletions test/brev-remote-vitest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
buildBrevRemoteVitestCommand,
} from "../tools/e2e/brev-remote-vitest.mts";

const TARGET = "test/e2e/live/credential-sanitization.test.ts";
const TARGET = "test/e2e/live/telegram-injection.test.ts";

type Fixture = {
fakeBin: string;
Expand Down Expand Up @@ -109,18 +109,15 @@ describe("Brev remote Vitest command", () => {
});

it("does not seed shared harness state for suites that own their sandbox lifecycle", () => {
expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false);
expect(brevSuiteHarnessSandboxName("all")).toBeUndefined();
expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined();
expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined();
});

it("preserves harness onboarding for single-target suites", () => {
expect(brevSuiteNeedsHarnessSandbox("credential-sanitization")).toBe(true);
expect(brevSuiteNeedsHarnessSandbox("telegram-injection")).toBe(true);
expect(brevSuiteNeedsHarnessSandbox("dashboard-remote-bind")).toBe(true);
expect(brevSuiteHarnessSandboxName("dashboard-remote-bind")).toBe("e2e-test");
Expand Down
Loading
Loading