Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/brev-nightly-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ name: E2E / Brev Nightly
# main E2E workflow remains isolated from Brev platform flakiness.
#
# Suites:
# all credential-sanitization + telegram-injection
# telegram-injection Telegram command-injection regression coverage
# messaging-providers Telegram + Discord provider/L7 proxy validation
# messaging-compatible-endpoint local compatible-endpoint Telegram validation
# full install/onboard/inference/CLI path
Expand Down Expand Up @@ -43,7 +43,7 @@ jobs:
strategy:
fail-fast: false
matrix:
test_suite: [all, messaging-providers, messaging-compatible-endpoint, full]
test_suite: [telegram-injection, messaging-providers, messaging-compatible-endpoint, full]
uses: ./.github/workflows/e2e-branch-validation.yaml
with:
# Bind tested code to the ref that supplied this reviewed workflow. Do
Expand Down
12 changes: 2 additions & 10 deletions .github/workflows/e2e-branch-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: E2E / Branch Validation
# WHEN TO USE:
# - Before merging a PR that touches onboard, sandbox, security, or infra
# - To validate a branch works end-to-end on a clean Linux environment
# - To run security regression suites (credential-sanitization, injection)
# - To run the Telegram command-injection regression suite
# - Manually via workflow_dispatch, or called from other workflows
#
# NOTE: This does NOT test a published Brev Launchable image. It bootstraps a
Expand All @@ -29,10 +29,6 @@ name: E2E / Branch Validation
# full — Install → onboard → sandbox verify → live inference
# against NVIDIA Endpoints → CLI operations. Tests the
# complete user journey. (~10 min, destroys sandbox)
# credential-sanitization — 24 tests validating PR #743: credential stripping from
# migration snapshots, auth-profiles.json deletion, blueprint
# digest verification, symlink traversal protection, and
# runtime sandbox credential checks. Requires running sandbox.
# telegram-injection — 18 tests validating PR #584: command injection prevention
# through $(cmd), backticks, quote breakout, ${VAR} expansion,
# process table leak checks, and SANDBOX_NAME validation.
Expand All @@ -47,8 +43,6 @@ name: E2E / Branch Validation
# dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0.
# gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E
# sandbox proof suite from source. (~45 min)
# all — Runs credential-sanitization + telegram-injection, each with
# its own sandbox lifecycle (NOT the independent full journey).
#
# Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY
# Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance)
Expand All @@ -67,13 +61,11 @@ on:
type: choice
options:
- full
- credential-sanitization
- telegram-injection
- messaging-providers
- messaging-compatible-endpoint
- dashboard-remote-bind
- gpu
- all
keep_alive:
description: "Keep Brev instance alive after tests (for SSH debugging)"
required: false
Expand Down Expand Up @@ -179,7 +171,7 @@ jobs:
TEST_SUITE: ${{ inputs.test_suite }}
run: |
case "$TEST_SUITE" in
full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;;
full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu) ;;
*)
echo "::error::test_suite is not one of the supported Brev E2E suites"
exit 1
Expand Down
150 changes: 1 addition & 149 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ jobs:

retired-selector-compatibility:
needs: generate-matrix
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -1982,103 +1982,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

credential-sanitization:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.targets), ',credential-sanitization,') }}
runs-on: ubuntu-latest
timeout-minutes: 65
env:
E2E_JOB: "1"
E2E_TARGET_ID: "credential-sanitization"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-sanitization
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-credential-sanitization"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run credential sanitization live test
# Preserves the
# same ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running
# install.sh, onboarding a real sandbox, and probing sandbox state from
# Vitest while fixture redaction owns evidence logs.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-sanitization.test.ts

- name: Upload credential sanitization artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

credential-migration:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.targets), ',credential-migration,') }}
runs-on: ubuntu-latest
timeout-minutes: 50
env:
E2E_JOB: "1"
E2E_TARGET_ID: "credential-migration"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-migration
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-cred-migration"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run credential migration live test
# This live test
# stages NVIDIA_INFERENCE_API_KEY through legacy credentials.json as the
# custom provider's COMPATIBLE_API_KEY. The hosted service behind this
# repo-scoped secret is inference-api.nvidia.com, not Build/NVIDIA
# Endpoints, so the test must exercise the compatible-provider route.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
NEMOCLAW_PROVIDER: custom
NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1
NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_PREFERRED_API: openai-completions
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-migration.test.ts

- name: Upload credential migration artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

sessions-agents-cli:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sessions-agents-cli,') || contains(format(',{0},', inputs.targets), ',sessions-agents-cli,') }}
Expand Down Expand Up @@ -4724,54 +4627,6 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

diagnostics:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.targets), ',diagnostics,') }}
runs-on: ubuntu-latest
timeout-minutes: 60
env:
E2E_JOB: "1"
E2E_TARGET_ID: "diagnostics"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/diagnostics
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-diag"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75

- name: Run diagnostics live test
# This preserves the
# ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running
# debug archives, install.sh/onboard, sandbox exec/status, and
# gateway-backed credentials list/reset from Vitest.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/diagnostics.test.ts

- name: Upload diagnostics artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

# Scheduled release-baseline coverage for #6108. This runs independently of
# the current lifecycle lane so the exact v0.0.71 build does not extend that
# lane's critical path.
Expand Down Expand Up @@ -5883,8 +5738,6 @@ jobs:
brave-search,
ollama-auth-proxy,
cron-preflight-inference-local,
credential-sanitization,
credential-migration,
sessions-agents-cli,
hermes-e2e,
hermes-gpu-startup,
Expand Down Expand Up @@ -5920,7 +5773,6 @@ jobs:
model-router-provider-routed-inference,
sandbox-operations,
sandbox-survival,
diagnostics,
snapshot-commands,
openclaw-plugin-runtime-exdev-release,
openclaw-plugin-runtime-exdev,
Expand Down
5 changes: 0 additions & 5 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -246,11 +246,6 @@
"test": "rejects credential-backed provider smokes in the PR-safe inference-routing job",
"category": "security"
},
{
"file": "test/e2e/support/e2e-workflow.test.ts",
"test": "rejects diagnostics workflow-boundary drift for secret and Docker auth handling",
"category": "security"
},
{
"file": "test/e2e/support/e2e-workflow.test.ts",
"test": "rejects duplicate unguarded Docker Hub auth in messaging-compatible-endpoint",
Expand Down
6 changes: 6 additions & 0 deletions src/lib/security/credential-filter-secret-patterns.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ describe("isCredentialField", () => {
expect(isCredentialField("clientSecret")).toBe(true);
expect(isCredentialField("bearerToken")).toBe(true);
expect(isCredentialField("privateKey")).toBe(true);
expect(isCredentialField("signingKey")).toBe(true);
expect(isCredentialField("sessionToken")).toBe(true);
expect(isCredentialField("sessionKey")).toBe(true);
expect(isCredentialField("authKey")).toBe(true);
// OpenClaw channel token fields (#5027).
expect(isCredentialField("botToken")).toBe(true);
expect(isCredentialField("appToken")).toBe(true);
Expand Down Expand Up @@ -103,6 +105,10 @@ describe("isCredentialField", () => {
expect(isCredentialField("PATH")).toBe(false);
expect(isCredentialField("tokenizer")).toBe(false);
expect(isCredentialField("maxTokens")).toBe(false);
expect(isCredentialField("displayName")).toBe(false);
expect(isCredentialField("sortKey")).toBe(false);
expect(isCredentialField("sessionId")).toBe(false);
expect(isCredentialField("accessLevel")).toBe(false);
expect(isCredentialField("X-Request-Id")).toBe(false);
expect(isCredentialField("author")).toBe(false);
});
Expand Down
2 changes: 1 addition & 1 deletion test/brev-nightly-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ describe("Brev nightly workflow contract", () => {

expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}");
expect(validation?.run).toContain(
"full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all",
"full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu",
);
expect(validation?.run).toContain("exit 1");
expect(steps.indexOf(validation as NonNullable<typeof validation>)).toBeLessThan(
Expand Down
5 changes: 1 addition & 4 deletions test/brev-remote-vitest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
buildBrevRemoteVitestCommand,
} from "../tools/e2e/brev-remote-vitest.mts";

const TARGET = "test/e2e/live/credential-sanitization.test.ts";
const TARGET = "test/e2e/live/telegram-injection.test.ts";

type Fixture = {
fakeBin: string;
Expand Down Expand Up @@ -109,18 +109,15 @@ describe("Brev remote Vitest command", () => {
});

it("does not seed shared harness state for suites that own their sandbox lifecycle", () => {
expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false);
expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false);
expect(brevSuiteHarnessSandboxName("all")).toBeUndefined();
expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined();
expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined();
});

it("preserves harness onboarding for single-target suites", () => {
expect(brevSuiteNeedsHarnessSandbox("credential-sanitization")).toBe(true);
expect(brevSuiteNeedsHarnessSandbox("telegram-injection")).toBe(true);
expect(brevSuiteNeedsHarnessSandbox("dashboard-remote-bind")).toBe(true);
expect(brevSuiteHarnessSandboxName("dashboard-remote-bind")).toBe("e2e-test");
Expand Down
Loading
Loading