diff --git a/scripts/debug.sh b/scripts/debug.sh index ab7b3677c8d..74ae95faf7f 100755 --- a/scripts/debug.sh +++ b/scripts/debug.sh @@ -2,376 +2,33 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Collect NemoClaw diagnostic information for bug reports. -# -# Outputs to stdout and optionally writes a tarball. +# Compatibility wrapper for the TypeScript NemoClaw debug collector. # # Usage: -# ./scripts/debug.sh # full diagnostics to stdout -# ./scripts/debug.sh --quick # minimal diagnostics -# ./scripts/debug.sh --sandbox mybox # target a specific sandbox -# ./scripts/debug.sh --output /tmp/diag.tar.gz # also save tarball -# nemoclaw debug [--quick] [--output path] # via CLI wrapper -# -# Can also be run without cloning: -# curl -fsSL https://raw.githubusercontent.com/NVIDIA/NemoClaw/main/scripts/debug.sh | bash -s -- --quick +# ./scripts/debug.sh [--quick] [--sandbox NAME] [--output PATH] +# nemoclaw debug [--quick] [--sandbox NAME] [--output PATH] set -euo pipefail -# ── Setup ──────────────────────────────────────────────────────── - -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -CYAN='\033[0;36m' -NC='\033[0m' - -info() { echo -e "${GREEN}[debug]${NC} $1"; } -warn() { echo -e "${YELLOW}[debug]${NC} $1"; } -fail() { - echo -e "${RED}[debug]${NC} $1" - exit 1 -} -section() { echo -e "\n${CYAN}═══ $1 ═══${NC}\n"; } - -# ── Parse flags ────────────────────────────────────────────────── - -SANDBOX_NAME="${NEMOCLAW_SANDBOX:-${SANDBOX_NAME:-}}" -QUICK=false -OUTPUT="" - -while [ $# -gt 0 ]; do - case "$1" in - --sandbox) - SANDBOX_NAME="${2:?--sandbox requires a name}" - shift 2 - ;; - --quick) - QUICK=true - shift - ;; - --output | -o) - OUTPUT="${2:?--output requires a path}" - shift 2 - ;; - --help | -h) - cat <<'USAGE' -Usage: scripts/debug.sh [OPTIONS] - -Collect NemoClaw diagnostic information for bug reports. - -Options: - --sandbox NAME Target sandbox (default: $NEMOCLAW_SANDBOX or auto-detect) - --quick Collect minimal diagnostics only - --output PATH Write tarball to PATH (e.g. /tmp/nemoclaw-debug.tar.gz) - --help Show this help - -Examples: - nemoclaw debug - nemoclaw debug --quick - nemoclaw debug --output /tmp/diag.tar.gz - curl -fsSL https://raw.githubusercontent.com/NVIDIA/NemoClaw/main/scripts/debug.sh | bash -s -- --quick -USAGE - exit 0 - ;; - *) - fail "Unknown option: $1 (see --help)" - ;; - esac -done - -# ── Helpers ────────────────────────────────────────────────────── - -TMPDIR_BASE="${TMPDIR:-/tmp}" -COLLECT_DIR=$(mktemp -d "${TMPDIR_BASE}/nemoclaw-debug-XXXXXX") -SANDBOX_SSH_CONFIG="" -SANDBOX_SSH_KNOWN="" -cleanup() { - rm -rf "$COLLECT_DIR" - if [ -n "$SANDBOX_SSH_CONFIG" ]; then - rm -f "$SANDBOX_SSH_CONFIG" - fi - if [ -n "$SANDBOX_SSH_KNOWN" ]; then - rm -f "$SANDBOX_SSH_KNOWN" - fi -} -trap cleanup EXIT - -# Platform detection -IS_MACOS=false -if [ "$(uname -s)" = "Darwin" ]; then - IS_MACOS=true -fi - -# Detect timeout binary (GNU coreutils; gtimeout on macOS via brew) -TIMEOUT_BIN="" -if command -v timeout >/dev/null 2>&1; then - TIMEOUT_BIN="timeout" -elif command -v gtimeout >/dev/null 2>&1; then - TIMEOUT_BIN="gtimeout" -fi - -SCRIPT_DIR="" -REPO_ROOT="" -ONBOARD_SESSION_HELPER="" -SCRIPT_PATH="${BASH_SOURCE[0]:-}" -if [ -n "$SCRIPT_PATH" ] && [ -f "$SCRIPT_PATH" ]; then - SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_PATH")" && pwd)" - REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - ONBOARD_SESSION_HELPER="${REPO_ROOT}/dist/lib/onboard-session.js" -fi - -# Redact known sensitive patterns from stdin. -# Primary path: delegate to the compiled TypeScript redact module. -# Fallback: minimal sed for environments without node or dist/. -# Ref: https://github.com/NVIDIA/NemoClaw/issues/2381 -redact() { - if command -v node &>/dev/null && [ -n "$REPO_ROOT" ] && [ -f "${REPO_ROOT}/dist/lib/redact.js" ]; then - node -e " - const {redactFull} = require(process.argv[1]); - let d = ''; - process.stdin.on('data', c => d += c); - process.stdin.on('end', () => process.stdout.write(redactFull(d))); - " "${REPO_ROOT}/dist/lib/redact" - else - sed -E \ - -e 's/(NVIDIA_API_KEY|API_KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|_KEY)=\S+/\1=/gi' \ - -e 's/nvapi-[A-Za-z0-9_-]{10,}//g' \ - -e 's/nvcf-[A-Za-z0-9_-]{10,}//g' \ - -e 's/ghp_[A-Za-z0-9_-]{10,}//g' \ - -e 's/sk-[A-Za-z0-9_-]{20,}//g' \ - -e 's/(Bearer )[^ ]+/\1/gi' - fi -} - -# Run a command, print output, and save to a file in the collect dir. -# Silently skips commands that are not found. Output is redacted for secrets. -collect() { - local label="$1" - shift - local filename - filename=$(echo "$label" | tr ' /' '_-') - local outfile="${COLLECT_DIR}/${filename}.txt" - - if ! command -v "$1" &>/dev/null; then - echo " ($1 not found, skipping)" | tee "$outfile" - return 0 - fi - - local rc=0 - local tmpout="${outfile}.raw" - if [ -n "$TIMEOUT_BIN" ]; then - "$TIMEOUT_BIN" 30 "$@" >"$tmpout" 2>&1 || rc=$? - else - "$@" >"$tmpout" 2>&1 || rc=$? - fi - - redact <"$tmpout" >"$outfile" - rm -f "$tmpout" - - cat "$outfile" - if [ "$rc" -ne 0 ]; then - echo " (command exited with non-zero status)" - fi -} - -# ── Auto-detect sandbox name if not given ──────────────────────── - -if [ -z "$SANDBOX_NAME" ]; then - if command -v openshell &>/dev/null; then - SANDBOX_NAME=$( - openshell sandbox list 2>/dev/null \ - | awk 'NF { if (tolower($1) == "name") next; print $1; exit }' - ) || true - fi - SANDBOX_NAME="${SANDBOX_NAME:-default}" -fi - -# ── Collect diagnostics ────────────────────────────────────────── - -info "Collecting diagnostics for sandbox '${SANDBOX_NAME}'..." -info "Quick mode: ${QUICK}" -[ -n "$OUTPUT" ] && info "Tarball output: ${OUTPUT}" -echo "" - -# -- System basics -- - -section "System" -collect "date" date -collect "uname" uname -a -collect "uptime" uptime -if [ "$IS_MACOS" = true ]; then - # shellcheck disable=SC2016 - collect "memory" sh -c 'echo "Physical: $(($(sysctl -n hw.memsize) / 1048576)) MB"; vm_stat' -else - collect "free" free -m -fi - -if [ "$QUICK" = false ]; then - collect "df" df -h -fi - -# -- Processes -- - -section "Processes" -if [ "$IS_MACOS" = true ]; then - collect "ps-cpu" sh -c 'ps -eo pid,ppid,comm,%mem,%cpu | sort -k5 -rn | head -30' -else - collect "ps-cpu" sh -c 'ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%cpu | head -30' -fi - -if [ "$QUICK" = false ]; then - if [ "$IS_MACOS" = true ]; then - collect "ps-mem" sh -c 'ps -eo pid,ppid,comm,%mem,%cpu | sort -k4 -rn | head -30' - collect "top" sh -c 'top -l 1 | head -50' - else - collect "ps-mem" sh -c 'ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%mem | head -30' - collect "top" sh -c 'top -b -n 1 | head -50' - fi -fi - -# -- GPU -- - -section "GPU" -collect "nvidia-smi" nvidia-smi - -if [ "$QUICK" = false ]; then - collect "nvidia-smi-dmon" nvidia-smi dmon -s pucvmet -c 10 - collect "nvidia-smi-query" nvidia-smi --query-gpu=name,utilization.gpu,utilization.memory,memory.total,memory.used,temperature.gpu,power.draw --format=csv -fi - -# -- Docker -- - -section "Docker" -collect "docker-ps" docker ps -a -collect "docker-stats" docker stats --no-stream - -if [ "$QUICK" = false ]; then - collect "docker-info" docker info - collect "docker-df" docker system df -fi - -# Collect logs for NemoClaw-related containers -for cid in $(docker ps -a --filter "label=com.nvidia.nemoclaw" --format '{{.Names}}' 2>/dev/null || true); do - collect "docker-logs-${cid}" docker logs --tail 200 "$cid" - if [ "$QUICK" = false ]; then - collect "docker-inspect-${cid}" docker inspect "$cid" - fi -done - -# -- OpenShell -- - -section "OpenShell" -collect "openshell-status" openshell status -collect "openshell-sandbox-list" openshell sandbox list -collect "openshell-sandbox-get" openshell sandbox get "$SANDBOX_NAME" -collect "openshell-logs" openshell logs "$SANDBOX_NAME" - -if [ "$QUICK" = false ]; then - collect "openshell-gateway-info" openshell gateway info -fi - -# -- Onboard session state -- - -section "Onboard Session" -if [ -n "$ONBOARD_SESSION_HELPER" ] && [ -f "$ONBOARD_SESSION_HELPER" ] && command -v node >/dev/null 2>&1; then - # shellcheck disable=SC2016 - collect "onboard-session-summary" node -e ' - const helper = require(process.argv[1]); - const summary = helper.summarizeForDebug(); - if (!summary) { - process.stdout.write("No onboard session state found.\n"); - process.exit(0); - } - process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); - ' "$ONBOARD_SESSION_HELPER" -else - echo " (onboard session helper not available, skipping)" -fi - -# -- Sandbox internals (via SSH using openshell ssh-config) -- - -if command -v openshell &>/dev/null \ - && openshell sandbox list 2>/dev/null \ - | awk 'NF { if (tolower($1) == "name") next; print $1 }' \ - | grep -Fxq -- "$SANDBOX_NAME"; then - section "Sandbox Internals" - - # Build a temporary SSH config so we can run commands inside the sandbox. - # This follows the pattern from OpenShell's own demo.sh. - SANDBOX_SSH_CONFIG=$(mktemp "${TMPDIR_BASE}/nemoclaw-ssh-XXXXXX") - if openshell sandbox ssh-config "$SANDBOX_NAME" >"$SANDBOX_SSH_CONFIG" 2>/dev/null; then - SANDBOX_SSH_HOST="openshell-${SANDBOX_NAME}" - SANDBOX_SSH_KNOWN=$(mktemp "${TMPDIR_BASE}/nemoclaw-ssh-known-XXXXXX") - SANDBOX_SSH_OPTS=(-F "$SANDBOX_SSH_CONFIG" -o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$SANDBOX_SSH_KNOWN" -o ConnectTimeout=10) - - collect "sandbox-ps" ssh "${SANDBOX_SSH_OPTS[@]}" "$SANDBOX_SSH_HOST" ps -ef - collect "sandbox-free" ssh "${SANDBOX_SSH_OPTS[@]}" "$SANDBOX_SSH_HOST" free -m - if [ "$QUICK" = false ]; then - collect "sandbox-top" ssh "${SANDBOX_SSH_OPTS[@]}" "$SANDBOX_SSH_HOST" 'top -b -n 1 | head -50' - collect "sandbox-gateway-log" ssh "${SANDBOX_SSH_OPTS[@]}" "$SANDBOX_SSH_HOST" tail -200 /tmp/gateway.log - fi - else - warn "Could not generate SSH config for sandbox '${SANDBOX_NAME}', skipping internals" - fi -fi +SCRIPT_PATH="${BASH_SOURCE[0]:-$0}" +case "$SCRIPT_PATH" in + */*) SCRIPT_DIR="${SCRIPT_PATH%/*}" ;; + *) SCRIPT_DIR="." ;; +esac +SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd -P)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" +CLI_JS="${NEMOCLAW_CLI_JS:-$REPO_ROOT/dist/nemoclaw.js}" -# -- Network (full mode only) -- - -if [ "$QUICK" = false ]; then - section "Network" - if [ "$IS_MACOS" = true ]; then - collect "listening" sh -c 'netstat -anp tcp | grep LISTEN' - collect "ifconfig" ifconfig - collect "routes" netstat -rn - collect "dns-config" scutil --dns - else - collect "ss" ss -ltnp - collect "ip-addr" ip addr - collect "ip-route" ip route - collect "resolv-conf" cat /etc/resolv.conf +if [ -f "$CLI_JS" ]; then + NODE_BIN="${NEMOCLAW_NODE:-${NODE:-}}" + if [ -z "$NODE_BIN" ]; then + NODE_BIN="$(command -v node || true)" fi - collect "nslookup" nslookup integrate.api.nvidia.com - # shellcheck disable=SC2016 - collect "curl-models" sh -c 'code=$(curl -s -o /dev/null -w "%{http_code}" https://integrate.api.nvidia.com/v1/models); echo "HTTP $code"; if [ "$code" -ge 200 ] && [ "$code" -lt 500 ]; then echo "NIM API reachable"; else echo "NIM API unreachable"; exit 1; fi' - collect "lsof-net" sh -c 'lsof -i -P -n 2>/dev/null | head -50' - _dp="$(printf '%s' "${NEMOCLAW_DASHBOARD_PORT:-18789}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - case "$_dp" in *[!0-9]* | '') _dp=18789 ;; esac - [ "$_dp" -ge 1024 ] && [ "$_dp" -le 65535 ] 2>/dev/null || _dp=18789 - collect "lsof-dashboard" lsof -i ":${_dp}" -fi - -# -- Kernel / IO (full mode only) -- - -if [ "$QUICK" = false ]; then - section "Kernel / IO" - if [ "$IS_MACOS" = true ]; then - collect "vmstat" vm_stat - collect "iostat" iostat -c 5 -w 1 - else - collect "vmstat" vmstat 1 5 - collect "iostat" iostat -xz 1 5 + if [ -z "$NODE_BIN" ]; then + echo "ERROR: node is required to run NemoClaw diagnostics." >&2 + exit 127 fi + exec "$NODE_BIN" "$CLI_JS" debug "$@" fi -# -- dmesg (always, last 100 lines) -- - -section "Kernel Messages" -if [ "$IS_MACOS" = true ]; then - collect "system-log" sh -c 'log show --last 5m --predicate "eventType == logEvent" --style compact 2>/dev/null | tail -100' -else - collect "dmesg" sh -c 'dmesg | tail -100' -fi - -# ── Produce tarball if requested ───────────────────────────────── - -if [ -n "$OUTPUT" ]; then - tar czf "$OUTPUT" -C "$(dirname "$COLLECT_DIR")" "$(basename "$COLLECT_DIR")" - info "Tarball written to ${OUTPUT}" - warn "Known secrets are auto-redacted, but please review for any remaining sensitive data before sharing." - info "Attach this file to your GitHub issue." -fi - -echo "" -info "Done. If filing a bug, run with --output and attach the tarball to your issue:" -info " nemoclaw debug --output /tmp/nemoclaw-debug.tar.gz" +exec nemoclaw debug "$@" diff --git a/test/secret-redaction.test.ts b/test/secret-redaction.test.ts index a5da6386f0d..8ab5f8d77de 100644 --- a/test/secret-redaction.test.ts +++ b/test/secret-redaction.test.ts @@ -16,7 +16,7 @@ const require = createRequire(import.meta.url); const { redact: runnerRedact } = require("../dist/lib/runner"); describe("secret redaction consistency (#1736)", () => { - // Tokens whose prefix is a literal string that must appear in debug.sh. + // Tokens whose prefix is a literal string that must be redacted by the shared debug redactor. const LITERAL_PREFIX_TOKENS = [ { name: "NVIDIA API key", token: "nvapi-" + "a".repeat(30) }, { name: "NVIDIA Cloud Functions", token: "nvcf-" + "b".repeat(30) }, @@ -27,10 +27,8 @@ describe("secret redaction consistency (#1736)", () => { }, ]; - // Tokens added for messaging integrations (#2336). debug.sh uses - // character-class regexes for these, so the prefix-containment sub-test - // does not apply — they are covered by the runner/debug TS blocks and - // by the EXPECTED_SHELL_PREFIXES substring check (xox) where applicable. + // Tokens added for messaging integrations (#2336). They are covered by + // the shared runner/debug TypeScript redactors. const MESSAGING_TOKENS = [ { name: "Slack bot token", token: "xoxb-" + "1".repeat(12) + "-" + "e".repeat(24) }, { name: "Slack app token", token: "xapp-" + "1".repeat(12) + "-" + "f".repeat(24) }, @@ -82,7 +80,12 @@ describe("secret redaction consistency (#1736)", () => { try { const result = spawnSync("bash", [join(import.meta.dirname, "..", "scripts", "debug.sh"), "--quick"], { encoding: "utf-8", - env: { ...process.env, TMPDIR: tmp, PATH: `${fakeBin}:${process.env.PATH || ""}` }, + env: { + ...process.env, + NEMOCLAW_NODE: process.execPath, + TMPDIR: tmp, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + }, timeout: 30_000, }); expect(result.status).toBe(0); @@ -94,26 +97,20 @@ describe("secret redaction consistency (#1736)", () => { }, 40_000); }); - describe("debug.sh sed fallback includes essential prefixes", () => { - it("redacts essential token prefixes when node is unavailable", () => { - const tmp = mkdtempSync(join(tmpdir(), "nemoclaw-debug-sed-redact-")); + describe("debug.sh wrapper locates node from env", () => { + it("uses NEMOCLAW_NODE and the compiled redactor when node is absent from PATH", () => { + const tmp = mkdtempSync(join(tmpdir(), "nemoclaw-debug-node-env-redact-")); const fakeBin = join(tmp, "bin"); mkdirSync(fakeBin); for (const name of [ "cat", - "dirname", "dmesg", "free", "head", - "mktemp", "ps", - "pwd", - "rm", - "sed", + "sh", "sort", "tail", - "tee", - "tr", "uname", "uptime", ]) { @@ -134,7 +131,12 @@ describe("secret redaction consistency (#1736)", () => { try { const result = spawnSync("/bin/bash", [join(import.meta.dirname, "..", "scripts", "debug.sh"), "--quick"], { encoding: "utf-8", - env: { ...process.env, TMPDIR: tmp, PATH: fakeBin }, + env: { + ...process.env, + NEMOCLAW_NODE: process.execPath, + TMPDIR: tmp, + PATH: fakeBin, + }, timeout: 30_000, }); expect(result.status).toBe(0);