diff --git a/scripts/sync-skill-cli-contract.sh b/scripts/sync-skill-cli-contract.sh new file mode 100755 index 0000000..115f108 --- /dev/null +++ b/scripts/sync-skill-cli-contract.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) +# Portable skills cannot reference sibling skill directories. Package +# byte-identical shared references inside both report skills; validate-skills.sh +# rejects drift. +canonical="$repo_root/skills/nvfleetint/references/cli-contract.md" + +targets=( + "$repo_root/skills/fleet-health-report/references/cli-contract.md" + "$repo_root/skills/node-rca-rcca/references/cli-contract.md" +) +for target in "${targets[@]}"; do + cp -- "$canonical" "$target" +done + +cp -- "$repo_root/skills/fleet-health-report/references/html-theme.md" \ + "$repo_root/skills/node-rca-rcca/references/html-theme.md" +cp -- "$repo_root/skills/fleet-health-report/references/workspace.md" \ + "$repo_root/skills/node-rca-rcca/references/workspace.md" + +echo "Synchronized portable shared report references." diff --git a/scripts/validate-skills.sh b/scripts/validate-skills.sh index 0346998..f5e0ac6 100755 --- a/scripts/validate-skills.sh +++ b/scripts/validate-skills.sh @@ -9,6 +9,8 @@ fail() { exit 1 } +repo_root=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) + found=0 for skill_md in skills/*/SKILL.md; do [ -f "$skill_md" ] || continue @@ -85,4 +87,270 @@ for skill_md in skills/*/SKILL.md; do done [ "$found" -eq 1 ] || fail "no skills/*/SKILL.md files found" + +canonical_contract="$repo_root/skills/nvfleetint/references/cli-contract.md" +contracts=( + "$repo_root/skills/fleet-health-report/references/cli-contract.md" + "$repo_root/skills/node-rca-rcca/references/cli-contract.md" +) +for contract in "${contracts[@]}"; do + if ! cmp -s "$canonical_contract" "$contract"; then + fail "$contract differs from the canonical CLI contract; run scripts/sync-skill-cli-contract.sh" + fi +done + +if grep -R -n --include='*.md' -- '--timeout 60s' "$repo_root/skills" >/dev/null; then + fail "skills contain the obsolete shortened --timeout 60s example" +fi +if grep -R -nE --include='*.md' 'nvfleetctl|pkg/fleetintelligence' "$repo_root/skills" >/dev/null; then + fail "skills contain a removed client or package name" +fi +if grep -R -nE --include='*.md' 'node list.*--sort-by health([[:space:]]|$)' "$repo_root/skills" >/dev/null; then + fail "skills use invalid node sort key health; use healthStatus" +fi +if grep -R -nE --include='*.md' 'rsync.*--delete.*(\.agents|skills/)' "$repo_root/skills" >/dev/null; then + fail "skills contain a broad destructive install command" +fi + +fleet_skill="$repo_root/skills/fleet-health-report/SKILL.md" +required_fleet_rules=( + 'nvfleetint auth list --output json' + 'nvfleetint auth status --profile --output json' + 'same explicit `--profile `' + 'Probe each list with the same filters, `--view basic` where supported' + 'Resolve supplied names to IDs' + 'alert options --view active' + 'alert summary ' + '--sort-by alert --order desc --page-size 10' + '--component-type --all' + 'agent_liveness' + 'comma-separated list of component IDs excluding exact IDs `psirt` and `agent_liveness`' + 'Alert collection invokes at most 12 `nvfleetint` commands' + 'An `--all` command may make multiple paginated requests.' + 'Do not fetch every affected node merely to count or rank them.' + 'Nodes with Active Alerts' + 'Machines Needing Immediate Attention (up to 10)' + '1. Fleet Summary:' + '2. Fleet Distribution:' + '3. Active Alerts:' + '4. Error Distribution:' + 'Fleet Health Percentage' + 'Do not show both or repeat the percentage' + 'report error --view list --group-by error' + 'without an aggregate fleet severity badge' + 'date -u -d "@$now"' + 'date -u -r "$now" -v-24H' +) +for required in "${required_fleet_rules[@]}"; do + if ! grep -Fq -- "$required" "$fleet_skill"; then + fail "$fleet_skill is missing required collection rule: $required" + fi +done + +collection_order=( + 'nvfleetint auth list --output json' + 'nvfleetint auth status --profile --output json' + 'nvfleetint overview --profile --output json' + 'nvfleetint computezone list' + 'nvfleetint nodegroup list' + 'nvfleetint node list ' + 'nvfleetint report error' + 'nvfleetint alert options' + 'nvfleetint alert summary' + 'nvfleetint alert node' +) +previous_line=0 +for command in "${collection_order[@]}"; do + line=$(awk -v needle="$command" 'index($0, needle) { print NR; exit }' "$fleet_skill") + if [ -z "$line" ]; then + fail "$fleet_skill is missing ordered workflow command: $command" + fi + if [ "$line" -le "$previous_line" ]; then + fail "$fleet_skill has workflow command out of order: $command" + fi + previous_line=$line +done + +for obsolete in '5,000 alerts' 'alert list --severity Critical --all' \ + 'alert list --severity Warning --all' 'Critical.count + Warning.count' \ + 'alert list --all --output json' 'prev_start' 'id="trend"' 'Overall status:' \ + 'alert node --view active --without-psirt' 'server-ranked top 10' \ + 'only the top 10' 'top-10 summary nodes' 'top-10 drill-down' \ + 'top 10 machines' 'top machines needing attention'; do + if grep -Fq "$obsolete" "$fleet_skill"; then + fail "$fleet_skill contains obsolete fleet-report guidance: $obsolete" + fi +done + +nvfleet_skill="$repo_root/skills/nvfleetint/SKILL.md" +required_nvfleet_rules=( + '--compute-zone-names' + '--nodegroup-names' + 'supports sorting only by `hostname` or `nodeUUID`' + 'alert summary --output json' + 'alert node --output json' + 'It does not support' +) +for required in "${required_nvfleet_rules[@]}"; do + if ! grep -Fq -- "$required" "$nvfleet_skill"; then + fail "$nvfleet_skill is missing required CLI guidance: $required" + fi +done + +for required in healthNodeCount '--view basic' '--firmware-check'; do + if ! grep -Fq -- "$required" "$canonical_contract"; then + fail "$canonical_contract is missing required shared guidance: $required" + fi +done + +node_skill="$repo_root/skills/node-rca-rcca/SKILL.md" +for required in 'nvfleetint auth list --output json' \ + 'nvfleetint auth status --profile --output json' \ + 'node list --hostname --view basic --all' \ + 'node describe --profile --output json' \ + 'Fetch current alerts and complete historical alerts' \ + 'Aggregate current and historical alerts by component ID/display name and status' \ + 'deduplicating the same `alertUuid` across both sets' \ + 'count prior historical rows with the same component ID after excluding its own `alertUuid`' \ + 'Describe every unique current alert' \ + 'Use each description' \ + 'timeline, messages, errors, incidents, and suggested actions' \ + 'Run at most four describe calls concurrently' \ + 'collapsed `
` breakdown for every current alert' \ + 'Prefer official NVIDIA documentation' \ + 'Never include hostname, node UUID, profile, tenant, or customer data in a query'; do + if ! grep -Fq "$required" "$node_skill"; then + fail "$node_skill is missing required RCA workflow guidance: $required" + fi +done +for required in 'alert node --without-psirt --all' \ + 'alert node --view historical --without-psirt --all' \ + 'alert describe --node --profile --output json'; do + if ! grep -Fq "$required" "$node_skill"; then + fail "$node_skill is missing required alert command: $required" + fi +done + +node_collection_order=( + 'nvfleetint auth list' + 'nvfleetint auth status' + 'nvfleetint node list' + 'nvfleetint node describe' + 'nvfleetint alert node --without-psirt' + 'nvfleetint alert node --view historical' + 'nvfleetint alert describe ' +) +previous_line=0 +for command in "${node_collection_order[@]}"; do + line=$(awk -v needle="$command" 'index($0, needle) { print NR; exit }' "$node_skill") + if [ -z "$line" ]; then + fail "$node_skill is missing ordered workflow command: $command" + fi + if [ "$line" -le "$previous_line" ]; then + fail "$node_skill has workflow command out of order: $command" + fi + previous_line=$line +done + +if grep -R -n --include='*.md' 'nvfleetint alert timeline' "$repo_root/skills" >/dev/null; then + fail "skills reference the removed alert timeline command" +fi +if grep -niE '\bevents?\b|nvfleetint event (list|buckets)' "$node_skill" >/dev/null; then + fail "node-rca-rcca must use alert evidence without event API calls" +fi +if grep -niE '48-hour|172800|date -u|alert window' "$node_skill" >/dev/null; then + fail "node-rca-rcca must use the complete historical alert response without a local time cutoff" +fi + +fleet_theme="$repo_root/skills/fleet-health-report/references/html-theme.md" +node_theme="$repo_root/skills/node-rca-rcca/references/html-theme.md" +fleet_workspace="$repo_root/skills/fleet-health-report/references/workspace.md" +node_workspace="$repo_root/skills/node-rca-rcca/references/workspace.md" + +for report_skill in fleet-health-report node-rca-rcca; do + references="$repo_root/skills/$report_skill/references" + for reference in cli-contract.md html-theme.md workspace.md; do + if [ ! -f "$references/$reference" ]; then + fail "$report_skill is missing shared reference $reference" + fi + done + reference_count=$(find "$references" -mindepth 1 -maxdepth 1 -type f | wc -l | tr -d ' ') + if [ "$reference_count" -ne 3 ]; then + fail "$report_skill must contain exactly the three shared references" + fi +done + +if ! cmp -s "$fleet_theme" "$node_theme"; then + fail "report HTML theme references differ; run scripts/sync-skill-cli-contract.sh" +fi +if ! cmp -s "$fleet_workspace" "$node_workspace"; then + fail "report workspace references differ; run scripts/sync-skill-cli-contract.sh" +fi + +for theme in "$fleet_theme" "$node_theme"; do + for theme_rule in '--surface-canvas: #f7f7f7' 'html[data-theme="dark"]' \ + 'maximum width 1180px' '16px corners' '
' 'Unverified' 'For print'; do + if ! grep -Fq -- "$theme_rule" "$theme"; then + fail "$theme is missing shared theme guidance: $theme_rule" + fi + done + for report_section in at-a-glance distribution errors alerts summary node-details root-cause; do + if grep -Fq "id=\"$report_section\"" "$theme"; then + fail "$theme must not define report-specific section $report_section" + fi + done +done + +for workspace in "$fleet_workspace" "$node_workspace"; do + for workspace_rule in 'mktemp -d "/tmp/nvfleet-report.XXXXXXXX"' \ + 'Write the final HTML outside the scratch directory' \ + 'find "$work" -mindepth 1 -maxdepth 1' 'rmdir -- "$work"'; do + if ! grep -Fq -- "$workspace_rule" "$workspace"; then + fail "$workspace is missing shared workspace guidance: $workspace_rule" + fi + done +done + +for section_id in at-a-glance distribution errors alerts; do + if ! grep -Fq "$section_id" "$fleet_skill"; then + fail "$fleet_skill is missing required section check $section_id" + fi +done +fleet_section_order=( + '1. Fleet Summary:' + '2. Fleet Distribution:' + '3. Active Alerts:' + '4. Error Distribution:' +) +previous_line=0 +for section in "${fleet_section_order[@]}"; do + line=$(awk -v needle="$section" 'index($0, needle) { print NR; exit }' "$fleet_skill") + if [ -z "$line" ] || [ "$line" -le "$previous_line" ]; then + fail "$fleet_skill has a missing or out-of-order report section: $section" + fi + previous_line=$line +done +for section_id in summary node-details evidence root-cause corrective-actions references unknowns; do + if ! grep -Fq "$section_id" "$node_skill"; then + fail "$node_skill is missing required section id $section_id" + fi +done +node_section_order=( + '1. Executive Summary:' + '2. Node Details:' + '3. Alert Evidence:' + '4. Root Cause Analysis:' + '5. Corrective Action Plan:' + '6. References:' + '7. Assumptions and Unknowns:' +) +previous_line=0 +for section in "${node_section_order[@]}"; do + line=$(awk -v needle="$section" 'index($0, needle) { print NR; exit }' "$node_skill") + if [ -z "$line" ] || [ "$line" -le "$previous_line" ]; then + fail "$node_skill has a missing or out-of-order report section: $section" + fi + previous_line=$line +done + echo "All agent skills are valid." diff --git a/skills/fleet-health-report/SKILL.md b/skills/fleet-health-report/SKILL.md index b3680fb..57c59e1 100644 --- a/skills/fleet-health-report/SKILL.md +++ b/skills/fleet-health-report/SKILL.md @@ -1,325 +1,117 @@ --- name: fleet-health-report -description: Generate a standalone fleet-wide HTML health snapshot from live nvfleetint data, including node health, capacity, alerts, error trends, and machines needing attention. Use for fleet dashboards, executive summaries, recurring issue analysis, or fleet-wide reports. Do not use for a single-node root-cause investigation. +description: Generate a standalone fleet-wide HTML health snapshot from live nvfleetint data, including node health, capacity, active-alert impact, recent errors, and machines needing immediate attention. Use for fleet dashboards, executive summaries, or scoped fleet reports. Do not use for a single-node root-cause investigation. --- # Fleet Health Report -Produce a self-contained HTML snapshot from live Fleet Intelligence data across -the whole fleet. Gather fresh `nvfleetint` evidence about node health, active -alerts, inventory distribution, and error trends, then derive fleet-wide metrics -and compose the report. Compose the layout and presentation to fit the data -available on each invocation, working within the required sections and visual -system below; do not use a fixed renderer or a deterministic report-generation -script. - -## When to use - -Use this skill for a **fleet-wide snapshot** across many nodes — the user wants -an at-a-glance health summary, status dashboard, alert or error trend, recurring -issue list, or most-alerted ranking for the whole fleet. For a single-node -investigation of why one degraded or unhealthy node broke and what to do about -it, use the `node-rca-rcca` skill instead. - -## Instructions - -1. Read the **Operating rules** before running anything. -2. **Collect live data** with fresh `nvfleetint` queries using - `--all --output json`, including the two adjacent trend windows. -3. **Prove completeness** by validating every response before deriving metrics. -4. **Derive metrics** exactly as specified — never inventing semantics or values. -5. **Build the HTML snapshot** using the required sections and visual system. -6. **Deliver** the single `.html` file, remove temporary artifacts, and return a - clickable path. - -## Operating rules - -- Use read-only Fleet Intelligence commands only. This skill never changes fleet - state; do not run write/delete/tag commands. -- Run fresh `nvfleetint` backend queries during the current invocation. Never - substitute examples, fixtures, cached output, prior reports, assumptions, or - invented values. -- Treat `nvfleetint` output as **evidence**. Prefer `--output json` for capture; - use table output only for quick human scanning, never as the parsed source of - a metric. -- Add both `--all` and `--output json` to every command that requests fleet, - alert, inventory, or report data and that supports `--all` (see the per-command - notes under Collect live data). The one exception is `overview`, a single-object - summary read that has no paginated form: run it with `--output json` and no - `--all`. Apart from that exception, do not substitute a non-paginated data - command or a view that rejects `--all` for one that supports it. -- Treat CLI help, version, and local authentication inspection as metadata - operations rather than fleet-data requests. Do not cite them as evidence of - fleet health. -- Capture stdout, stderr, exit status, the exact command, and collection time - for every query. Parse stdout as JSON only after a zero exit status. -- Preserve exact timestamps and time zones from command output. State when a - timestamp is absent or ambiguous instead of guessing. -- Never expose API keys, credentials, environment variables, authorization - headers, or raw config file contents in commands, logs, source data, HTML, or - chat. -- Do not create persistent intermediate artifacts. Among artifacts generated by - this report run, the only file left in the workspace must be the final - standalone `.html` report. Keep command envelopes, parsed JSON, helper - scripts, and scratch data in memory or OS-managed temporary locations that are - cleaned up before finishing. Do not leave generated `*.envelope.json`, - `*.json`, `*.py`, logs, or auxiliary files behind. -- Do not publish a successful report when any required query is missing, stale, - invalid, unauthorized, or incomplete. Explain the failure and identify the - affected command without guessing the missing results. - -## Running the CLI - -Use the installed `nvfleetint` binary and run each query with a suitable -`--timeout` (for example `--timeout 60s`). If the installed CLI differs from the -invocations below, inspect `nvfleetint --help`. - -## Prerequisites - -- Run commands through the harness's local command-execution capability. The - examples use POSIX shell syntax; on Windows, use equivalent PowerShell while - preserving the `nvfleetint` arguments and evidence rules. -- `nvfleetint` is installed and on `PATH`. Confirm with `command -v nvfleetint` - on POSIX or `Get-Command nvfleetint` in PowerShell. -- A structured JSON processor is available. The examples use `jq`; an equivalent - parser is acceptable, but grepping human-readable table output is not. -- The session is authenticated. Confirm without exposing secrets: - - ```bash - nvfleetint auth status - ``` - - `auth status` is diagnostic: it exits `0` and reports a `Connection:` line - rather than failing on a bad key. Require `Connection: ok`. Treat - `Connection: unauthorized` (a missing, invalid, or expired key), - `unauthenticated`, or `error: ...` as an authentication/authorization failure, - never an empty fleet. Likewise, treat exit code `77` or an HTTP 401/403 - `api_error` on any subsequent data command as an auth failure: ask the user to - authenticate (`nvfleetint auth add --api-key `) without - asking them to paste a key into chat, and stop. Other `api_error` responses - (for example HTTP 400 or 5xx) are backend, not auth, failures — report the - concrete blocker instead of prompting for re-authentication. -- Credentials live in named profiles. Run `nvfleetint auth list` first. If the - user named an environment or tenant, use that profile. Otherwise, if more than - one profile exists, **ask the user which one to report on and wait for their - answer** — do not fall back to the current profile (the one marked `*`) or the - first one listed, because a fleet report attributed to the wrong tenant is - worse than a delayed one. Once the profile is known, pass the same - `--profile ` to *every* command in the report, including `auth status`, - so the whole snapshot comes from one fleet. - -## Collect live data - -Let `T` be the UTC collection time and record the exact RFC3339 boundaries of -every window you use. - -### 1. Snapshot the current fleet, inventory, and alerts +Generate one offline HTML snapshot from fresh `nvfleetint` JSON. Read the [CLI contract](references/cli-contract.md), [HTML theme](references/html-theme.md), and [workspace guide](references/workspace.md) before collecting data. + +## Workflow + +### 1. Resolve and verify the profile + +Resolve credentials before any report query: + +```bash +nvfleetint auth list --output json +nvfleetint auth status --profile --output json +``` + +Use the user-named profile or the sole configured profile. If multiple profiles exist and none was requested, ask which one to use and identify the current one as the default suggestion. Require `connection` equal to `ok`, then pass the same explicit `--profile ` to every API-backed command below. + +### 2. Collect overview + +Collect the tenant overview before inventory: + +```bash +nvfleetint overview --profile --output json +``` + +Use it for the entire-fleet headline. In a scoped report, label it fleet-wide context and derive scoped totals from the filtered node list instead. + +### 3. Collect inventory and resolve scope + +Accept the entire fleet, compute-zone names, or node-group names. List compute zones first, node groups second, and nodes third. Resolve supplied names to IDs internally; clarify only ambiguous name matches. + +Probe each list with the same filters, `--view basic` where supported, and `--page-size 1` before its full pull. Collect the full lists in this order: + +```bash +nvfleetint computezone list --all --profile --output json +nvfleetint nodegroup list --all --profile --output json +nvfleetint node list --all --profile --output json +``` + +After the first two lists, apply resolved `--compute-zone-ids` or `--nodegroup-ids` to the node query. + +### 4. Collect recent errors + +Pin one 24-hour error window. Use GNU `date -u -d "@$now"` and `date -u -d "@$((now - 86400))"`, or BSD/macOS `date -u -r "$now"` and `date -u -r "$now" -v-24H`, formatted as RFC3339 UTC. + +```bash +nvfleetint report error --view list --group-by error \ + --start "$start" --end "$end" --all \ + --profile --output json +``` + +Sum row `count`; pagination total counts grouped rows, not error occurrences. The error API cannot filter by zone/group, so label it tenant-wide in a scoped report or omit it when strictly scoped evidence is required. + +### 5. Collect filtered active alerts + +Discover the server-supported filter values first: ```bash -nvfleetint overview --output json --timeout 60s -nvfleetint computezone list --all --output json --timeout 60s -nvfleetint nodegroup list --all --output json --timeout 60s -nvfleetint node list --all --output json --timeout 60s -nvfleetint alert list --all --output json --timeout 60s +nvfleetint alert options --view active --profile --output json ``` -`overview` is the backend's own fleet-wide summary and the headline anchor for -fleet totals. It returns a single object (no `items`/`pagination` envelope), so -it takes `--output json` but **not** `--all`. Its fields are authoritative -totals (`nodesCount`, `gpusCount`, `cpuCoresCount`, `nodeGroupCount`, -`computeZoneCount`), health-state counts (`healthNodeCount` — the healthy count, -note the backend spelling without the "y" — plus `degradedNodeCount`, -`unhealthyNodeCount`, `unknownNodeCount`), a backend `healthPercentage`, and a -`metrics` array of fleet-level metrics (each with `name`, `description`, `unit`, -`value`, `aggregation`, `lastUpdated`). Metrics are included by default; keep -them (do not pass `--include-metrics=false`). +From the returned `componentTypes` options, build a comma-separated list of component IDs excluding exact IDs `psirt` and `agent_liveness`. Stop if no component IDs remain. -The `list` commands remain the anchor evidence for per-entity detail: fleet -composition, per-node health/agent/firmware/verification state, and active alerts -by severity and component. Use the `list` datasets — not `overview` — for -anything that needs per-node joins, per-entity breakdowns, or rankings; -`overview` supplies only fleet-wide aggregates. +Request the filtered count of all affected nodes and up to 10 machines ordered by active-alert count: -### 2. Compare two adjacent, equal-duration trend windows +```bash +nvfleetint alert summary --view active \ + --component-type \ + --sort-by alert --order desc --page-size 10 \ + --profile --output json +``` -Default to 24-hour windows unless the user requests another horizon. Compare -`[T-24h, T)` with `[T-48h, T-24h)`: +Use summary `.total` for all Nodes with Active Alerts and `.totalCritical`/`.totalWarning` for filtered fleet-wide severity totals. The bounded page contains up to 10 machines ordered by active-alert count; state `showing N of total` when applicable. Do not fetch every affected node merely to count or rank them. + +For each returned UUID only, fetch its full filtered drill-down: ```bash -nvfleetint report error --view list --group-by error --start --end --all --output json --timeout 60s -nvfleetint report error --view list --group-by error --start --end --all --output json --timeout 60s +nvfleetint alert node --view active \ + --component-type --all \ + --profile --output json ``` -Use `report error --view list`, not `overview` or `graph`, because only list -view supports `--all`. Sum each returned row's `count` to calculate the number -of errors in a window; `pagination.total` is the number of grouped rows, not the -error-event count. +Run at most four node calls concurrently. Alert collection invokes at most 12 `nvfleetint` commands: one options command, one summary command, and up to 10 node commands. An `--all` command may make multiple paginated requests. + +### 6. Derive and write the report + +- Entire fleet: show `overview.healthPercentage` once as `Fleet Health Percentage`. Scoped: show `100 * healthy / total` once as `Healthy Node Percentage` and its formula. Do not show both or repeat the percentage in Fleet Summary. +- Keep node health/severity at source level; present fleet counts and distributions without an aggregate fleet severity badge. +- Join the returned summary machines to inventory by UUID and preserve the returned order. +- Use node-alert rows only for those machines' drill-down detail; do not infer unseen component distribution. + +Apply the shared HTML theme and use these four report sections: + +1. Fleet Summary: executive summary, fleet totals, and health distribution. +2. Fleet Distribution: compute zones, node groups, capacity, and operational signals. +3. Active Alerts: filtered Nodes with Active Alerts, severity aggregates, Machines Needing Immediate Attention (up to 10), and one expandable per-machine drill-down showing component, status, start time, last update, and available evidence text. +4. Error Distribution: grouped recent errors over the pinned window. + +### 7. Validate and deliver -### 3. Optional deeper context +Apply the CLI contract's completeness checks. The bounded summary page is the only intentional partial result; use its top-level aggregates for fleet-wide claims. For an entire-fleet report, reconcile overview totals with complete inventory totals. -Run additional queries only when they materially clarify the report, and still -include both required flags: +Validate the final HTML: ```bash -nvfleetint alert timeline --all --output json --timeout 60s -nvfleetint alert timeline --node --all --output json --timeout 60s -nvfleetint report error --view list --group-by node --start --end --all --output json --timeout 60s +for id in at-a-glance distribution alerts errors; do + grep -q "id=\"$id\"" "$out" || exit 1 +done +grep -q '' "$out" && ! grep -qE '\[[a-z_]+\]' "$out" ``` -## Prove completeness - -Validate every fleet-data response independently before calculating metrics: - -1. Require exit status `0`, nonempty stdout, valid JSON, and no top-level `error` - / `api_error` object. -2. Require a top-level `items` array and `pagination` object from every `--all` - query. The `overview` read is exempt: it returns a single summary object with - no `items`/`pagination`. Validate it as a single-object read — exit `0`, - nonempty stdout, valid JSON, and no top-level `error`/`api_error` — and treat - any of its summary fields as `N/A` when the backend omits them rather than - substituting a value. -3. Require `pagination.hasMore` to be `false`, `pagination.pagesFetched` to be at - least `1`, and `items.length` to equal `pagination.total`. -4. Treat an empty fleet as valid only when `items` is empty, `total` is `0`, and - `hasMore` is `false`. Report metrics as `N/A` where a denominator is zero. -5. Retry a completeness mismatch once because the backend may change during - pagination. If it still mismatches, stop and report the dataset as incomplete - rather than analyzing a partial set. -6. Keep every validated dataset through report generation. Displaying a top-N - table is allowed, but collecting or analyzing only a top-N subset is not. -7. Cross-check `overview` against the validated `list` datasets before trusting - either blindly: `overview.nodesCount` should agree with the `node list` total, - `overview.nodeGroupCount` with `nodegroup list`, and `overview.computeZoneCount` - with `computezone list`. Minor drift can occur because the summary and the - lists are separate backend reads taken moments apart; when they differ - slightly, prefer `overview` for the headline totals but surface the - discrepancy in the report rather than hiding it. A large or structural - mismatch (for example, an order-of-magnitude difference) means one dataset is - stale or incomplete — retry once, then stop and report it instead of - publishing. - -## Derive metrics without inventing semantics - -- Join nodes and alerts by the stable node UUID. Preserve unmatched alert records - and label unavailable node metadata instead of dropping them. -- Count health, agent, firmware, and verification states from the complete node - dataset. Keep `Unknown`, missing, and unexpected values visible rather than - coercing them to healthy. - - The backend JSON keeps its original field names: read verification state from - `integrityCheck` (with `integrityCheckReason`, `lastIntegrityCheckTS`) and - location from `geoLocation`. "Verification" and "location" are display terms - only — the JSON never contains those keys. -- Count active alerts by severity and component from the complete alert dataset. - Preserve unexpected severities as `Other`. -- Source fleet-wide totals from `overview` when present: use `nodesCount`, - `gpusCount`, `cpuCoresCount`, `nodeGroupCount`, and `computeZoneCount` as the - authoritative headline counts, and use `healthNodeCount` (healthy), - `degradedNodeCount`, `unhealthyNodeCount`, and `unknownNodeCount` for the - health-state breakdown. Fall back to counts derived from `node list` (or the - other `list` datasets) only when a field is absent from `overview`, and label - which source produced each number. -- Present two health figures and keep them distinct: - - The backend `healthPercentage` from `overview`, labeled as a - backend-supplied fleet health percentage. - - The report-derived **node health score** = `100 * healthy nodes / total - nodes`, using the `overview` health counts (or `node list` when `overview` - omits them), rounded reasonably, shown with its formula and labeled as - report-derived. Do not imply that the backend supplied this composite score. - - Reconcile the two: they usually agree, but may differ if the backend weights - states differently. When they differ, show both and note the difference - rather than averaging or hiding it. If `overview` omits `healthPercentage`, - show only the derived score. -- Present `overview.metrics` as backend-supplied fleet-level metrics (for example - GPU utilization or temperature). Show each metric's `name` (or `description` - when the name is blank) with its `value`, `unit`, `aggregation`, and - `lastUpdated` timestamp verbatim. Do not rename, recompute, convert units, or - invent a metric the backend did not return. Omit the metrics block entirely - when `overview` returns no metrics. -- Derive at-a-glance status transparently: - - Use **Critical** when any active Critical alert or Unhealthy node exists. - - Otherwise use **Needs attention** when any active Warning or `Other` - (unrecognized-severity) alert, Degraded or Unknown node, offline or unknown - agent, failed or unknown firmware check, or degraded, unverified, pending, - unsupported, unknown, or missing verification state exists. - - Otherwise use **Healthy** when at least one node exists and no attention - signal exists. - - Use **No data** only when no nodes were returned, or when *every* returned - node is missing the required health fields. When only some nodes lack a - required field, keep the known values, count the missing ones as - `Unknown`/`N/A`, and still derive the status from the populated nodes — never - suppress a valid metric because part of the fleet is unpopulated. -- Rank machines needing attention using explicit evidence. Prioritize - critical-alert count, Unhealthy health, total active-alert count, warning-alert - count, Degraded or Unknown health, offline agent, failed firmware, and - verification problems. Show the reasons and counts used; do not present the - ranking as a backend-defined risk score. -- Calculate trend from the summed error counts in the two equal windows. Show - current, previous, absolute delta, direction, and percentage change. If the - previous value is zero, show `new increase` or `no change` instead of an - infinite percentage. -- Identify recurring issues as error names present with positive counts in both - windows. Rank by current count, persistence, affected-node count when - available, and change. Label types found only in the current window as new and - types found only in the previous window as no longer observed. Do not claim - event-level recurrence from aggregate data. -- Mark unavailable fields as `N/A`. Never manufacture timestamps, hostnames, - alert causes, remediation, utilization, or capacity. - -## Build the HTML snapshot - -Produce a standalone `.html` file in the current workspace or the user's -requested path. Read -[`references/html-report-template.md`](references/html-report-template.md) before -drafting it, and follow that skeleton, visual system, and status styling. Name it -`fleet-health-report-.html` unless the user provides an output path. - -Requirements: - -- Use semantic HTML with inline CSS and only optional inline JavaScript. Avoid - external fonts, CDNs, remote images, network calls, and runtime dependencies so - the file opens offline. -- HTML-escape every backend-derived string before inserting it. -- Do not embed raw credentials or a full raw-data dump. When showing only the - highest-ranked rows, state the displayed count and the full population count. -- Choose charts, tables, color, typography, density, and layout based on the - actual dataset, within the visual system. Make the result responsive, - accessible, and print-friendly. -- Include, in order: fleet-wide health / at a glance (with the report-derived - node health score and formula, the backend `healthPercentage` when present, - overall status, node/GPU/CPU-core totals, health counts, active-alert totals by - severity, and collection timestamp), fleet distribution and operational signals - (including the `overview` fleet metrics when returned), trend direction over the - two equal windows, issue concentration by component or type, and machines - needing immediate attention. -- Add a short evidence-based action summary when useful. Use remediation or - suggested actions only when returned by the backend; otherwise describe what - deserves investigation without prescribing unsupported fixes. - -## Deliver - -Open or inspect the generated HTML locally enough to catch malformed markup, -missing sections, unescaped content, and obviously inconsistent totals. -Cross-check headline values (node health score, overall status, active-alert -counts) against the validated JSON before finishing. Confirm the report is the -only generated file left in the workspace and remove any temporary artifacts. -Return a clickable path to the HTML report and briefly note its collection time -and comparison horizon. If generation stops for data integrity, authentication, -or backend access, return no fabricated report and state the concrete blocker. - -## Examples - -**Default 24-hour snapshot** - -> User: "Generate a fleet health report." - -Confirm auth with `nvfleetint auth status`, collect the required snapshot and -trend queries, compare `[T-24h, T)` against `[T-48h, T-24h)`, validate -completeness, then produce `fleet-health-report-.html` and return its path with -the collection time and comparison horizon. - -**Custom window** - -> User: "Give me an executive health summary over the last 7 days." - -Same flow, but set the two adjacent windows to 7-day spans (`[T-7d, T)` versus -`[T-14d, T-7d)`) and record the exact RFC3339 boundaries. +Leave only the final HTML and return its path, scope, collection time, and error window. diff --git a/skills/fleet-health-report/references/cli-contract.md b/skills/fleet-health-report/references/cli-contract.md new file mode 100644 index 0000000..7ee6865 --- /dev/null +++ b/skills/fleet-health-report/references/cli-contract.md @@ -0,0 +1,41 @@ +# nvfleetint CLI contract + +Read this before querying live data. + +## Commands and retries + +- Use `--output json`; never parse tables. Confirm flags with `nvfleetint [] --help`. +- `--timeout` is per request (default 2m). Don't lower it. Use 3m–5m for demonstrated slow node pulls above 1,000 rows and 5m–10m for demonstrated slow alert pulls; still bound the whole workflow separately. +- The client retries idempotent GET/HEAD requests up to three attempts for transient network failures and HTTP 408, 429, 500, 502, 503, and 504. During `--all`, only the failed page retries. Never retry the whole command in a loop. +- Exit 127 means not installed; use a release from . + +## Auth and profiles + +- Require `Connection: ok` from `auth status`; it exits 0 even when auth fails. Exit 77 or HTTP 401/403 is auth failure, not an empty fleet. +- Never request or expose an API key. Direct users to and `nvfleetint auth add --api-key `. +- If the user names a tenant/environment, use that profile. If multiple profiles exist and none is named, ask which one. Pass the same explicit `--profile ` to every report command. +- Explicit `--profile` ignores `NVFLEETINT_API_KEY` and `NVFLEETINT_API_URL`. + +## Pagination + +- Lists accept `--page`, `--page-size` (1–100), and `--all`. Non-paginated: `overview`, `node describe`, `node health`, `alert options`, `alert describe`, `event buckets`, `tag list`, and `report error --view overview|graph`. +- Single pages use backend arrays (`nodes`, `nodeGroups`, `computezones`, `alerts`, `events`) plus top-level `total`, `page`, and `pageSize`. Most use `hasMore`; `alert list` uses nonempty `pageCursorNext` instead. `alert summary` uses `nodes`, `alert node` uses `alerts`, and report-error list uses `nodes`. +- `--all` normalizes every paginated list to `{items, pagination:{total,hasMore,pagesFetched}}`; non-paginated commands keep their native shape. +- Count cheaply with the same filters plus `--page-size 1`, reading `.total`. For `alert summary`, `.total` is nodes with matching alerts; use `totalCritical` and `totalWarning` for alert aggregates. `tag list` is the exception: count its non-paginated `tags`. +- `--view basic` returns identities only: node returns hostname/UUID; compute zone and node group return `id`/`name`. Node basic rejects `--health`, `--agent-status`, `--verification-check`, and `--firmware-check`, and sorts only by hostname or nodeUUID. Node-group basic rejects health, gpu-type, and sorting. + +## JSON names + +Use backend names: `healthStatus`, `integrityCheck`, `integrityCheckReason`, `lastIntegrityCheckTS`, and `geoLocation`. `overview` calls the healthy-node count `healthNodeCount`, not `healthyNodeCount`. + +## Completeness + +Before analysis require exit 0, nonempty valid JSON, and no top-level `error`/`api_error`. For every `--all` response require an `items` array, `hasMore == false`, `pagesFetched >= 1`, and—when `total > 0`—item count equal to total. A missing or null total is unreported; a nonempty result with explicit `total: 0` is malformed. Empty with `hasMore: false` is valid. + +Filtered pulls may be composed only when each is complete and the disjoint filters cover the domain. Record each reported total or validated item count, then union. Never analyze partial pages. On malformed/incomplete data, use only the report workflow's allowed fallback or stop. + +The fleet-health workflow's deliberately bounded `alert summary --page-size 10` is the sole partial-page exception: use only its server-computed top-level aggregates for fleet-wide claims, label returned rows `showing N of total`, and never infer the attributes of unseen nodes. + +## Secrets + +Never expose credentials, authorization headers, environment values, or raw config. Check whether auth environment variables are set; never print them. diff --git a/skills/fleet-health-report/references/html-report-template.md b/skills/fleet-health-report/references/html-report-template.md deleted file mode 100644 index 3fd73a9..0000000 --- a/skills/fleet-health-report/references/html-report-template.md +++ /dev/null @@ -1,312 +0,0 @@ -# HTML Fleet Health Report Template - -Produce a standalone `.html` file for the final fleet health snapshot. - -## File Requirements - -- Use one self-contained HTML file with ``, ``, - ``, and a responsive ``. -- Use inline CSS and only optional inline JavaScript. Do not reference external - fonts, scripts, CDNs, remote images, or stylesheets, so the file opens offline. -- HTML-escape all dynamic values from `nvfleetint` output. -- Do not embed raw credentials or a full raw-data dump. When showing only the - highest-ranked rows, state the displayed count and the full population count. -- Use the NVIDIA-aligned dark visual system defined below: a flat dark canvas, - panel containers, restrained semantic status colors, and quiet scannable - tables. Keep it responsive, accessible, and print-friendly. - -## Visual system - -The palette, tokens, and styling rules below are the source of truth for this -report; do not rely on any external design system or prior styling knowledge. -Because the output is standalone HTML, encode these conventions as inline CSS -variables and semantic class names instead of importing packages or external -assets. Render in dark mode by default; do not add a light-mode fallback unless -the user explicitly asks for one. The `node-rca-rcca` report shares this system — -keep them consistent so the two reports read as one product. - -Use this dark baseline palette: - -```css -:root { - --nv-green: #76b900; - --surface-base: #0b1117; - --surface-panel: #121a23; - --surface-sunken: #0f1620; - --surface-raised: #182230; - --text-primary: #f3f6fb; - --text-secondary: #a8b3c4; - --border-base: #2b3645; - --status-healthy: #7ce4aa; - --status-warning: #f7c566; - --status-critical: #ff8a80; - --status-info: #8bbcff; - --status-unknown: #c1cad8; - --status-healthy-bg: #063f2a; - --status-warning-bg: #4a3004; - --status-critical-bg: #4a1514; - --status-info-bg: #102f58; - --status-unknown-bg: #273241; -} -``` - -Apply these styling rules: - -- Treat `--nv-green` as the NVIDIA brand accent for masthead rules, links, focus - states, and small highlights. Do not use brand green to mean healthy; healthy - uses `--status-healthy`. -- Use a flat dark page canvas (`--surface-base`) with panel containers - (`--surface-panel`), sunken chart/table regions (`--surface-sunken`), and - subtle borders (`--border-base`). Avoid decorative gradients, radial washes, - bokeh, and nested card-on-card layouts. -- Use `NVIDIA Sans, Arial, Helvetica, sans-serif`; do not load remote fonts. Set - `color-scheme: dark`, keep text contrast high, and reserve `--text-secondary` - for supporting labels and notes, not primary metrics. -- Use semantic status colors only for meaningful thresholds: healthy/success/ - running = green, warning/degraded/pending/needs attention = amber, - critical/failed/offline = red, informational/in-progress = blue, - unknown/inactive/no data = gray. -- Pair status color with visible text labels and concise evidence; never rely on - color alone. -- Render status badges as solid pills with background + text color. Use badges - for status only, not for categories such as compute zone, node group, GPU - model, or component. -- Keep tables quiet and scannable on the dark canvas: no heavy filled table - backgrounds, subtle row dividers, uppercase header labels, and horizontal - scrolling on narrow screens. -- Match chart backgrounds to their containing panel, use secondary text for chart - labels, and use low-contrast grid lines. For standalone SVG/canvas charts, use - resolved hex or rgba values, not CSS variables inside SVG attributes. -- Choose chart type by the question: line charts for trends over time, horizontal - bars for ranked categories, stacked bars for composed counts, donut charts only - for 2-5 part-to-whole segments, and tables when exact values matter most. - -## Page Structure - -Use these sections in this order: - -1. Header with report title, overall status badge, node health score, and - generated timestamp. -2. Fleet-wide health / at a glance — report-derived node health score and - formula, the backend `healthPercentage` from `overview` when present, overall - status, node/GPU/CPU-core totals when present, healthy/degraded/unhealthy/ - unknown counts, active-alert totals by severity, and collection timestamp. -3. Fleet distribution and operational signals — concise health breakdowns by - compute zone and node group, GPU type/capacity, agent connectivity, firmware, - and verification, plus the `overview` fleet metrics (name, value with unit, - aggregation, last-updated) when returned. Omit unavailable metrics rather than - estimating them. -4. Trend direction — current versus previous equal windows, error totals, delta, - percentage or zero-baseline wording, direction, and exact time boundaries. -5. Issue concentration — alert/error distribution by component or type and the - share concentrated in the leading nodes, when fields support it. -6. Machines needing immediate attention — most-alerted/highest-risk nodes with - hostname, UUID or shortened UUID, health, critical and warning counts, - agent/firmware/verification signals, and concise evidence-based reasons. - -## Status Styling - -Map each status label to a semantic status color; use the badge classes in the -skeleton (`.ok`, `.warn`, `.bad`, `.info`, `.unknown`). Choose the class from the -label's actual value so a `Healthy` node never renders red: - -- `Healthy`, `Online`, `Resolved`, `Verified`, `Passed`: green - (`--status-healthy`). -- `Needs attention`, `Warning`, `Degraded`, `Detected`, `Pending`, `Unsupported`: - amber (`--status-warning`). -- `Critical`, `Unhealthy`, `Offline`, `Failed`, `Triggered`: red - (`--status-critical`). -- Informational labels: blue (`--status-info`). -- `Unknown`, inactive, `Other`, or no-data labels: gray (`--status-unknown`). - -## HTML Skeleton - -Copy and adapt this skeleton. Replace bracketed placeholders with report content, -and add charts and tables sized to the actual dataset within the visual system. - -```html - - - - - - Fleet Health Report — [status] on [date] - - - -
-

Fleet Health Report

-
Generated from read-only NVIDIA Fleet Intelligence data.
-
- Status: [status] - Node health score: [health_score]% - Backend health: [backend_health_percentage]% - Generated: [timestamp] -
-
-
-
-

Fleet-wide Health — At a Glance

-
[at_a_glance_stats]
-

[health_score_formula]

-
- -
-

Fleet Distribution and Operational Signals

- [distribution_breakdowns] -
- -
-

Trend Direction

-

[trend_summary]

- [trend_detail] -
- -
-

Issue Concentration

- [concentration_detail] -
- -
-

Machines Needing Immediate Attention

- - - [attention_rows] -
HostnameUUIDHealthCriticalWarningSignalsReason
-
-
- - -``` diff --git a/skills/fleet-health-report/references/html-theme.md b/skills/fleet-health-report/references/html-theme.md new file mode 100644 index 0000000..daeecbf --- /dev/null +++ b/skills/fleet-health-report/references/html-theme.md @@ -0,0 +1,89 @@ +# Shared HTML Report Theme + +Use this visual system for Fleet Intelligence HTML reports. Report sections and content belong in the calling skill, not this reference. + +## File contract + +- Produce one standalone semantic HTML file with UTF-8 and a responsive viewport. +- Use inline CSS and optional inline theme-control JavaScript only; load no remote fonts, scripts, stylesheets, images, or other assets. +- HTML-escape dynamic values. Pair every status color with visible text. +- Default to light mode and support a user-selectable dark mode by swapping CSS variables on `html[data-theme="dark"]`. + +## Palette + +```css +:root { + color-scheme: light; + --nv-green: #76b900; + --surface-canvas: #f7f7f7; + --surface-panel: #ffffff; + --surface-sunken: #f7f7f7; + --text-primary: #000000; + --text-secondary: #636363; + --border-base: rgba(0, 0, 0, 0.20); + --status-healthy: #265600; + --status-warning: #8d2600; + --status-critical: #961515; + --status-info: #0046a4; + --status-unknown: #4b4b4b; + --status-healthy-bg: #dafb7d; + --status-warning-bg: #fcde7b; + --status-critical-bg: #ffd7d7; + --status-info-bg: #cbf5ff; + --status-unknown-bg: #eeeeee; + --shadow-panel: 0 4px 6px rgba(0, 0, 0, 0.12); +} + +html[data-theme="dark"] { + color-scheme: dark; + --surface-canvas: #0c0c0c; + --surface-panel: #000000; + --surface-sunken: #161616; + --text-primary: #ffffff; + --text-secondary: #cccccc; + --border-base: rgba(255, 255, 255, 0.20); + --status-healthy: #76b900; + --status-warning: #ef9100; + --status-critical: #ff8181; + --status-info: #10b1fb; + --status-unknown: #eeeeee; + --status-healthy-bg: #142700; + --status-warning-bg: #441000; + --status-critical-bg: #4b0404; + --status-info-bg: #002050; + --status-unknown-bg: #4b4b4b; + --shadow-panel: none; +} +``` + +Use NVIDIA green only for brand accents, links, and focus states—not as the generic healthy color. Use local `NVIDIA Sans, Arial, Helvetica, sans-serif` with a 14px base size and 1.5 line height. + +## Layout and components + +- Canvas: `--surface-canvas`; centered content, maximum width 1180px, 24px desktop gutters and 12px mobile gutters. +- Product bar: 48px high, `--surface-panel`, bottom border, product name left, compact accessible Light/Dark control right. Hide it when printing. +- Page header: floating panel with 16px corners, 24px padding, subtle border and `--shadow-panel`. +- Report panels: white/light or black/dark `--surface-panel`, 16px corners, 24px padding, 16px vertical gap. Use one panel per report section. +- Metric cards: responsive grid, `--surface-sunken`, 8px corners, subtle border; uppercase 12px label and 24px bold value. +- Tables: full width, quiet background, 14px text, 8px block/12px inline cell padding, semibold header with a 2px divider, 1px row dividers, wrapping long values. Allow horizontal scrolling on narrow screens. +- Code: local monospace, `--surface-sunken`, 4px corners, subtle border, wrapping. +- Drill-downs: use `
` with an 8px bordered sunken container and a bold ``; keep expanded content compact. +- Charts: use panel backgrounds, secondary labels, low-contrast grid lines, and resolved colors for SVG/canvas attributes. Prefer horizontal bars for ranked categories and tables when exact values matter. + +## Status labels + +Use compact solid badges with 4px corners, 12px bold text, and a 1px semantic border. Use them only for statuses, not zone/group/model/component categories. + +| Class | Meaning | Colors | +| --- | --- | --- | +| `ok` | Confirmed, Healthy, Online, Resolved, Verified, Passed | `--status-healthy*` | +| `warn` | Likely, Warning, Degraded, Detected, Pending, Unsupported, Unverified | `--status-warning*` | +| `bad` | Not confirmed, Critical, Unhealthy, Offline, Failed, Triggered | `--status-critical*` | +| `info` | Informational or in progress | `--status-info*` | +| `unknown` | Unknown, inactive, Other, or no data | `--status-unknown*` | + +## Responsive and print behavior + +Below 720px, reduce panel padding to 16px, panel radius to 12px, heading size, and table cell padding. Preserve table scrolling and readable status labels. + +For print, force a white canvas with dark text, remove shadows, hide the product bar/theme control, avoid breaking a short panel across pages, and keep semantic labels legible without relying on background color. diff --git a/skills/fleet-health-report/references/workspace.md b/skills/fleet-health-report/references/workspace.md new file mode 100644 index 0000000..042f632 --- /dev/null +++ b/skills/fleet-health-report/references/workspace.md @@ -0,0 +1,32 @@ +# Shared Report Workspace + +Use this workflow when a report needs saved API responses. + +## Capture + +Create one unpredictable private directory and preserve its exact path: + +```bash +umask 077 +work=$(mktemp -d "/tmp/nvfleet-report.XXXXXXXX") || exit 1 +printf 'scratch: %s\n' "$work" +``` + +Before later access or cleanup require `[ -d "$work" ]`, `[ ! -L "$work" ]`, and `[ -O "$work" ]`. Write each response to a distinct file. Fetch costly data once, then inspect and parse the saved response; do not share a target between concurrent commands. + +On PowerShell, use a GUID directory under system temp, restrict its ACL, reject reparse points, and retain the exact literal path. + +## Write and validate + +Write the final HTML outside the scratch directory. Sanitize a generated filename to `A-Za-z0-9._-`, or honor an explicit output path. Compose the document once, then validate the required section IDs supplied by the calling skill, closing ``, and absence of unresolved `[placeholder]` tokens. + +## Cleanup + +After successful validation, delete only direct files/links and the empty exact directory: + +```bash +find "$work" -mindepth 1 -maxdepth 1 \( -type f -o -type l \) -delete && + rmdir -- "$work" +``` + +Retain evidence after validation failure. Do not recursively delete, use globs, reconstruct the path, or remove unknown subdirectories. PowerShell cleanup uses exact literal paths without recursion. diff --git a/skills/node-rca-rcca/SKILL.md b/skills/node-rca-rcca/SKILL.md index acbfaa6..0a47912 100644 --- a/skills/node-rca-rcca/SKILL.md +++ b/skills/node-rca-rcca/SKILL.md @@ -1,448 +1,94 @@ --- name: node-rca-rcca -description: Investigate one degraded or unhealthy NVIDIA Fleet Intelligence node and generate an evidence-backed HTML RCA/RCCA from live nvfleetint data. Use for node incident analysis, health-transition explanations, root-cause analysis, corrective actions, or post-incident reports. Do not use for fleet-wide status reporting. +description: Investigate one NVIDIA Fleet Intelligence node and generate an evidence-backed HTML RCA/RCCA from live current and historical alerts plus authoritative corrective-action research. Use for node incident analysis, root-cause analysis, corrective actions, or post-incident reports. --- # Node RCA/RCCA -Produce a structured, evidence-backed RCA/RCCA document for one degraded or -unhealthy node: gather live `nvfleetint` evidence about its health transitions, -recurring events, alert history, and affected components, then synthesize a root -cause and corrective/preventive actions. Compose each report to fit the evidence -on hand, within the required sections and visual system below — no fixed renderer -or deterministic generation script. - -## When to use - -For a **single-node investigation** — the user names one node (UUID or hostname) -and wants to know why it degraded and what to do. For a fleet-wide snapshot across -many nodes, use the `fleet-health-report` skill instead. - -## Operating rules - -- **Read-only.** Use read-only Fleet Intelligence commands; never run - write/delete/tag commands. -- **Live evidence only.** Run fresh `nvfleetint` queries this invocation. Never - substitute examples, fixtures, cached output, prior reports, or invented values - (UUIDs, hostnames, timestamps, causes, remediation). Separate observed facts - from inference; label uncertain conclusions `Likely` or `Not confirmed`. -- **Fenced enrichment only.** Looking up plain-language explanations of raw codes - (XID numbers) and firmware/integrity reason strings is the *only* permitted - non-`nvfleetint` input, strictly bounded per "Enrich codes and check-reasons": - it may inform only the RCCA actions and a **Reference** subsection, never the - timeline, impact, root cause, or confidence. -- **JSON is the source.** Prefer `--output json`; parse stdout as JSON only after - a zero exit status. Preserve exact timestamps and time zones; state when one is - absent or ambiguous rather than guessing. Mark unavailable fields `N/A`. -- **No secrets.** Never expose API keys, credentials, env vars, auth headers, - or raw config contents in commands, logs, the report, or chat. -- **One artifact.** The only file left in the workspace is the final report. Keep - envelopes, parsed JSON, and scratch in memory or OS temp files that are cleaned - up before finishing. -- **Fail loud.** If access, auth, or API failures block evidence collection, do - not publish a report — state the exact command attempted and what is missing. - -## Running the CLI - -Use the installed `nvfleetint` binary with a suitable `--timeout` (e.g. -`--timeout 60s`); if it differs from the invocations below, check -`nvfleetint --help`. - -## Prerequisites - -- Run commands through the harness's local command-execution capability. The - examples use POSIX shell syntax; on Windows, use equivalent PowerShell while - preserving the `nvfleetint` arguments and evidence rules. -- `nvfleetint` on `PATH` — confirm with `command -v nvfleetint` on POSIX or - `Get-Command nvfleetint` in PowerShell. -- A structured JSON processor is available. The examples use `jq`; an equivalent - parser is acceptable, but grepping human-readable table output is not. -- An authenticated session — confirm with `nvfleetint auth status` (diagnostic: - exits `0` and prints a `Connection:` line rather than failing on a bad key). - **Require `Connection: ok`.** Treat `Connection: unauthorized`/`unauthenticated` - or `error: ...` — and on any data command, exit code `77`, HTTP 401/403, or a - JSON `api_error` — as an auth failure, not an empty result. On any of these, ask - the user to run `nvfleetint auth add --api-key ` (never ask - them to paste a key into chat) and stop. -- Credentials live in named profiles. Run `nvfleetint auth list` first. If the - user named an environment or tenant, use that profile. Otherwise, if more than - one profile exists, **ask the user which one owns the node and wait for their - answer** — do not fall back to the current profile (the one marked `*`) or the - first one listed. Querying the wrong tenant yields either no such node or, if - the hostname collides, evidence from a different machine entirely. Once the - profile is known, pass the same `--profile ` to every command below, - including `auth status`, so all the evidence comes from the fleet that owns - the node. -- A target node — a UUID, or a hostname/partial hostname to resolve. - -## Collect live evidence - -Replace `` throughout. Default the window to the last 7 days -(`[T-7d, T)`, `T` = UTC collection time) unless the user specifies another -horizon, and record the exact RFC3339 boundaries. - -**Fix the window once, then use it everywhere.** Pin three values up front and -substitute them into every query below — ``, ``, and -``, the same span as a duration (`168h` for the 7-day default). The -literal `168h` and `-v-7d` in the commands below are the *default* spelling: when -the user asks for a different horizon, recompute all three and change `node -health`, `event list`, `event buckets`, and `report error` together. A report -whose health window and event window disagree is wrong even if each query -succeeded. - -**One rule governs speed and context: project large payloads at the source with -`jq`, and keep only anchor fields, non-resolved alerts, and aggregate counts in -context.** Raw `node describe` / `alert timeline` / `alert describe` payloads are -re-read on every later turn — that prefill is the single biggest time sink here. -Filtering governs only what enters context, never what you retain: fetch full -payloads to temp files (below) so you can widen a `jq` projection against the same -file without re-fetching. - -**Batching plan** (minimize turns): - -- **Batch A (serial, first):** resolve the UUID (step 1) — everything depends on it. -- **Batch B (parallel, one turn):** steps 2–5 together — `node describe`, - `node health`, `event list`, `event buckets`, and both `alert timeline` queries - (full and `--active`) are independent once the UUID is known. Do **not** include - `alert list` (a step-5 fallback). -- **Fast-path gate:** after validating Batch B, decide whether the incident is - trivial and skip most of Batch C (see "Fast path"). -- **Batch C (parallel, after B):** the capped `alert describe` calls (step 6). - -Fold these into the batches rather than spending separate turns: - -- **Inline the window** in `node health` with shell substitution - (`--start "$(date -u -v-7d +%Y-%m-%dT%H:%M:%SZ)" --end "$(date -u +%Y-%m-%dT%H:%M:%SZ)"` - on macOS; `date -u -d '7 days ago'` on Linux). Still record the boundaries used. -- **Read the report template early** — - [`references/html-report-template.md`](references/html-report-template.md), a - local file independent of evidence — during the prereq checks or Batch B. It - contains the complete document skeleton, including the invariant ``/CSS. - -Before creating any scratch files, read and follow -[`references/scratch-workspace.md`](references/scratch-workspace.md). It defines -the exclusively created, validated scratch path and cleanup rules for both -POSIX and PowerShell environments. - -### 1. Resolve the target node - -If given only a hostname/partial hostname, resolve it and confirm when multiple -match — use `--all` (not just the first page) before deciding: +Investigate one node and produce an evidence-backed offline HTML RCA/RCCA. Read the [CLI contract](references/cli-contract.md), [HTML theme](references/html-theme.md), and [workspace guide](references/workspace.md) before collecting data. -```bash -nvfleetint node list --hostname --view detail --all --output json --timeout 60s -``` +## Workflow -### 2. Capture current node state (anchor) +### 1. Resolve the inputs and profile -Current health, agent status, node group, compute zone, GPU type/count, -verification (integrity) check + reason, firmware check, component health counts. -Read verification state from `integrityCheck` / `integrityCheckReason` / -`lastIntegrityCheckTS` ("verification" is a display term only). Fetch once to a -temp file, then project anchor fields: +Require a hostname or node UUID and a profile. Ask a concise clarification when either is missing or ambiguous. ```bash -work="" # exact path created using scratch-workspace.md -nvfleetint node describe --output json --timeout 60s > "$work/node-describe.json" -jq '{nodeUUID, hostname, healthStatus, nodeGroup, computeZone, - gpuType, gpuCount, gpuArchitecture: .resources.gpuInfo.architecture, - agentStatus, agentVersion, gpuDriverVersion, kernelVersion, - integrityCheck, integrityCheckReason, integrityCheckExtraInfo, firmwareCheck, - healthyComponentCount, degradedComponentCount, unhealthyComponentCount, - lastIntegrityCheckTS, lastUpdatedTS, enrolledAt, tags}' "$work/node-describe.json" +nvfleetint auth list --output json +nvfleetint auth status --profile --output json ``` -Widen the projection against the **same file** when a suspected component needs it -(e.g. `.resources.nicInfo` for `network-ethernet` — the hardware detail lives -under `resources`, alongside `gpuInfo`; or top-level `gpuFirmwareVersions` for a -firmware-check failure) — never re-invoke `node describe`; the payload is already -on disk. - -### 3. Capture node health transitions - -```bash -nvfleetint node health --start --end --output json --timeout 60s -``` +Require `connection` equal to `ok`. Pass the same explicit `--profile ` to every API-backed command. -Both timestamps required. There is no `items`/`pagination` envelope: read -per-interval segments from `machineStatus` (`status`, `startTime`, `endTime`) and -the aggregate `healthSummary`. Derive transitions from segment boundaries — last -known-normal state, first degraded/unhealthy interval, and any flapping. +### 2. Resolve exactly one node -### 4. Capture recurring events +For a hostname, search all identity pages and require one exact match. Ask the user to choose when a partial name returns multiple candidates. ```bash -nvfleetint event list --node --window 168h --all --output json --timeout 60s -nvfleetint event buckets --node --window 168h --output json --timeout 60s +nvfleetint node list --hostname --view basic --all \ + --profile --output json ``` -`event list` enumerates events; `event buckets` aggregates them into time buckets -to reveal recurrence/bursts (`--max-buckets` to tune). A time range is required -(`--window` or `--start`/`--end`, consistent with the health window). Add -`--component ` to focus a suspected component. Project the list to -aggregates in context: +For a UUID, or after resolving a hostname, verify and capture the node once: ```bash -jq '{total: (.items|length), pagination, - by_component: (.items|group_by(.component)|map({component: .[0].component, count: length}))}' +nvfleetint node describe --profile --output json ``` -### 5. Capture alert history and active alerts +Use the saved description for hostname, health, placement, GPU, agent, integrity, firmware, and component context. -```bash -nvfleetint alert timeline --node --active --all --output json --timeout 60s -nvfleetint alert timeline --node --all --output json --timeout 60s -``` +### 3. Collect current and historical alerts -`--active` isolates what is still firing (usually small — pipe directly). The full -timeline can be large on a flapping node; reduce it to still-firing alerts plus -per-component counts, keeping `pagination` for the completeness checks: +Fetch current alerts and complete historical alerts: ```bash -jq '{firing: [.items[] | select(.alertStatus == "Critical" or .alertStatus == "Warning")], - total: (.items | length), - by_component: (.items | group_by(.component) | map({component: .[0].component, count: length})), - pagination}' +nvfleetint alert node --without-psirt --all \ + --profile --output json +nvfleetint alert node --view historical --without-psirt --all \ + --profile --output json ``` -**`alertStatus` on the timeline is not an alert state.** It carries -`Critical`/`Warning` — a *severity* — while the alert is active, and -`Detected`/`Resolved` once it is inactive (from the audit history). So -`select(.alertStatus != "Resolved")` does **not** mean "active": it keeps -`Detected`, which is an inactive value, and inflates the count. Treat only -`Critical`/`Warning` as still firing here, and let the `--active` query be the -authority for the active-alert count the report headlines. - -For a large timeline, fetch to a temp file first (as in step 6) so a `jq` typo -can't trigger a re-fetch. **Fallback only** — when the timeline lacks fields you -need (state, severity, message) for the alerts that matter, add a node-scoped, -**narrowed** `alert list` (never `--all`); it supports -`--severity Critical|Warning`, `--state Detected|Triggered|Resolved`, -`--component `: +Describe every unique current alert: ```bash -nvfleetint alert list --node --state Triggered --output json --timeout 60s +nvfleetint alert describe --node --profile --output json ``` -### Fast path (trivial incident) +Run at most four describe calls concurrently. Use each description's timeline, messages, errors, incidents, and suggested actions to identify the exact issue; treat missing optional fields as unavailable evidence. -Most degraded nodes have a single dominant cause and no event noise. **After -validating Batch B**, if all of these hold: +Treat the default node view as current active alerts. Aggregate current and historical alerts by component ID/display name and status, deduplicating the same `alertUuid` across both sets. For each current alert, count prior historical rows with the same component ID after excluding its own `alertUuid`, and record the most recent prior occurrence. Empty alert sets are valid evidence. -- **≤ 1 active alert** (from `alert timeline --active`), and -- **0 events** in the window (`event list` total `0`, `event buckets` empty), and -- **no flapping** — a single `machineStatus` segment over the window (no - intra-window transition), +### 4. Determine the root cause -then take the fast path: **skip the multi-alert selection and huge-timeline -scanning machinery of step 6.** Describe just the single active alert (if any) to -get its reason string and onset — one `alert describe` — and go straight to -analysis and the report. Root cause can still be `Confirmed` when the anchor -(`node describe` integrity/component fields) and that one alert independently name -the same component and cause. With **0 active alerts**, skip Batch C entirely and -root-cause from the anchor and health segments alone. Note in the report that the -resolved-alert history was summarized from the timeline aggregate, not described -individually. Otherwise (multiple active alerts, event bursts, or flapping), run -the full step 6. +Validate every response with the CLI contract before analysis. Correlate node state, current alert descriptions, and historical alerts by component and time. State: -### 6. Describe the alerts that matter +- observed symptoms and impact; +- the most specific supported root cause; +- confidence as `Confirmed`, `Likely`, or `Not confirmed`; +- competing explanations and missing evidence when they affect the conclusion. -Pull full detail only for alerts that change the analysis: the earliest relevant -alert, each active alert, and the most-repeated alert on the suspected component. -**Cap at ~3–5 UUIDs** — describing every alert on a flapping node is the largest -open-ended time sink and rarely adds signal past the first few. Select UUIDs from -Batch B, then run the `alert describe` calls as one parallel batch (`--node` -required). +Do not promote correlation to causation. If evidence is insufficient, report the root cause as not confirmed and identify the next evidence needed. -**Fetch once to a temp file, then parse the file — never pipe the CLI into `jq`.** -A response can be hundreds of KB, so every fetch is expensive: +### 5. Research corrective actions -```bash -work="" # exact path created using scratch-workspace.md -out="$work/alert-.json" -nvfleetint alert describe --node --output json --timeout 60s > "$out" -jq '' "$out" # re-run jq against the file as many times as needed -``` +After forming the evidence-based RCA, search the web using only observed generic component names, error codes, firmware/driver versions, and root-cause terms. Never include hostname, node UUID, profile, tenant, or customer data in a query. -- **Never** `alert describe ... | jq ...` — a `jq` typo silently re-runs the fetch. -- **Never** recover from a `jq` error by re-invoking the CLI; a parse failure means - your `jq` was wrong — fix the expression and re-run it against the same file. -- Use a plain ASCII pipe `|`. If the schema is unknown, learn it in the **same** - pass: the decisive values often live inside `timeline[]`, not the top-level - object, so dump keys plus a sample (`jq '{keys: keys, sample: .timeline[0]}'`) or - write one broad expression (first + last timeline event, `severity_changed` - event, `[.timeline[].message] | unique`). **Stop as soon as the reason string - and onset are in hand** — do not re-scan a multi-MB timeline to confirm what the - list/timeline already established. Extract all aggregates in a single `jq` pass. +Prefer official NVIDIA documentation, release notes, support articles, and knowledge-base material. Use other primary vendor documentation only when no relevant NVIDIA source exists. Cite the source title and URL beside each supported recommendation. -If more than ~5 alerts look relevant, describe the capped set and note in the -report which additional UUIDs were summarized rather than described, so the -omission is explicit. +Turn the research into containment, corrective, preventive, and validation actions. Keep sourced guidance distinct from fleet evidence and mark any environment-dependent recommendation for operator confirmation. -### 7. Optional blast-radius context +### 6. Build and deliver -When the user asks whether the failure is isolated or fleet-wide: +Apply the shared HTML theme and workspace workflow. Summarize saved JSON rather than embedding raw payloads. Use these sections: -```bash -nvfleetint report error --view list --group-by node --window 168h --all --output json --timeout 60s -``` +1. Executive Summary: node, collection time, impact, root cause, and confidence. +2. Node Details: relevant node metadata. +3. Alert Evidence: first show aggregate current and historical counts grouped by component and status, then show a collapsed `
` breakdown for every current alert with its described issue, timeline evidence, status, component, timing, prior occurrence count, and most recent prior occurrence. +4. Root Cause Analysis: reasoning, competing explanations, and evidence gaps. +5. Corrective Action Plan: containment, corrective, preventive, and validation actions. +6. References: cited corrective-action sources. +7. Assumptions and Unknowns: assumptions and information still required. -## Prove completeness - -Validate every response before analysis: - -1. Require exit status `0`, nonempty stdout, valid JSON, no top-level - `error`/`api_error`. -2. For every `--all` query, require the merged collection `items` and a - `pagination` with `hasMore` false and `pagesFetched` ≥ `1`. `--all` always - normalizes to `{items, pagination}`, whatever the backend called the array — - `alert timeline` returns `alerts` and `report error --view list` returns - `nodes` on a single page, but both become `items` under `--all`. Do not look - for the backend's key on an `--all` response; it is absent, and treating that - as a failed check would reject a complete dataset. (Without `--all`, the - payload is the raw backend body and the backend's own key applies.) - When `pagination.total` is present and nonzero, require - the collection length to equal it; when `total` is `0` or absent, rely on - `hasMore` false (do not flag a nonzero collection with `total` `0` as a - mismatch). -3. An empty result is valid only when genuinely empty (`total` `0`, `hasMore` - false) — a node with no alerts/events is a real finding. But a node `describe` - cannot find is an error: re-check the UUID. -4. Retry a completeness mismatch once; if it persists, stop and report the dataset - as incomplete rather than analyzing a partial set. -5. Keep every validated dataset through report generation. A top-N *table* is - fine; collecting or analyzing only a top-N *subset* is not. - -## Analyze the evidence - -- **Timeline** from health transitions, event buckets/list, alert timeline, and - alert detail. Preserve timestamps/zones. Identify last known-normal, first - symptom, current state, and whether alerts are active or resolved. -- **Impact** from evidence only: health state, affected component, node group, - compute zone, GPU type/count, agent status, integrity/firmware checks, alert - severity. If operational impact is not in the evidence, say it was not confirmed. -- **Root cause** — name the most specific cause the evidence supports, with a - confidence level: - - `Confirmed` — evidence directly identifies the failed component and cause. - - `Likely` — evidence points to a probable cause but does not prove it. - - `Not confirmed` — evidence supports only a symptom; state the strongest - hypothesis and what would confirm it. -- **Contributing factors** only when evidence-backed (repeated alert transitions, - flapping, stale/offline agent, failed integrity/firmware check, recurring - component event). -- **RCCA** items: containment, corrective, preventive, validation. Owners/due - dates only when provided; otherwise `TBD`. -- **Post-remediation validation** using read-only commands, usually - `node describe` and `alert timeline --node --active`, with the - expected healthy result for each. -- Never manufacture timestamps, hostnames, causes, remediation, or utilization. - -## Enrich codes and check-reasons (optional) - -Optional, additive, tightly fenced. Inputs are only the **raw codes/reason strings -already in the telemetry** (GPU `XID` numbers, `integrityCheckReason` / -firmware-check reason strings). Purpose: translate opaque tokens into -plain-language explanations that make corrective actions actionable. It does not -change how root cause, confidence, timeline, or impact are derived. - -- **Generic tokens only.** Query the bare code/string (e.g. "GPU XID 79", - "double-bit ECC error"). Never send node UUIDs, hostnames, IPs, serial numbers, - or any fleet identifier to an external source. -- **Authoritative first:** official NVIDIA docs (XID reference, DCGM/GPU health) → - internal KB → general web. Capture source title + URL. -- **Confine it** to (a) the corrective/preventive RCCA actions and (b) a - **Reference** subsection listing each code/reason, its meaning, and its source. - It must not alter the timeline, impact, root cause, or confidence. -- **Attribute explicitly**; never present an external explanation as - telemetry-derived, and never let it raise the confidence tier. -- **Bake it in** as static, attributed prose so the file still opens offline. -- **Omit when unsure** — leave the token as-is and omit the Reference subsection - entirely when no enrichment was performed. - -## Build the RCA/RCCA document - -Standalone `.html`, named `node-rca-rcca-.html` unless the -user gives a path. Follow the skeleton and status styling in -[`references/html-report-template.md`](references/html-report-template.md). - -**Write the whole file in one shot.** The document chrome — ``, -``, the full `` block is **invariant** — copy -it byte-for-byte every run, and do not restyle, reorder, or "improve" the CSS. -The palette/tokens described above are exactly what this CSS encodes. - -The `` is a fixed, generic string; the per-node title -(`RCA/RCCA: [node] [health] on [date]`) lives in the `<h1>` masthead. - -Only the body content varies: replace bracketed placeholders with report content -and choose each `[*_class]` from the Status Styling map above. - -```html -<!doctype html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <title>NVIDIA Fleet Intelligence — Node RCA/RCCA - - - -
-

RCA/RCCA: [node] [health] on [date]

-
Generated from read-only NVIDIA Fleet Intelligence evidence.
-
- Health: [health] - Root Cause: [confidence] - Generated: [timestamp] -
-
-
-
-

Executive Summary

-

[summary]

-
- -
-

Node Details

- - - - - - - - - - - - -
Node UUID[node_uuid]
Hostname[hostname]
Health[health]
Agent status[agent_status]
Node group[node_group]
Compute zone[compute_zone]
GPU type/count[gpu_type] / [gpu_count]
Verification (integrity) check[integrity_check]
Firmware check[firmware_check]
-
- -
-

Impact

-
    [impact_items]
-
- -
-

Incident Timeline

- - - [timeline_rows] -
TimeSourceEventEvidence
-
- -
-

Root Cause

-

Confidence: [confidence]

-

Cause: [cause]

-

Evidence

-
    [root_cause_evidence_items]
-

Reasoning

-

[reasoning]

-
- -
-

Contributing Factors

-
    [contributing_factor_items]
-
- -
-

Corrective and Preventive Actions

- - - [action_rows] -
TypeActionOwnerDueStatusValidation
-
- -
-

Validation Plan

-
    [validation_items]
-
- - -
-

Reference

-

Plain-language explanations of codes and check-reasons from external sources, provided for attribution only. These are not telemetry-derived facts and did not affect the timeline, impact, root cause, or confidence.

- - - [reference_rows] -
Code / ReasonMeaningSource
-
- -
-

Evidence Appendix

- - - [evidence_rows] -
CommandPurposeResult Summary
-
- -
-

Assumptions and Unknowns

-
    [unknown_items]
-
-
- - -``` diff --git a/skills/node-rca-rcca/references/html-theme.md b/skills/node-rca-rcca/references/html-theme.md new file mode 100644 index 0000000..daeecbf --- /dev/null +++ b/skills/node-rca-rcca/references/html-theme.md @@ -0,0 +1,89 @@ +# Shared HTML Report Theme + +Use this visual system for Fleet Intelligence HTML reports. Report sections and content belong in the calling skill, not this reference. + +## File contract + +- Produce one standalone semantic HTML file with UTF-8 and a responsive viewport. +- Use inline CSS and optional inline theme-control JavaScript only; load no remote fonts, scripts, stylesheets, images, or other assets. +- HTML-escape dynamic values. Pair every status color with visible text. +- Default to light mode and support a user-selectable dark mode by swapping CSS variables on `html[data-theme="dark"]`. + +## Palette + +```css +:root { + color-scheme: light; + --nv-green: #76b900; + --surface-canvas: #f7f7f7; + --surface-panel: #ffffff; + --surface-sunken: #f7f7f7; + --text-primary: #000000; + --text-secondary: #636363; + --border-base: rgba(0, 0, 0, 0.20); + --status-healthy: #265600; + --status-warning: #8d2600; + --status-critical: #961515; + --status-info: #0046a4; + --status-unknown: #4b4b4b; + --status-healthy-bg: #dafb7d; + --status-warning-bg: #fcde7b; + --status-critical-bg: #ffd7d7; + --status-info-bg: #cbf5ff; + --status-unknown-bg: #eeeeee; + --shadow-panel: 0 4px 6px rgba(0, 0, 0, 0.12); +} + +html[data-theme="dark"] { + color-scheme: dark; + --surface-canvas: #0c0c0c; + --surface-panel: #000000; + --surface-sunken: #161616; + --text-primary: #ffffff; + --text-secondary: #cccccc; + --border-base: rgba(255, 255, 255, 0.20); + --status-healthy: #76b900; + --status-warning: #ef9100; + --status-critical: #ff8181; + --status-info: #10b1fb; + --status-unknown: #eeeeee; + --status-healthy-bg: #142700; + --status-warning-bg: #441000; + --status-critical-bg: #4b0404; + --status-info-bg: #002050; + --status-unknown-bg: #4b4b4b; + --shadow-panel: none; +} +``` + +Use NVIDIA green only for brand accents, links, and focus states—not as the generic healthy color. Use local `NVIDIA Sans, Arial, Helvetica, sans-serif` with a 14px base size and 1.5 line height. + +## Layout and components + +- Canvas: `--surface-canvas`; centered content, maximum width 1180px, 24px desktop gutters and 12px mobile gutters. +- Product bar: 48px high, `--surface-panel`, bottom border, product name left, compact accessible Light/Dark control right. Hide it when printing. +- Page header: floating panel with 16px corners, 24px padding, subtle border and `--shadow-panel`. +- Report panels: white/light or black/dark `--surface-panel`, 16px corners, 24px padding, 16px vertical gap. Use one panel per report section. +- Metric cards: responsive grid, `--surface-sunken`, 8px corners, subtle border; uppercase 12px label and 24px bold value. +- Tables: full width, quiet background, 14px text, 8px block/12px inline cell padding, semibold header with a 2px divider, 1px row dividers, wrapping long values. Allow horizontal scrolling on narrow screens. +- Code: local monospace, `--surface-sunken`, 4px corners, subtle border, wrapping. +- Drill-downs: use `
` with an 8px bordered sunken container and a bold ``; keep expanded content compact. +- Charts: use panel backgrounds, secondary labels, low-contrast grid lines, and resolved colors for SVG/canvas attributes. Prefer horizontal bars for ranked categories and tables when exact values matter. + +## Status labels + +Use compact solid badges with 4px corners, 12px bold text, and a 1px semantic border. Use them only for statuses, not zone/group/model/component categories. + +| Class | Meaning | Colors | +| --- | --- | --- | +| `ok` | Confirmed, Healthy, Online, Resolved, Verified, Passed | `--status-healthy*` | +| `warn` | Likely, Warning, Degraded, Detected, Pending, Unsupported, Unverified | `--status-warning*` | +| `bad` | Not confirmed, Critical, Unhealthy, Offline, Failed, Triggered | `--status-critical*` | +| `info` | Informational or in progress | `--status-info*` | +| `unknown` | Unknown, inactive, Other, or no data | `--status-unknown*` | + +## Responsive and print behavior + +Below 720px, reduce panel padding to 16px, panel radius to 12px, heading size, and table cell padding. Preserve table scrolling and readable status labels. + +For print, force a white canvas with dark text, remove shadows, hide the product bar/theme control, avoid breaking a short panel across pages, and keep semantic labels legible without relying on background color. diff --git a/skills/node-rca-rcca/references/scratch-workspace.md b/skills/node-rca-rcca/references/scratch-workspace.md deleted file mode 100644 index 3920a46..0000000 --- a/skills/node-rca-rcca/references/scratch-workspace.md +++ /dev/null @@ -1,40 +0,0 @@ -# Scratch workspace - -Create one unpredictable, private temporary directory and keep all evidence -inside it. Preserve the exact path returned at creation; never reconstruct it -from a UUID or hostname. - -## POSIX shell - -```bash -node_uuid="" -umask 077 -work=$(mktemp -d "/tmp/node-rca-$node_uuid.XXXXXXXX") || exit 1 -printf 'scratch: %s\n' "$work" -``` - -Before each later read, write, or cleanup, set `work` to that exact printed path -and require `[ -d "$work" ]`, `[ ! -L "$work" ]`, and `[ -O "$work" ]`. -Keep files directly inside it. After the report passes validation, delete only -those files and remove the empty directory: - -```bash -find "$work" -mindepth 1 -maxdepth 1 \( -type f -o -type l \) -delete -rmdir -- "$work" -``` - -Never use recursive deletion, a wildcard, or a reconstructed path. If an -unexpected subdirectory exists, cleanup must fail and retain the evidence. - -## PowerShell - -```powershell -$Work = Join-Path ([System.IO.Path]::GetTempPath()) ("node-rca-" + [guid]::NewGuid().ToString("N")) -New-Item -ItemType Directory -Path $Work -ErrorAction Stop | Out-Null -Write-Output "scratch: $Work" -``` - -Restrict the directory ACL to the current user before writing evidence. For -later commands, reuse the exact printed path, reject reparse points, and verify -ownership. Delete known files with exact `-LiteralPath` values, then remove the -empty directory without `-Recurse`, `-Force`, or wildcards. diff --git a/skills/node-rca-rcca/references/workspace.md b/skills/node-rca-rcca/references/workspace.md new file mode 100644 index 0000000..042f632 --- /dev/null +++ b/skills/node-rca-rcca/references/workspace.md @@ -0,0 +1,32 @@ +# Shared Report Workspace + +Use this workflow when a report needs saved API responses. + +## Capture + +Create one unpredictable private directory and preserve its exact path: + +```bash +umask 077 +work=$(mktemp -d "/tmp/nvfleet-report.XXXXXXXX") || exit 1 +printf 'scratch: %s\n' "$work" +``` + +Before later access or cleanup require `[ -d "$work" ]`, `[ ! -L "$work" ]`, and `[ -O "$work" ]`. Write each response to a distinct file. Fetch costly data once, then inspect and parse the saved response; do not share a target between concurrent commands. + +On PowerShell, use a GUID directory under system temp, restrict its ACL, reject reparse points, and retain the exact literal path. + +## Write and validate + +Write the final HTML outside the scratch directory. Sanitize a generated filename to `A-Za-z0-9._-`, or honor an explicit output path. Compose the document once, then validate the required section IDs supplied by the calling skill, closing ``, and absence of unresolved `[placeholder]` tokens. + +## Cleanup + +After successful validation, delete only direct files/links and the empty exact directory: + +```bash +find "$work" -mindepth 1 -maxdepth 1 \( -type f -o -type l \) -delete && + rmdir -- "$work" +``` + +Retain evidence after validation failure. Do not recursively delete, use globs, reconstruct the path, or remove unknown subdirectories. PowerShell cleanup uses exact literal paths without recursion. diff --git a/skills/nvfleetint/SKILL.md b/skills/nvfleetint/SKILL.md index c086d9e..33626fe 100644 --- a/skills/nvfleetint/SKILL.md +++ b/skills/nvfleetint/SKILL.md @@ -3,248 +3,105 @@ name: nvfleetint description: Query NVIDIA Fleet Intelligence with the nvfleetint CLI. Use for ad hoc questions about fleets, nodes, GPUs, node groups, compute zones, alerts, agent health, firmware, verification, inventory, errors, or authentication. For a fleet-wide HTML snapshot use fleet-health-report; for a single-node RCA/RCCA use node-rca-rcca. --- -# Answering fleet questions with nvfleetint +# Query Fleet Intelligence -**Fleet Intelligence** is the NVIDIA backend product for understanding a GPU fleet — its inventory, health, alerts, and reports. `nvfleetint` is the command-line client for that backend's customer API. +Use live `nvfleetint` JSON; never answer fleet-state questions from memory. Read [`references/cli-contract.md`](references/cli-contract.md) before querying. Read [`references/auth.md`](references/auth.md) only for setup/profile work. -Treat `nvfleetint` as **your data-access tool**: it's how you read live state from the Fleet Intelligence backend. Whenever the user asks about the state of their fleet, don't answer from memory or guess — run the right `nvfleetint` command(s), parse the output, and answer their actual question in plain language. The data is live, so every answer should be grounded in a command you just ran. +## Method -## Runtime requirements +1. Run the smallest server-filtered query that answers the question. +2. Use `--output json`; give the user prose, not command dumps. +3. For counts, use `--page-size 1` and read `.total`. For identities, add `--view basic`. Use `--all` only when every row is required. +4. Lead with the answer; add a small table only for comparisons/listings. +5. Never infer absent fields or mistake auth failure for no results. -- Run `nvfleetint` through the harness's local command-execution capability. -- Require `nvfleetint` on `PATH` and authenticated network access to the Fleet - Intelligence backend. -- Treat shell snippets as POSIX examples. On Windows, use equivalent PowerShell - commands without changing the `nvfleetint` arguments or evidence rules. -- Use `jq` only when it is available; otherwise parse the JSON with another - local structured-data tool rather than grepping table output. +## Choose a command -## How to work - -The user wants an *answer*, not a command dump. So: - -1. **Run commands with `--output json`** (or the command's JSON `--format`). JSON parses reliably; table output is for humans and is easy to misread. You read the JSON, the user gets prose. -2. **Summarize findings in plain language.** Lead with the answer ("3 of your 48 nodes are unhealthy"), then the supporting detail. Show a small table only when the user is comparing items or explicitly wants a listing. -3. **Filter at the source.** These commands have rich filter flags (`--health`, `--severity`, `--gpu-type`, etc.). Filtering server-side is faster and more accurate than pulling everything and grepping. -4. **Keep commands small and fast.** A fleet can hold hundreds of thousands of nodes/alerts/events, so an unbounded pull is slow and can time out or blow up your context. Ask for the least data that answers the question: filter tightly, and page in small chunks (`--page-size` is 1–100, default is fine). **For a count from a paginated list, don't fetch the rows at all** — run with `--page-size 1` and read the top-level `total` (see [Counting without fetching](#counting-without-fetching)). `tag list` is the exception: it has no pagination flags and returns `tags` without a top-level `total`. Avoid `--all` unless you truly need every row, and ask for confirmation before using it unless the user explicitly requested all records, a complete export, or a fleet-wide report. Even then, prefer a tight filter first. -5. **Don't invent data.** If a field isn't in the output, say so rather than guessing. UUIDs, hostnames, and counts must come from real command output. - -Before relying on output, confirm the tool is installed and authenticated — see [Setup and auth](#setup-and-auth). If `nvfleetint` isn't found (e.g. `command not found`, exit code **127**), it isn't installed — don't try to build it from source; point the user at the releases page as described in [Setup and auth](#setup-and-auth). If a command exits with code **77**, that's an auth/permission failure, not a real "no results" — handle it as described there. - -## Mapping questions to commands - -Use this to pick the entry point. Each command takes `--output json` and -`--timeout ` (e.g. `30s`). Paginated `list` commands also take `--all`, -`--page`, and `--page-size` (1–100); `tag list` does not paginate. - -| The user is asking about… | Start here | +| Need | Command | | --- | --- | -| A one-shot fleet summary — total/healthy/unhealthy counts, top-line metrics | `overview` | -| Regions / zones / where capacity lives | `computezone list` | -| Node groups, their health %, GPU utilization | `nodegroup list` | -| Individual nodes — health, GPU type/count, agent online/offline, firmware/verification | `node list`, then `node describe ` for one node | -| How one node's health changed over a time window | `node health ` | -| Active problems, severities, what's firing now | `alert list`, `alert timeline`, `alert describe` | -| Raw event stream or an event histogram over time | `event list`, `event buckets` | -| What customer tags exist (optionally scoped to a resource) | `tag list` | -| A full inventory snapshot (export, audit, signed bundle) | `report inventory` | -| Error trends / counts over a time range | `report error` | -| Checking a previously downloaded signed report | `report verify` | - -### Fleet overview - -For a fast, top-line answer ("how's the fleet doing?", "how many nodes total / unhealthy?") start with `overview` — a single call returns fleet-wide counts plus summary metrics, no pagination. +| Fleet totals/health/metrics | `overview` | +| Zones or node groups | `computezone list`, `nodegroup list` | +| Nodes/current detail | `node list`, `node describe ` | +| Node health history | `node health ` | +| Fleet alert records | `alert list` | +| Alert impact/investigation | `alert summary`, `alert node`, `alert describe`, `alert options` | +| Raw events/histogram | `event list`, `event buckets` | +| Customer tags | `tag list` | +| Inventory/error reports | `report inventory`, `report error` | +| Verify signed report | `report verify` | + +### Overview and inventory ```bash nvfleetint overview --output json -nvfleetint overview --include-metrics=false --output json # counts only, skip the metrics block +nvfleetint overview --include-metrics=false --output json +nvfleetint computezone list --output json +nvfleetint nodegroup list --health Degraded,Unhealthy --output json +nvfleetint nodegroup list --gpu-type H100 --sort-by health --order desc --output json ``` -Use it for the headline number; drop to `node list` / `nodegroup list` when the user wants the actual nodes behind the count. +`overview` is one non-paginated object. Use lists for rows behind its counts. -### Inspecting the fleet +### Nodes ```bash -# Compute zones (regions/sites and their node counts) -nvfleetint computezone list --output json -nvfleetint computezone list --zone-ids zone-1,zone-2 --output json - -# Node groups — filter by health, GPU type; sort by health/nodes -nvfleetint nodegroup list --output json -nvfleetint nodegroup list --health Degraded,Unhealthy --output json -nvfleetint nodegroup list --gpu-type H100 --sort-by health --order desc --output json - -# Nodes — the most filterable command -nvfleetint node list --output json nvfleetint node list --health Degraded,Unhealthy --output json nvfleetint node list --agent-status Offline --output json -nvfleetint node list --hostname gpu-node-7 --output json # partial hostname match -nvfleetint node list --gpu-type H100 --sort-by hostname --order asc --output json - -# Everything about one node (system info, resources, network, health, components) +nvfleetint node list --hostname gpu-node-7 --view basic --output json +nvfleetint node list --compute-zone-names ord --output json +nvfleetint node list --nodegroup-names training --output json nvfleetint node describe --output json - -# One node's health status timeline + summary over a window (both --start and --end REQUIRED, RFC3339) -nvfleetint node health --start 2026-07-14T00:00:00Z --end 2026-07-21T00:00:00Z --output json +nvfleetint node health --start --end --output json ``` -`node describe` is the current-state snapshot; `node health` answers "when did this node go bad / how has its health trended?" over an explicit window (there's no `--window` shortcut here — pass absolute `--start`/`--end`). +Ask users for human-readable zone/group names, never IDs. Name filters are comma-separated partial matches. For exact scope, resolve with `computezone list --view basic` or `nodegroup list --view basic`, clarify ambiguous names with recognizable detail metadata, then use IDs internally. Accept an ID already supplied by the user. -Filter vocabularies (case-sensitive, comma-separate multiple values): -- **health**: `Healthy`, `Degraded`, `Unhealthy`, `Unknown` -- **agent-status**: `Online`, `Offline`, `Unknown` -- **verification-check**: `Verified`, `Unverified`, `Degraded`, `Pending`, `Unsupported`, `Unknown` -- **firmware-check**: `Passed`, `Failed`, `Unknown` -- **node sort-by**: `hostname`, `nodeUUID`, `healthStatus`, `nodegroup`, `computezone`, `gpuType`, `gpuCount`, `verificationCheck`, `agentStatus`, `agentVersion`, `kernelVersion`, `gpuDriverVersion`, `gpuFirmwareVersions` (+ `--order asc|desc`) — `verificationCheck` sorts on the same data as the `verification-check` filter (the backend spelling `integrityCheck` is still accepted) +Node basic rejects health, agent, verification, and firmware filters and supports sorting only by `hostname` or `nodeUUID`. `node health` requires both absolute boundaries. It does not support `--window`. -`--output json` returns the raw backend field names, not the display terms: verification state is `integrityCheck` (with `integrityCheckReason`, `lastIntegrityCheckTS`) and location is `geoLocation`. The `verification-check`/"location" naming applies only to the CLI flag and table output. +Filter values: -To get an exact **count**, use the [Counting without fetching](#counting-without-fetching) trick — `--page-size 1` and read the top-level `total`. Don't eyeball the length of one page (it's only the first page), and don't pull `--all` just to count — on a large fleet that fetches thousands of rows you'll immediately throw away. - -### Counting without fetching - -When the user only wants a number ("how many nodes are unhealthy?", "how many critical alerts?"), you don't need the rows — you need the `total`. For a paginated `list`, run the same filtered command with **`--page-size 1`** and read the **top-level `total`** field. This is one tiny request regardless of fleet size, and every filter still applies, so the count is exactly the filtered count. Do not apply this shortcut to `tag list`: it has no pagination flags and returns only `tags`, with no top-level `total`; count the returned tags instead. - -```bash -nvfleetint node list --health Unhealthy --page-size 1 --output json # -> read .total -nvfleetint alert list --severity Critical --page-size 1 --output json # -> read .total -nvfleetint event list --window 24h --page-size 1 --output json # -> read .total -``` - -Two different JSON shapes carry the total, depending on whether you paged or pulled everything: - -- **Single page** (default / `--page-size N`, no `--all`): `total` is at the **top level**, alongside the resource-specific item array. Keys are `total`, `page`, `pageSize`, and a "more pages?" indicator; the items live under a per-resource key — `nodes` / `nodeGroups` / `computezones` (lowercase) / `alerts` / `events`. Read `.total`. The more-pages indicator is `hasMore` (a bool) on every list **except `alert list`**, which instead exposes `pageCursorNext` (a string that's non-empty when more pages exist and absent/empty otherwise) and has **no `hasMore` field** — so for alerts, check `pageCursorNext`, not `hasMore`. -- **`--all`** (merged across every page): the shape is `{"items": [...], "pagination": {"total": N, "hasMore": ..., "pagesFetched": ...}}`. Read `.pagination.total`. +| Flag | Values | +| --- | --- | +| `--health` | Healthy, Degraded, Unhealthy, Unknown | +| `--agent-status` | Online, Offline, Unknown | +| `--verification-check` | Verified, Unverified, Degraded, Pending, Unsupported, Unknown | +| `--firmware-check` | Passed, Failed, Unknown | -So for a count, `--page-size 1` → `.total`; only reach for `--all` → `.pagination.total` when you actually want the rows too. +Node sort keys are `hostname`, `nodeUUID`, `healthStatus`, `nodegroup`, `computezone`, `gpuType`, `gpuCount`, `verificationCheck`, `agentStatus`, `agentVersion`, `kernelVersion`, `gpuDriverVersion`, and `gpuFirmwareVersions`. The backend spelling `integrityCheck` remains accepted as an alias for `verificationCheck`. Node-group sort keys are `health` and `nodes`. -### Alerts +### Alerts and events ```bash -# Alerts firing now; filter by severity, state, component, and/or node -nvfleetint alert list --output json nvfleetint alert list --severity Critical --output json -nvfleetint alert list --severity Critical --node --output json -nvfleetint alert list --state Triggered --component GPU --output json - -# Timeline: which nodes have alert history, or one node's history -nvfleetint alert timeline --output json # all nodes with history -nvfleetint alert timeline --active --output json # only currently-active alerts -nvfleetint alert timeline --node --output json - -# Full event history for a single alert (note: --node is REQUIRED here) +nvfleetint alert list --node --state Triggered --output json +nvfleetint alert summary --output json +nvfleetint alert summary --view historical --output json +nvfleetint alert node --output json +nvfleetint alert node --view historical --output json nvfleetint alert describe --node --output json -``` - -`alert list` filter vocabularies: **severity** = `Critical`, `Warning`; **state** = `Detected`, `Triggered`, `Resolved`; `--component` matches a component name (e.g. `GPU`). - -When the user says "what's wrong right now?" prefer `alert list --severity Critical` and `node list --health Unhealthy,Degraded`. Use the timeline when they ask about history or recurrence. - -### Events - -Events are the raw, time-stamped fleet event stream (below the alert layer). Every event command **requires a time range** — either `--window ` (relative) or `--start`/`--end` (absolute RFC3339, used together) — and can be narrowed by `--node` and `--component`. - -```bash -# Individual events over a range +nvfleetint alert options --output json nvfleetint event list --window 24h --output json -nvfleetint event list --window 168h --node --component GPU --output json -nvfleetint event list --start 2026-05-01T00:00:00Z --end 2026-05-08T00:00:00Z --output json - -# Time-bucketed counts for a histogram (--max-buckets 1-1000, default 100) -nvfleetint event buckets --window 24h --output json nvfleetint event buckets --window 168h --max-buckets 50 --output json ``` -`event list` paginates (`--all`, `--page`, `--page-size`); `event buckets` does not — it returns the bucketed series in one call. Reach for events when the user wants the granular "what happened, and when" detail that `alert list` (current problems) and `report error` (aggregate counts) don't give. - -### Tags - -```bash -# All unique customer tags across the fleet -nvfleetint tag list --output json -nvfleetint tag list --prefix gpu --output json # case-insensitive prefix filter - -# Scope to one resource (use at MOST one of --node / --nodegroup / --computezone) -nvfleetint tag list --node --output json -nvfleetint tag list --computezone --prefix env --output json -``` +Alert severity is Critical/Warning; state is Detected/Triggered/Resolved. `alert summary`, `alert node`, and `alert options` default to the active view; use `--view historical` for history. Summary returns impacted nodes plus fleet-wide alert aggregates. Node returns alerts for one node. Describe returns one alert's event history. In node-alert results, Critical/Warning values are active severity; Detected/Resolved are inactive audit values. Don't count every non-Resolved row as active. -`tag list` answers "what tags are in use?" — the resource filters are mutually exclusive, but `--prefix` can combine with any one of them. There's no pagination here. +Events require `--window` or both `--start`/`--end`. Durations use Go units through hours—no `d`. Event list paginates; buckets do not. -### Reports +### Tags and reports ```bash -# Inventory snapshot of the whole fleet +nvfleetint tag list --prefix gpu --output json +nvfleetint tag list --computezone --output json nvfleetint report inventory --all --output json -nvfleetint report inventory --format csv > inventory.csv # CSV for the user -nvfleetint report inventory --format csv --signed # signed bundle (CSV + cosign signature) nvfleetint report inventory --format csv --signed --output-path ./reports/ - -# Error report over a time range. Pick ONE time selector: -# --window relative, e.g. 24h, 168h (Go duration; no d unit) -# --start ... --end ... absolute RFC3339, used together -nvfleetint report error --window 24h --output json # overview (totals) nvfleetint report error --view list --group-by error --window 168h --output json -nvfleetint report error --view list --group-by node \ - --start 2026-05-01T00:00:00Z --end 2026-05-08T00:00:00Z --output json -nvfleetint report error --view graph --window 24h --output json # time series - -# Verify a signed inventory bundle the user already downloaded -nvfleetint report verify --csv inventory_report_.csv --bundle inventory_report_.sig.bundle -nvfleetint report verify --csv report.csv --bundle report.sig.bundle --key signing-key.pub # offline +nvfleetint report error --view graph --window 24h --output json +nvfleetint report verify --csv report.csv --bundle report.sig.bundle ``` -`report error` notes: `--view list` requires `--group-by error|node`. `--format csv` is only valid with `--view list`. Default view is `overview`. - -## Setup and auth - -### Installation - -If `nvfleetint` isn't on the user's PATH (a command fails with `command not found` / exit code **127**), it isn't installed. Don't build it from source — direct the user to download a prebuilt binary for their platform from the releases page: - - - -Tell them to grab the latest release asset matching their OS/architecture, extract it, and put `nvfleetint` somewhere on their PATH. Once it's installed, re-run `nvfleetint auth status` and continue. - -### Auth - -Credentials live in named **profiles** in `~/.config/nvfleetint/config.yaml` (mode 0600). A profile pairs an API key with an API URL, so one machine can reach several tenants or endpoints. - -```bash -nvfleetint auth status # check before querying if unsure -nvfleetint auth list # which profiles exist, and which is current -nvfleetint auth add --api-key # no name: the "default" profile -nvfleetint auth add --api-key -nvfleetint auth add --api-key --api-url https://api.fleet-intelligence.nvidia.com -nvfleetint auth use # change the default -nvfleetint auth add --api-key --yes # existing name: rotate the key -nvfleetint auth remove -``` - -`auth add/remove/use` take the profile as a **positional** `` — it is the thing being changed. Don't pass `--profile` to them; they don't accept it. On `auth add` the name is optional and means the profile called `default`; prefer that form when the user hasn't mentioned multiple tenants, rather than inventing a name for them. `auth remove` and `auth use` always require the name. - -There is no `auth update`: `auth add` on an existing name changes that profile in place (partial — an omitted flag keeps the stored value), which is also the key-rotation path. Replacing a key a profile already has prompts for confirmation, and **you cannot answer that prompt** — you have no terminal, so the command fails with "cannot prompt for confirmation". Pass `--yes` only when the user has actually asked to replace that profile's key; otherwise report the prompt back and let them decide. Nothing else prompts, so the fixes the CLI suggests in its own error messages (`auth add --api-key ...` for a profile with no key, `auth add --api-url ...` for a rejected endpoint) are safe to run as printed. **Check `auth list` before adding**, or a mistyped name that happens to exist will overwrite a working key. The output says `added` vs `updated` — read it back to the user. - -Every API-backed command, plus `auth status`, instead accepts `--profile ` to use one profile for a single invocation (`nvfleetint node list --profile dev`). Without it, commands use the current profile — the one marked `*` in `auth list`. If the user mentions more than one environment, tenant, or org, run `auth list` first and ask which profile they mean rather than guessing. - -The API URL must be `https` (plain `http` is only allowed for `localhost`), so never suggest an `http://` endpoint. - -`auth status` verifies the **effective** credentials against the backend and prints `Profile:`, `API URL:`, and `API key:` lines showing what resolved and where each value came from. Pass `--profile ` to check a specific profile. It's diagnostic: it exits `0` and reports a `Connection:` line rather than failing on bad credentials. Read that line — don't treat exit 0 as "authenticated." Require `Connection: ok`; treat `Connection: unauthorized` (missing, invalid, or expired key), `unauthenticated`, or `error: ...` as an auth failure and stop. The `API URL:` line tells you which endpoint was checked, which is how you spot a profile or env override pointing somewhere unexpected. - -Credentials resolve highest-first: `--profile`, then the current profile with `NVFLEETINT_API_KEY` / `NVFLEETINT_API_URL` overlaid on top. **Selecting a profile explicitly ignores those two env vars entirely** — that is deliberate, so a stale variable can't send one tenant's key to another tenant's endpoint. So a bad env override only affects commands that *don't* pass `--profile`. If `auth status` (without `--profile`) reports a wrong `API URL:` or an `API key:` sourced from the environment, have the user `unset NVFLEETINT_API_KEY NVFLEETINT_API_URL` in their shell (and remove them from any shell profile that exports them), then re-run `auth status`. Check whether the vars are *set*, not what they contain — never print the value of `NVFLEETINT_API_KEY`. - -If a query fails with **exit code 77** or a 401/403, the user isn't authenticated (or the key lacks permission). Don't report this as "no nodes found." Instead, run `nvfleetint auth status` to confirm, then tell the user to generate an NGC API key at and run `nvfleetint auth add --api-key ` (add a `` before `--api-key` only if they use more than one tenant; the same command rotates the key of a profile that already exists). Never ask the user to paste an API key into the chat, and never echo a key you happen to see — it's a secret. `auth list` and `auth status` never print keys, only whether one is configured. - -## Worked example - -User: *"Are any of my H100 nodes having problems?"* - -1. `nvfleetint node list --gpu-type H100 --health Degraded,Unhealthy,Unknown --output json` — include `Unknown` so nodes the backend can't report on don't silently disappear. The tight filter keeps the result small, so the first page is usually the whole answer (check `hasMore`; only page further or add `--all` if it's set and you need the rest). For just the *count*, `--page-size 1` and read `total`. (On `alert list` the more-pages field is `pageCursorNext`, not `hasMore` — see [Counting without fetching](#counting-without-fetching).) -2. Read `total` and the `nodes` — `total` counts only the filtered matches, not the H100 fleet. Break the count out by state: *"Yes — 2 H100 nodes are in trouble: `gpu-node-12` (firmware check Failed) and `gpu-node-31` (agent Offline). 1 more, `gpu-node-07`, is reporting Unknown health."* Don't claim anything about the rest without querying for it. -3. If they want detail on one, follow with `nvfleetint node describe --output json` and `nvfleetint alert list --node --output json`. - -That's the loop: pick the command, filter tightly, run with JSON, answer in prose, drill down on request. +Use at most one tag scope flag: `--node`, `--nodegroup`, or `--computezone`; tag list has no pagination. Report-error list requires `--group-by error|node`; only list supports `--all` and CSV. Signed inventory requires CSV. -## Discovering flags +## Example -This file covers the common paths. For the authoritative, current flag list of any command, run `nvfleetint --help` (or `nvfleetint --help`) rather than guessing — the CLI is the source of truth if it has changed. +For “Are any H100 nodes having problems?”, query H100 nodes filtered to Degraded/Unhealthy/Unknown. Include Unknown, read the filtered total, list only returned problem nodes, and do not claim the remaining H100 fleet is healthy without a separate query. Drill into requested UUIDs with `node describe` and node-scoped alerts. diff --git a/skills/nvfleetint/references/auth.md b/skills/nvfleetint/references/auth.md new file mode 100644 index 0000000..e4b2536 --- /dev/null +++ b/skills/nvfleetint/references/auth.md @@ -0,0 +1,33 @@ +# Installation and profiles + +Read only for setup, key rotation, or credential-source diagnosis. + +## Installation + +Exit 127/`command not found` means the CLI is absent. Don't build from source; use the matching release asset from , put it on `PATH`, then run `nvfleetint auth status`. + +## Profiles + +Profiles pair API URL and key in `~/.config/nvfleetint/config.yaml` (0600): + +```bash +nvfleetint auth list +nvfleetint auth status [--profile ] +nvfleetint auth add --api-key # default profile +nvfleetint auth add --api-key [--api-url https://...] +nvfleetint auth use +nvfleetint auth add --api-key --yes # rotate +nvfleetint auth remove +``` + +`auth add/remove/use` take a positional name, not `--profile`; add's name is optional and defaults to `default`. There is no `auth update`: add updates an existing profile. Run `auth list` before add so a typo cannot overwrite another profile. Use `--yes` only when the user explicitly requested key replacement. + +Require HTTPS except localhost. Never request, print, or log a key. + +## Diagnose effective credentials + +`auth status` reports Profile, API URL, API-key source, and Connection but exits 0 on bad auth. Require `Connection: ok`. + +Without explicit `--profile`, the current profile is overlaid by `NVFLEETINT_API_KEY`/`NVFLEETINT_API_URL`; explicit profile ignores both. If status shows an unexpected environment source, ask the user to unset the variables without printing their values. + +Exit 77 or HTTP 401/403 is auth/permission failure. Direct the user to create an NGC service key at and run auth add. Never interpret auth failure as an empty fleet. diff --git a/skills/nvfleetint/references/cli-contract.md b/skills/nvfleetint/references/cli-contract.md new file mode 100644 index 0000000..7ee6865 --- /dev/null +++ b/skills/nvfleetint/references/cli-contract.md @@ -0,0 +1,41 @@ +# nvfleetint CLI contract + +Read this before querying live data. + +## Commands and retries + +- Use `--output json`; never parse tables. Confirm flags with `nvfleetint [] --help`. +- `--timeout` is per request (default 2m). Don't lower it. Use 3m–5m for demonstrated slow node pulls above 1,000 rows and 5m–10m for demonstrated slow alert pulls; still bound the whole workflow separately. +- The client retries idempotent GET/HEAD requests up to three attempts for transient network failures and HTTP 408, 429, 500, 502, 503, and 504. During `--all`, only the failed page retries. Never retry the whole command in a loop. +- Exit 127 means not installed; use a release from . + +## Auth and profiles + +- Require `Connection: ok` from `auth status`; it exits 0 even when auth fails. Exit 77 or HTTP 401/403 is auth failure, not an empty fleet. +- Never request or expose an API key. Direct users to and `nvfleetint auth add --api-key `. +- If the user names a tenant/environment, use that profile. If multiple profiles exist and none is named, ask which one. Pass the same explicit `--profile ` to every report command. +- Explicit `--profile` ignores `NVFLEETINT_API_KEY` and `NVFLEETINT_API_URL`. + +## Pagination + +- Lists accept `--page`, `--page-size` (1–100), and `--all`. Non-paginated: `overview`, `node describe`, `node health`, `alert options`, `alert describe`, `event buckets`, `tag list`, and `report error --view overview|graph`. +- Single pages use backend arrays (`nodes`, `nodeGroups`, `computezones`, `alerts`, `events`) plus top-level `total`, `page`, and `pageSize`. Most use `hasMore`; `alert list` uses nonempty `pageCursorNext` instead. `alert summary` uses `nodes`, `alert node` uses `alerts`, and report-error list uses `nodes`. +- `--all` normalizes every paginated list to `{items, pagination:{total,hasMore,pagesFetched}}`; non-paginated commands keep their native shape. +- Count cheaply with the same filters plus `--page-size 1`, reading `.total`. For `alert summary`, `.total` is nodes with matching alerts; use `totalCritical` and `totalWarning` for alert aggregates. `tag list` is the exception: count its non-paginated `tags`. +- `--view basic` returns identities only: node returns hostname/UUID; compute zone and node group return `id`/`name`. Node basic rejects `--health`, `--agent-status`, `--verification-check`, and `--firmware-check`, and sorts only by hostname or nodeUUID. Node-group basic rejects health, gpu-type, and sorting. + +## JSON names + +Use backend names: `healthStatus`, `integrityCheck`, `integrityCheckReason`, `lastIntegrityCheckTS`, and `geoLocation`. `overview` calls the healthy-node count `healthNodeCount`, not `healthyNodeCount`. + +## Completeness + +Before analysis require exit 0, nonempty valid JSON, and no top-level `error`/`api_error`. For every `--all` response require an `items` array, `hasMore == false`, `pagesFetched >= 1`, and—when `total > 0`—item count equal to total. A missing or null total is unreported; a nonempty result with explicit `total: 0` is malformed. Empty with `hasMore: false` is valid. + +Filtered pulls may be composed only when each is complete and the disjoint filters cover the domain. Record each reported total or validated item count, then union. Never analyze partial pages. On malformed/incomplete data, use only the report workflow's allowed fallback or stop. + +The fleet-health workflow's deliberately bounded `alert summary --page-size 10` is the sole partial-page exception: use only its server-computed top-level aggregates for fleet-wide claims, label returned rows `showing N of total`, and never infer the attributes of unseen nodes. + +## Secrets + +Never expose credentials, authorization headers, environment values, or raw config. Check whether auth environment variables are set; never print them.