Skip to content
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,49 @@ Refer to the documentation for more information on NemoClaw.
- [Troubleshooting](https://docs.nvidia.com/nemoclaw/latest/reference/troubleshooting.html): Troubleshoot common issues and resolution steps.
- [Discord](https://discord.gg/XFpfPv9Uvx): Join the community for questions and discussion.

---

## Observability

NemoClaw includes an optional observability layer for tracking agent execution and blueprint performance.

### Enabling Metrics

Metrics are disabled by default. To enable them, set the `NEMOCLAW_METRICS_ENABLED` environment variable:

```bash
export NEMOCLAW_METRICS_ENABLED=true
export NEMOCLAW_METRICS_PORT=9090 # Optional, defaults to 9090
```

### Accessing Metrics

When enabled, NemoClaw starts a lightweight Prometheus-compatible metrics server. You can view the metrics by curling the `/metrics` endpoint:

```bash
curl http://localhost:9090/metrics
```

### Example Output

```text
# HELP blueprint_execution_total Total count of blueprint_execution
# TYPE blueprint_execution_total counter
blueprint_execution_total{action="apply",profile="default",status="success"} 1

# HELP blueprint_execution_latency_seconds Latency histogram for blueprint_execution
# TYPE blueprint_execution_latency_seconds histogram
blueprint_execution_latency_seconds_bucket{action="apply",profile="default",status="success",le="0.1"} 0
blueprint_execution_latency_seconds_bucket{action="apply",profile="default",status="success",le="0.5"} 0
blueprint_execution_latency_seconds_bucket{action="apply",profile="default",status="success",le="1"} 0
blueprint_execution_latency_seconds_bucket{action="apply",profile="default",status="success",le="5"} 1
blueprint_execution_latency_seconds_bucket{action="apply",profile="default",status="success",le="+Inf"} 1
blueprint_execution_latency_seconds_sum{action="apply",profile="default",status="success"} 3.42
blueprint_execution_latency_seconds_count{action="apply",profile="default",status="success"} 1
```

---

## License

This project is licensed under the [Apache License 2.0](LICENSE).
12 changes: 12 additions & 0 deletions nemoclaw/src/blueprint/exec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { join } from "node:path";
import type { PluginLogger } from "../index.js";
import { observeLatency } from "../observability/metrics.js";

export type BlueprintAction = "plan" | "apply" | "status" | "rollback";

Expand Down Expand Up @@ -34,6 +35,17 @@ function failResult(action: BlueprintAction, message: string): BlueprintRunResul
export async function execBlueprint(
options: BlueprintRunOptions,
logger: PluginLogger,
): Promise<BlueprintRunResult> {
return observeLatency(
"blueprint_execution",
{ action: options.action, profile: options.profile },
() => execBlueprintInternal(options, logger),
);
}
Comment thread
ksapru marked this conversation as resolved.
Resolved

async function execBlueprintInternal(
options: BlueprintRunOptions,
logger: PluginLogger,
): Promise<BlueprintRunResult> {
const runnerPath = join(options.blueprintPath, "orchestrator", "runner.py");

Expand Down
3 changes: 3 additions & 0 deletions nemoclaw/src/commands/launch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { verifyBlueprintDigest, checkCompatibility } from "../blueprint/verify.j
import { execBlueprint } from "../blueprint/exec.js";
import { loadState, saveState } from "../blueprint/state.js";
import { detectHostOpenClaw } from "./migrate.js";
import { metrics } from "../observability/metrics.js";

export interface LaunchOptions {
force: boolean;
Expand Down Expand Up @@ -106,6 +107,8 @@ export async function cliLaunch(opts: LaunchOptions): Promise<void> {
return;
}

metrics.incrementCounter("sandbox_operation_total", { operation: "launch" });

// Save state
saveState({
...loadState(),
Expand Down
32 changes: 31 additions & 1 deletion nemoclaw/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,37 @@ export default function register(api: OpenClawPluginApi): void {
{ commands: ["nemoclaw"] },
);

// 3. Register nvidia-nim provider — use onboard config if available
// 3. Register Metrics Service if enabled
if (metrics.isEnabled()) {
let server: ReturnType<typeof createServer> | undefined;

api.registerService({
id: "nemoclaw-metrics",
start: ({ logger }) => {
const port = Number(process.env.NEMOCLAW_METRICS_PORT || 9090);
server = createServer((req, res) => {
if (req.url === "/metrics") {
res.writeHead(200, { "Content-Type": "text/plain" });
res.end(metrics.getPrometheusMetrics());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
} else {
res.writeHead(404);
res.end();
}
});
server.listen(port, "0.0.0.0", () => {
logger.info(`NemoClaw metrics server listening on port ${port}`);
});
Comment thread
ksapru marked this conversation as resolved.
Resolved
},
stop: ({ logger }) => {
if (server) {
server.close();
logger.info("NemoClaw metrics server stopped");
}
},
});
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

// 4. Register nvidia-nim provider — use onboard config if available
const onboardCfg = loadOnboardConfig();
const providerCredentialEnv = onboardCfg?.credentialEnv ?? "NVIDIA_API_KEY";
api.registerProvider(registeredProviderForConfig(onboardCfg, providerCredentialEnv));
Expand Down
153 changes: 153 additions & 0 deletions nemoclaw/src/observability/metrics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

/**
* Lightweight metrics implementation for NemoClaw.
*
* Provides counters and histograms for request tracking and latency observation.
* Enabled only when NEMOCLAW_METRICS_ENABLED=true.
*/

export interface MetricValue {
name: string;
help: string;
type: "counter" | "histogram";
labels: Record<string, string>;
value: number;
timestamp: number;
}

export interface HistogramValue extends MetricValue {
type: "histogram";
buckets: Record<number, number>;
sum: number;
count: number;
}

class MetricsRegistry {
private counters: Map<string, number> = new Map();
private histograms: Map<string, { sum: number; count: number; buckets: Record<number, number> }> =
new Map();

// Standard buckets for latency (seconds)
private defaultBuckets = [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60];

public isEnabled(): boolean {
return process.env.NEMOCLAW_METRICS_ENABLED === "true";
}

public incrementCounter(name: string, labels: Record<string, string> = {}): void {
if (!this.isEnabled()) return;
const key = this.formatKey(name, labels);
this.counters.set(key, (this.counters.get(key) || 0) + 1);
}

public observeHistogram(
name: string,
value: number,
labels: Record<string, string> = {},
buckets = this.defaultBuckets,
): void {
if (!this.isEnabled()) return;
const key = this.formatKey(name, labels);
let hist = this.histograms.get(key);
if (!hist) {
hist = { sum: 0, count: 0, buckets: {} };
buckets.forEach((b) => (hist!.buckets[b] = 0));
this.histograms.set(key, hist);
}

hist.sum += value;
hist.count += 1;
buckets.forEach((b) => {
if (value <= b) {
hist!.buckets[b] = (hist!.buckets[b] || 0) + 1;
}
});
}

public getPrometheusMetrics(): string {
let output = "";

// Export counters
for (const [key, value] of this.counters.entries()) {
const [name, labelStr] = this.parseKey(key);
output += `# HELP ${name} Total count of ${name}\n`;
output += `# TYPE ${name} counter\n`;
output += `${name}${labelStr} ${value}\n\n`;
}

// Export histograms
for (const [key, hist] of this.histograms.entries()) {
const [name, labelStr] = this.parseKey(key);
output += `# HELP ${name} Latency histogram for ${name}\n`;
output += `# TYPE ${name} histogram\n`;

const sortedBuckets = Object.keys(hist.buckets)
.map(Number)
.sort((a, b) => a - b);
const labelsBase = labelStr.length > 2 ? labelStr.slice(1, -1) + "," : "";

sortedBuckets.forEach((b) => {
output += `${name}_bucket{${labelsBase}le="${b === Infinity ? "+Inf" : b}"} ${hist.buckets[b]}\n`;
});
output += `${name}_bucket{${labelsBase}le="+Inf"} ${hist.count}\n`;
output += `${name}_sum${labelStr} ${hist.sum}\n`;
output += `${name}_count${labelStr} ${hist.count}\n\n`;
}

return output;
}

private formatKey(name: string, labels: Record<string, string>): string {
const labelPairs = Object.entries(labels)
.map(([k, v]) => `${k}="${v}"`)
.join(",");
return labelPairs ? `${name}{${labelPairs}}` : name;
}

private parseKey(key: string): [string, string] {
const braceIdx = key.indexOf("{");
if (braceIdx === -1) return [key, ""];
return [key.slice(0, braceIdx), key.slice(braceIdx)];
}
}

export const metrics = new MetricsRegistry();

/**
* Helper to measure execution time of a promise.
*/
export async function observeLatency<T>(
name: string,
labels: Record<string, string>,
fn: () => Promise<T>,
): Promise<T> {
if (!metrics.isEnabled()) return fn();

const start = process.hrtime.bigint();
try {
const result = await fn();
const end = process.hrtime.bigint();
const durationSec = Number(end - start) / 1e9;

metrics.observeHistogram(`${name}_latency_seconds`, durationSec, {
...labels,
status: "success",
});
metrics.incrementCounter(`${name}_total`, { ...labels, status: "success" });

return result;
} catch (error) {
const end = process.hrtime.bigint();
const durationSec = Number(end - start) / 1e9;

metrics.observeHistogram(`${name}_latency_seconds`, durationSec, {
...labels,
status: "error",
});
metrics.incrementCounter(`${name}_total`, { ...labels, status: "error" });

throw error;
}
}
13 changes: 13 additions & 0 deletions nemoclaw/src/onboard/validate.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { observeLatency } from "../observability/metrics.js";

export interface ValidationResult {
valid: boolean;
models: string[];
Expand All @@ -10,6 +12,17 @@ export interface ValidationResult {
export async function validateApiKey(
apiKey: string,
endpointUrl: string,
): Promise<ValidationResult> {
return observeLatency(
"tool_api_validate",
{ endpoint: endpointUrl },
() => validateApiKeyInternal(apiKey, endpointUrl),
);
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Metric status may not reflect validation outcome.

The observeLatency wrapper records status="error" only when the wrapped function throws. However, validateApiKeyInternal catches all errors internally and returns { valid: false, error: message } instead of throwing. This means:

  • HTTP 401/403 responses → recorded as status="success"
  • Request timeouts → recorded as status="success"
  • Network errors → recorded as status="success"

If the intent is to track whether the API call completed (regardless of validity), this is fine. If you want metrics to reflect validation success/failure, consider either:

  1. Having validateApiKeyInternal throw on invalid results, or
  2. Using a separate counter increment based on result.valid after the call.
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@nemoclaw/src/onboard/validate.ts` around lines 12 - 21, The observeLatency
wrapper currently only marks errors when the wrapped function throws, but
validateApiKeyInternal returns { valid: boolean, error?: string } on failures;
update validateApiKey to call validateApiKeyInternal inside observeLatency as
you do, then inspect the returned ValidationResult and emit a separate metric or
set status based on result.valid (or rethrow when valid is false) so failures
(validateApiKeyInternal returning valid: false) are recorded as errors;
reference the validateApiKey and validateApiKeyInternal symbols and ensure you
add the counter/instrumentation update immediately after the observeLatency call
to reflect validation success/failure.


async function validateApiKeyInternal(
apiKey: string,
endpointUrl: string,
): Promise<ValidationResult> {
const url = `${endpointUrl.replace(/\/+$/, "")}/models`;
const controller = new AbortController();
Expand Down
57 changes: 57 additions & 0 deletions test/metrics.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

const test = require("node:test");
const assert = require("node:assert");
const path = require("node:path");

// Load the compiled metrics module
const metricsPath = path.resolve(__dirname, "../nemoclaw/dist/observability/metrics.js");
const { metrics, observeLatency } = require(metricsPath);

// Enable metrics for testing
process.env.NEMOCLAW_METRICS_ENABLED = "true";

test("MetricsRegistry stores and exports counters", () => {
metrics.incrementCounter("test_counter", { foo: "bar" });
const output = metrics.getPrometheusMetrics();

assert.match(output, /# TYPE test_counter counter/);
assert.match(output, /test_counter\{foo="bar"\} 1/);
});

test("MetricsRegistry stores and exports histograms", () => {
metrics.observeHistogram("test_hist", 0.5, { abc: "123" });
const output = metrics.getPrometheusMetrics();

assert.match(output, /# TYPE test_hist histogram/);
assert.match(output, /test_hist_bucket\{abc="123",le="0\.5"\} 1/);
assert.match(output, /test_hist_sum\{abc="123"\} 0\.5/);
assert.match(output, /test_hist_count\{abc="123"\} 1/);
});

test("observeLatency tracks success metrics", async () => {
const result = await observeLatency("test_op", { op: "success" }, async () => {
return "done";
});

assert.strictEqual(result, "done");
const output = metrics.getPrometheusMetrics();

assert.match(output, /test_op_total\{op="success",status="success"\} 1/);
assert.match(output, /test_op_latency_seconds_count\{op="success",status="success"\} 1/);
});

test("observeLatency tracks error metrics", async () => {
try {
await observeLatency("test_op_err", { op: "fail" }, async () => {
throw new Error("oops");
});
} catch (err) {
assert.strictEqual(err.message, "oops");
}

const output = metrics.getPrometheusMetrics();
assert.match(output, /test_op_err_total\{op="fail",status="error"\} 1/);
assert.match(output, /test_op_err_latency_seconds_count\{op="fail",status="error"\} 1/);
});