Repository navigation
feat: add lightweight observability and metrics service #230
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 4 commits
1c74bc0
ad1f2e7
68bb16c
ceb38d1
ca431d5
c490415
d19fdab
e53dcdb
b0b11e8
a9a5887
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| /** | ||
| * Lightweight metrics implementation for NemoClaw. | ||
| * | ||
| * Provides counters and histograms for request tracking and latency observation. | ||
| * Enabled only when NEMOCLAW_METRICS_ENABLED=true. | ||
| */ | ||
|
|
||
| export interface MetricValue { | ||
| name: string; | ||
| help: string; | ||
| type: "counter" | "histogram"; | ||
| labels: Record<string, string>; | ||
| value: number; | ||
| timestamp: number; | ||
| } | ||
|
|
||
| export interface HistogramValue extends MetricValue { | ||
| type: "histogram"; | ||
| buckets: Record<number, number>; | ||
| sum: number; | ||
| count: number; | ||
| } | ||
|
|
||
| class MetricsRegistry { | ||
| private counters: Map<string, number> = new Map(); | ||
| private histograms: Map<string, { sum: number; count: number; buckets: Record<number, number> }> = | ||
| new Map(); | ||
|
|
||
| // Standard buckets for latency (seconds) | ||
| private defaultBuckets = [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60]; | ||
|
|
||
| public isEnabled(): boolean { | ||
| return process.env.NEMOCLAW_METRICS_ENABLED === "true"; | ||
| } | ||
|
|
||
| public incrementCounter(name: string, labels: Record<string, string> = {}): void { | ||
| if (!this.isEnabled()) return; | ||
| const key = this.formatKey(name, labels); | ||
| this.counters.set(key, (this.counters.get(key) || 0) + 1); | ||
| } | ||
|
|
||
| public observeHistogram( | ||
| name: string, | ||
| value: number, | ||
| labels: Record<string, string> = {}, | ||
| buckets = this.defaultBuckets, | ||
| ): void { | ||
| if (!this.isEnabled()) return; | ||
| const key = this.formatKey(name, labels); | ||
| let hist = this.histograms.get(key); | ||
| if (!hist) { | ||
| hist = { sum: 0, count: 0, buckets: {} }; | ||
| buckets.forEach((b) => (hist!.buckets[b] = 0)); | ||
| this.histograms.set(key, hist); | ||
| } | ||
|
|
||
| hist.sum += value; | ||
| hist.count += 1; | ||
| buckets.forEach((b) => { | ||
| if (value <= b) { | ||
| hist!.buckets[b] = (hist!.buckets[b] || 0) + 1; | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| public getPrometheusMetrics(): string { | ||
| let output = ""; | ||
|
|
||
| // Export counters | ||
| for (const [key, value] of this.counters.entries()) { | ||
| const [name, labelStr] = this.parseKey(key); | ||
| output += `# HELP ${name} Total count of ${name}\n`; | ||
| output += `# TYPE ${name} counter\n`; | ||
| output += `${name}${labelStr} ${value}\n\n`; | ||
| } | ||
|
|
||
| // Export histograms | ||
| for (const [key, hist] of this.histograms.entries()) { | ||
| const [name, labelStr] = this.parseKey(key); | ||
| output += `# HELP ${name} Latency histogram for ${name}\n`; | ||
| output += `# TYPE ${name} histogram\n`; | ||
|
|
||
| const sortedBuckets = Object.keys(hist.buckets) | ||
| .map(Number) | ||
| .sort((a, b) => a - b); | ||
| const labelsBase = labelStr.length > 2 ? labelStr.slice(1, -1) + "," : ""; | ||
|
|
||
| sortedBuckets.forEach((b) => { | ||
| output += `${name}_bucket{${labelsBase}le="${b === Infinity ? "+Inf" : b}"} ${hist.buckets[b]}\n`; | ||
| }); | ||
| output += `${name}_bucket{${labelsBase}le="+Inf"} ${hist.count}\n`; | ||
| output += `${name}_sum${labelStr} ${hist.sum}\n`; | ||
| output += `${name}_count${labelStr} ${hist.count}\n\n`; | ||
| } | ||
|
|
||
| return output; | ||
| } | ||
|
|
||
| private formatKey(name: string, labels: Record<string, string>): string { | ||
| const labelPairs = Object.entries(labels) | ||
| .map(([k, v]) => `${k}="${v}"`) | ||
| .join(","); | ||
| return labelPairs ? `${name}{${labelPairs}}` : name; | ||
| } | ||
|
|
||
| private parseKey(key: string): [string, string] { | ||
| const braceIdx = key.indexOf("{"); | ||
| if (braceIdx === -1) return [key, ""]; | ||
| return [key.slice(0, braceIdx), key.slice(braceIdx)]; | ||
| } | ||
| } | ||
|
|
||
| export const metrics = new MetricsRegistry(); | ||
|
|
||
| /** | ||
| * Helper to measure execution time of a promise. | ||
| */ | ||
| export async function observeLatency<T>( | ||
| name: string, | ||
| labels: Record<string, string>, | ||
| fn: () => Promise<T>, | ||
| ): Promise<T> { | ||
| if (!metrics.isEnabled()) return fn(); | ||
|
|
||
| const start = process.hrtime.bigint(); | ||
| try { | ||
| const result = await fn(); | ||
| const end = process.hrtime.bigint(); | ||
| const durationSec = Number(end - start) / 1e9; | ||
|
|
||
| metrics.observeHistogram(`${name}_latency_seconds`, durationSec, { | ||
| ...labels, | ||
| status: "success", | ||
| }); | ||
| metrics.incrementCounter(`${name}_total`, { ...labels, status: "success" }); | ||
|
|
||
| return result; | ||
| } catch (error) { | ||
| const end = process.hrtime.bigint(); | ||
| const durationSec = Number(end - start) / 1e9; | ||
|
|
||
| metrics.observeHistogram(`${name}_latency_seconds`, durationSec, { | ||
| ...labels, | ||
| status: "error", | ||
| }); | ||
| metrics.incrementCounter(`${name}_total`, { ...labels, status: "error" }); | ||
|
|
||
| throw error; | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,8 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import { observeLatency } from "../observability/metrics.js"; | ||
|
|
||
| export interface ValidationResult { | ||
| valid: boolean; | ||
| models: string[]; | ||
|
|
@@ -10,6 +12,17 @@ export interface ValidationResult { | |
| export async function validateApiKey( | ||
| apiKey: string, | ||
| endpointUrl: string, | ||
| ): Promise<ValidationResult> { | ||
| return observeLatency( | ||
| "tool_api_validate", | ||
| { endpoint: endpointUrl }, | ||
| () => validateApiKeyInternal(apiKey, endpointUrl), | ||
| ); | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Metric status may not reflect validation outcome. The
If the intent is to track whether the API call completed (regardless of validity), this is fine. If you want metrics to reflect validation success/failure, consider either:
🤖 Prompt for AI Agents |
||
|
|
||
| async function validateApiKeyInternal( | ||
| apiKey: string, | ||
| endpointUrl: string, | ||
| ): Promise<ValidationResult> { | ||
| const url = `${endpointUrl.replace(/\/+$/, "")}/models`; | ||
| const controller = new AbortController(); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| const test = require("node:test"); | ||
| const assert = require("node:assert"); | ||
| const path = require("node:path"); | ||
|
|
||
| // Load the compiled metrics module | ||
| const metricsPath = path.resolve(__dirname, "../nemoclaw/dist/observability/metrics.js"); | ||
| const { metrics, observeLatency } = require(metricsPath); | ||
|
|
||
| // Enable metrics for testing | ||
| process.env.NEMOCLAW_METRICS_ENABLED = "true"; | ||
|
|
||
| test("MetricsRegistry stores and exports counters", () => { | ||
| metrics.incrementCounter("test_counter", { foo: "bar" }); | ||
| const output = metrics.getPrometheusMetrics(); | ||
|
|
||
| assert.match(output, /# TYPE test_counter counter/); | ||
| assert.match(output, /test_counter\{foo="bar"\} 1/); | ||
| }); | ||
|
|
||
| test("MetricsRegistry stores and exports histograms", () => { | ||
| metrics.observeHistogram("test_hist", 0.5, { abc: "123" }); | ||
| const output = metrics.getPrometheusMetrics(); | ||
|
|
||
| assert.match(output, /# TYPE test_hist histogram/); | ||
| assert.match(output, /test_hist_bucket\{abc="123",le="0\.5"\} 1/); | ||
| assert.match(output, /test_hist_sum\{abc="123"\} 0\.5/); | ||
| assert.match(output, /test_hist_count\{abc="123"\} 1/); | ||
| }); | ||
|
|
||
| test("observeLatency tracks success metrics", async () => { | ||
| const result = await observeLatency("test_op", { op: "success" }, async () => { | ||
| return "done"; | ||
| }); | ||
|
|
||
| assert.strictEqual(result, "done"); | ||
| const output = metrics.getPrometheusMetrics(); | ||
|
|
||
| assert.match(output, /test_op_total\{op="success",status="success"\} 1/); | ||
| assert.match(output, /test_op_latency_seconds_count\{op="success",status="success"\} 1/); | ||
| }); | ||
|
|
||
| test("observeLatency tracks error metrics", async () => { | ||
| try { | ||
| await observeLatency("test_op_err", { op: "fail" }, async () => { | ||
| throw new Error("oops"); | ||
| }); | ||
| } catch (err) { | ||
| assert.strictEqual(err.message, "oops"); | ||
| } | ||
|
|
||
| const output = metrics.getPrometheusMetrics(); | ||
| assert.match(output, /test_op_err_total\{op="fail",status="error"\} 1/); | ||
| assert.match(output, /test_op_err_latency_seconds_count\{op="fail",status="error"\} 1/); | ||
| }); |
Uh oh!
There was an error while loading. Please reload this page.