diff --git a/scripts/build/browser-safe-exports.test.ts b/scripts/build/browser-safe-exports.test.ts index b87da6c770..844eb21301 100644 --- a/scripts/build/browser-safe-exports.test.ts +++ b/scripts/build/browser-safe-exports.test.ts @@ -76,3 +76,26 @@ Deno.test("browser error adapters do not retain Node imports", async () => { "the useAgent browser bundle must not retain a Node builtin import", ); }); + +Deno.test("the public observability barrel does not eagerly import Node v8", async () => { + const output = await new Deno.Command(Deno.execPath(), { + args: [ + "bundle", + "--platform=browser", + "--no-check", + "src/observability/index.ts", + ], + cwd: new URL("../../", import.meta.url), + stdin: "null", + stdout: "piped", + stderr: "piped", + }).output(); + const stderr = new TextDecoder().decode(output.stderr); + assert(output.success, `observability browser bundle failed:\n${stderr}`); + + const bundle = new TextDecoder().decode(output.stdout); + assert( + !/\b(?:from|import)\s*["']node:v8["']/.test(bundle), + "the public observability barrel must not retain a browser-eager node:v8 import", + ); +}); diff --git a/scripts/lint/test-typecheck-baseline.json b/scripts/lint/test-typecheck-baseline.json index 45e68b4494..000a4aa0f6 100644 --- a/scripts/lint/test-typecheck-baseline.json +++ b/scripts/lint/test-typecheck-baseline.json @@ -38,12 +38,6 @@ "src/middleware/core/pipeline/composer.test.ts", "src/modules/import-map/preloader.test.ts", "src/modules/react-loader/ssr-module-loader.stress.test.ts", - "src/observability/auto-instrument.test.ts", - "src/observability/auto-instrument/wrappers.test.ts", - "src/observability/instruments/error-instruments.test.ts", - "src/observability/log-buffer.test.ts", - "src/observability/metrics/recorder.test.ts", - "src/observability/tracing/span-operations.test.ts", "src/platform/adapters/fs/veryfront/adapter-helpers.test.ts", "src/platform/adapters/fs/veryfront/directory-operations.test.ts", "src/platform/adapters/redis/node.test.ts", diff --git a/src/extensions/observability/application-error-reporter.test.ts b/src/extensions/observability/application-error-reporter.test.ts new file mode 100644 index 0000000000..34ebdb79d3 --- /dev/null +++ b/src/extensions/observability/application-error-reporter.test.ts @@ -0,0 +1,41 @@ +import type { + ApplicationErrorContext as CanonicalApplicationErrorContext, +} from "#veryfront/observability/application-error-contract.ts"; +import { + type ApplicationErrorContext, + type ApplicationErrorReporterInitializer, + ApplicationErrorReporterInitializerName, +} from "./index.ts"; + +Deno.test("application-error initializer re-exports the canonical context contract", async () => { + const extensionContext: ApplicationErrorContext = { + boundary: "worker.request", + processRole: "worker", + }; + const canonicalContext: CanonicalApplicationErrorContext = extensionContext; + const roundTripContext: ApplicationErrorContext = canonicalContext; + let capturedProcessRole: string | undefined; + + const initializer: ApplicationErrorReporterInitializer = { + initialize: () => ({ + reporter: { + capture(_error, context) { + capturedProcessRole = context.processRole; + return "event-id"; + }, + flush: () => Promise.resolve(true), + }, + dispose() {}, + }), + }; + const session = await initializer.initialize({ serviceName: "worker" }); + if (!session) throw new Error("initializer unexpectedly disabled reporting"); + + const eventId = session.reporter.capture(new Error("failed"), roundTripContext); + if (eventId !== "event-id" || capturedProcessRole !== "worker") { + throw new Error("canonical application-error context was not preserved"); + } + if (ApplicationErrorReporterInitializerName !== "ApplicationErrorReporterInitializer") { + throw new Error("application-error initializer contract name changed"); + } +}); diff --git a/src/extensions/observability/application-error-reporter.ts b/src/extensions/observability/application-error-reporter.ts new file mode 100644 index 0000000000..749889889f --- /dev/null +++ b/src/extensions/observability/application-error-reporter.ts @@ -0,0 +1,30 @@ +import type { ApplicationErrorReporter } from "#veryfront/observability/application-error-contract.ts"; + +export type { + ApplicationErrorContext, + ApplicationErrorReporter, +} from "#veryfront/observability/application-error-contract.ts"; + +/** Runtime context passed to an explicitly selected reporter initializer. */ +export type ApplicationErrorReporterInitializationContext = { + serviceName: string; +}; + +/** Reporter and cleanup ownership returned by an application-selected initializer. */ +export type ApplicationErrorReporterSession = { + reporter: ApplicationErrorReporter; + dispose(): void | Promise; +}; + +/** Application-composition contract for an error-reporting implementation. */ +export type ApplicationErrorReporterInitializer = { + initialize( + context: ApplicationErrorReporterInitializationContext, + ): + | ApplicationErrorReporterSession + | undefined + | Promise; +}; + +/** Contract name used when an application composes a reporter through extensions. */ +export const ApplicationErrorReporterInitializerName = "ApplicationErrorReporterInitializer"; diff --git a/src/extensions/observability/index.ts b/src/extensions/observability/index.ts index 8217c155e4..3d29d54e3b 100644 --- a/src/extensions/observability/index.ts +++ b/src/extensions/observability/index.ts @@ -15,3 +15,11 @@ export type { NodeTelemetryProvider, } from "./node-telemetry-provider.ts"; export { NodeTelemetryProviderName } from "./node-telemetry-provider.ts"; +export type { + ApplicationErrorContext, + ApplicationErrorReporter, + ApplicationErrorReporterInitializationContext, + ApplicationErrorReporterInitializer, + ApplicationErrorReporterSession, +} from "./application-error-reporter.ts"; +export { ApplicationErrorReporterInitializerName } from "./application-error-reporter.ts"; diff --git a/src/middleware/core/pipeline/pipeline.test.ts b/src/middleware/core/pipeline/pipeline.test.ts index fda06161ad..a700754690 100644 --- a/src/middleware/core/pipeline/pipeline.test.ts +++ b/src/middleware/core/pipeline/pipeline.test.ts @@ -340,6 +340,40 @@ describe("middleware/core/pipeline/MiddlewarePipeline", () => { const pipeline = new MiddlewarePipeline(); let cleanupCount = 0; + pipeline.onTeardown(() => { + cleanupCount++; + }); + pipeline.use(() => { + throw new Error("middleware blew up"); + }); + // The pipeline turns a middleware failure into a 500, so the rejection + // has to come from the error response itself failing to build. + const adapter = { + env: { + get() { + throw new Error("adapter unavailable"); + }, + }, + }; + + await assertRejects( + () => + pipeline.execute( + new Request("http://localhost/"), + undefined, + undefined, + adapter as unknown as Parameters[3], + ), + Error, + "adapter unavailable", + ); + assertEquals(cleanupCount, 1); + }); + + it("should keep execution working when the tracer provider fails", async () => { + const pipeline = new MiddlewarePipeline(); + let cleanupCount = 0; + pipeline.onTeardown(() => { cleanupCount++; }); @@ -349,11 +383,10 @@ describe("middleware/core/pipeline/MiddlewarePipeline", () => { }, }); - await assertRejects( - () => pipeline.execute(new Request("http://localhost/")), - Error, - "tracing unavailable", - ); + const response = await pipeline.execute(new Request("http://localhost/")); + + assertEquals(response.status, 404); + assertEquals(await response.text(), "Not Found"); assertEquals(cleanupCount, 1); }); diff --git a/src/observability/README.md b/src/observability/README.md index e6f124d61a..a86b0ba6c0 100644 --- a/src/observability/README.md +++ b/src/observability/README.md @@ -1,595 +1,332 @@ -# Observability Module +# Observability reference -The Observability module provides comprehensive OpenTelemetry integration for distributed tracing, metrics collection, and automatic instrumentation across the entire framework. +The observability module defines Veryfront's tracing, metrics, instrumentation, +request profiling, and development-diagnostics contracts. -## Import Map Alias +## Public entry points -```typescript -// Using import map alias (recommended) -import { - initAutoInstrumentation, - initMetrics, - initTracing, - instrumentHttpHandler, - recordHttpRequest, - startSpan, -} from "#observability"; - -// Using barrel file -import { initTracing, startSpan } from "./observability/index.ts"; -``` - -## Public API Overview - -The Observability module exports: - -- **Tracing utilities** - OpenTelemetry distributed tracing and span management -- **Metrics utilities** - Performance metrics for HTTP, cache, rendering, RSC, builds -- **Auto-instrumentation** - Automatic instrumentation for handlers, fetch, React -- **Configuration** - Simple setup for common observability backends - -## File Structure - -``` -observability/ -├── index.ts # Public API (barrel file) ← USE THIS -├── README.md # This file -├── tracing/ # Distributed tracing -│ ├── index.ts -│ ├── init.ts # Tracing initialization -│ ├── spans.ts # Span management -│ ├── context.ts # Context propagation -│ └── config.ts # Tracing configuration -├── metrics/ # Metrics collection -│ ├── index.ts -│ ├── init.ts # Metrics initialization -│ ├── http.ts # HTTP metrics -│ ├── cache.ts # Cache metrics -│ ├── render.ts # Rendering metrics -│ ├── rsc.ts # RSC metrics -│ ├── build.ts # Build metrics -│ └── manager.ts # Metrics manager -└── auto-instrument/ # Auto-instrumentation - ├── index.ts - ├── orchestrator.ts # Instrumentation orchestrator - ├── http.ts # HTTP instrumentation - ├── fetch.ts # Fetch instrumentation - ├── react.ts # React instrumentation - └── error.ts # Error instrumentation -``` - -## Quick Start - -### Basic Setup - -```ts -import { initAutoInstrumentation, initMetrics, initTracing } from "#observability"; - -// Initialize observability (typically in your main server file) -await initTracing({ - serviceName: "my-veryfront-app", - endpoint: "http://localhost:4318", // OTLP endpoint - enabled: true, -}); - -await initMetrics({ - serviceName: "my-veryfront-app", - endpoint: "http://localhost:4318", - enabled: true, -}); - -// Enable automatic instrumentation -await initAutoInstrumentation({ - tracing: true, - metrics: true, - instruments: ["http", "fetch", "react", "error"], -}); -``` - -### Manual Tracing +| Specifier | Contract | +| ------------------------------------ | ------------------------------------------------------------------------------------------------- | +| `veryfront/observability` | Stable tracing, metrics, instrumentation, profiling, diagnostics, and application-error contracts | +| `veryfront/observability/otlp-setup` | Lower-level shim-based tracing helpers used by framework integrations | ```ts -import { endSpan, setSpanAttributes, startSpan, withSpan } from "#observability"; - -// Manual span management -async function processRequest(req: Request) { - const span = startSpan("process-request"); - - try { - setSpanAttributes(span, { - "http.method": req.method, - "http.url": req.url, - }); - - const result = await doWork(); - return result; - } finally { - endSpan(span); - } -} - -// Using withSpan helper (recommended) -async function processRequest(req: Request) { - return await withSpan("process-request", async (span) => { - setSpanAttributes(span, { - "http.method": req.method, - "http.url": req.url, - }); - - return await doWork(); - }); -} +import { initTracing, recordHttpRequest, withSpan } from "veryfront/observability"; ``` -### Manual Metrics +Core uses an OpenTelemetry-compatible shim. Without an observability extension, +the shim is a no-op and traced callbacks still run. Exporter creation, provider +wiring, flushing, and resource shutdown belong to the active observability +extension and bootstrap lifecycle. -```ts -import { - recordCacheGet, - recordHttpRequest, - recordHttpRequestComplete, - recordRender, -} from "#observability"; - -// Record HTTP request start -const requestId = recordHttpRequest("GET", "/api/users"); - -// Do work... -const response = await handleRequest(); - -// Record completion -recordHttpRequestComplete(requestId, { - statusCode: 200, - duration: 150, -}); - -// Record cache operations -recordCacheGet("user:123", true); // hit -recordCacheSet("user:123", 1024); // size in bytes - -// Record render -recordRender("page:/users", 250, false); // duration, isRSC -``` - -## Distributed Tracing +## Tracing ### Configuration -```ts -interface TracingConfig { - serviceName: string; - endpoint: string; // OTLP endpoint (e.g., 'http://localhost:4318') - enabled: boolean; - sampleRate?: number; // 0.0 to 1.0 (default: 1.0) - exporterType?: "otlp" | "console" | "jaeger"; - headers?: Record; // Auth headers -} - -await initTracing({ - serviceName: "veryfront-app", - endpoint: process.env.OTLP_ENDPOINT, - enabled: process.env.NODE_ENV === "production", - sampleRate: 0.1, // Sample 10% of traces - headers: { - "Authorization": `Bearer ${process.env.OTLP_TOKEN}`, - }, -}); -``` - -### Span Management - -```ts -import { - addSpanEvent, - createChildSpan, - endSpan, - setSpanAttributes, - startSpan, -} from "#observability"; - -// Start root span -const rootSpan = startSpan("http.request"); - -// Add attributes -setSpanAttributes(rootSpan, { - "http.method": "GET", - "http.url": "/api/users", - "http.user_agent": req.headers.get("user-agent"), -}); - -// Add events -addSpanEvent(rootSpan, "validation.start"); -await validateRequest(req); -addSpanEvent(rootSpan, "validation.complete"); - -// Create child span -const dbSpan = createChildSpan(rootSpan, "db.query"); -setSpanAttributes(dbSpan, { - "db.system": "postgresql", - "db.statement": "SELECT * FROM users", -}); -await db.query("SELECT * FROM users"); -endSpan(dbSpan); - -// End root span -endSpan(rootSpan); -``` - -### Context Propagation - -```ts -import { extractContext, getActiveContext, injectContext, withActiveSpan } from "#observability"; - -// Extract context from incoming request -const context = extractContext(req.headers); - -// Inject context into outgoing request -const headers = new Headers(); -injectContext(headers); -await fetch("https://api.example.com", { headers }); - -// Get current active span -const activeSpan = withActiveSpan((span) => { - console.log("Current span:", span); - return span; -}); -``` - -### Async Span Wrapping - -```ts -import { withSpan } from "#observability"; - -// Automatically creates and ends span -async function fetchUser(id: string) { - return await withSpan("fetch-user", async (span) => { - setSpanAttributes(span, { "user.id": id }); - - const user = await db.users.findById(id); - - if (!user) { - addSpanEvent(span, "user.not_found"); - throw new Error("User not found"); - } - - return user; - }); -} - -// Nested spans work automatically -async function processOrder(orderId: string) { - return await withSpan("process-order", async () => { - const order = await fetchOrder(orderId); // Child span - const user = await fetchUser(order.userId); // Child span - await sendEmail(user.email); // Child span - return { order, user }; - }); -} -``` - -## Metrics Collection - -### Configuration - -```ts -interface MetricsConfig { - serviceName: string; - endpoint: string; - enabled: boolean; - collectInterval?: number; // Collection interval in ms (default: 60000) - exportInterval?: number; // Export interval in ms (default: 60000) - headers?: Record; -} - -await initMetrics({ - serviceName: "veryfront-app", - endpoint: process.env.OTLP_ENDPOINT, - enabled: true, - collectInterval: 30000, // Collect every 30s - exportInterval: 60000, // Export every 60s -}); -``` - -### HTTP Metrics - -```ts -import { recordHttpRequest, recordHttpRequestComplete } from "#observability"; - -// Start recording request -const requestId = recordHttpRequest("POST", "/api/users", { - userAgent: req.headers.get("user-agent"), - remoteAddr: req.headers.get("x-forwarded-for"), -}); - -// Handle request -const response = await handleRequest(req); - -// Record completion -recordHttpRequestComplete(requestId, { - statusCode: response.status, - duration: performance.now() - startTime, - bytesWritten: response.headers.get("content-length"), -}); -``` - -### Cache Metrics - -```ts -import { - recordCacheGet, - recordCacheInvalidate, - recordCacheSet, - setCacheSize, -} from "#observability"; - -// Record cache operations -recordCacheGet("user:123", true); // Cache hit -recordCacheGet("user:456", false); // Cache miss - -recordCacheSet("user:789", 2048); // Set with size in bytes - -recordCacheInvalidate("user:*", 10); // Invalidated 10 entries - -// Update cache size -setCacheSize(1024 * 1024 * 10); // 10 MB -``` - -### Rendering Metrics - -```ts -import { recordRender, recordRenderError, recordRSCRender, recordRSCStream } from "#observability"; - -// SSR rendering -recordRender("page:/users", 250, false); // path, duration, isRSC - -// RSC rendering -recordRSCRender("component:UserList", 120); -recordRSCStream("payload:users", 5120); // size in bytes - -// Render errors -recordRenderError("page:/users", "TypeError: Cannot read property..."); -``` - -### Build Metrics - -```ts -import { recordBuild, recordBundle, recordDataFetch } from "#observability"; - -// Build process -recordBuild(45000, true); // duration, success - -// Bundle generation -recordBundle("client", 512000, 2500); // target, size, duration - -// Data fetching (SSG) -recordDataFetch("users", 150, true); // source, duration, success -recordDataFetchError("posts", "Network timeout"); -``` - -## Auto-Instrumentation +`initTracing(config?, adapter?)` accepts a partial `TracingConfig`: + +| Field | Type | Default | +| ------------- | --------------------------------------------- | ------------- | +| `enabled` | `boolean` | `false` | +| `exporter` | `"jaeger" \| "zipkin" \| "otlp" \| "console"` | `"console"` | +| `endpoint` | `string` | unset | +| `serviceName` | `string` | `"veryfront"` | +| `sampleRate` | `number` | `1` | +| `debug` | `boolean` | `false` | + +The runtime adapter or host environment can provide: + +- `VERYFRONT_OTEL=1` +- `OTEL_TRACES_ENABLED=true` +- `OTEL_SERVICE_NAME` +- `OTEL_EXPORTER_OTLP_ENDPOINT` +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` +- `OTEL_TRACES_EXPORTER` + +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` takes precedence over the generic +`OTEL_EXPORTER_OTLP_ENDPOINT`. Caller configuration is validated before +initialization. When an explicit runtime adapter owns environment access and +that access fails, caller configuration is preserved; core does not fall back +to a different host environment. + +The core manager records configuration and binds to the active shim provider. +Exporter-specific behavior, including sampling, is implemented by the provider +extension. Concurrent initialization callers share one readiness promise. +Shutdown invalidates an in-flight initialization, so obsolete asynchronous work +cannot reinstall tracing state. + +### Functions + +| Function | Contract | +| ----------------------------------------- | ---------------------------------------------------------------- | +| `initTracing(config?, adapter?)` | Initializes the core tracing manager once | +| `isTracingEnabled()` | Returns whether the manager has a tracer | +| `isTracingDegraded()` | Returns whether initialization failed | +| `shutdownTracing()` | Signals core shutdown; exporter teardown remains extension-owned | +| `startSpan(name, options?)` | Returns a `Span` or `null` | +| `endSpan(span, error?)` | Records status and ends a span; accepts `null` | +| `setSpanAttributes(span, attributes)` | Adds string, number, or boolean attributes | +| `addSpanEvent(span, name, attributes?)` | Adds an event | +| `createChildSpan(parent, name, options?)` | Creates a child span or a root span when `parent` is `null` | +| `extractContext(headers)` | Extracts a tracing context from headers | +| `injectContext(context, headers)` | Injects an explicit context into headers | +| `getActiveContext()` | Returns the current context when available | +| `withActiveSpan(span, asyncFn)` | Runs an async callback with `span` active | +| `withSpan(name, asyncFn, options?)` | Runs an async callback and completes its span | +| `withSpanSync(name, fn, options?)` | Synchronous form of `withSpan` | + +`SpanOptions` supports `kind`, `attributes`, and `parent`. `kind` is one of +`internal`, `server`, `client`, `producer`, or `consumer`. `parent` may be a +`Span` or a tracing `Context`. + +`SpanNames` contains the framework's standard span-name constants. + +## OTLP helper entry point + +`veryfront/observability/otlp-setup` uses the shim provider directly. Its +`withSpan` callback receives a non-null span, which is a no-op span when no real +provider is installed. + +| Function | Contract | +| ------------------------------------------------ | --------------------------------------------------------------------- | +| `withSpan(name, asyncFn, attributes?, options?)` | Runs an async callback in an active span context | +| `withSpanSync(name, fn, attributes?, options?)` | Runs a synchronous callback in an active span context | +| `startServerSpan(method, path, parentContext?)` | Returns `{ span, context }`, or `null` when span startup fails | +| `endServerSpan(span, statusCode, error?)` | Records HTTP status and ends the server span | +| `extractContext(headers)` | Extracts from incoming headers | +| `injectContext(headers)` | Injects the active context into outgoing headers | +| `withContext(context, asyncFn)` | Runs a callback in an explicit context | +| `getTraceContext()` | Returns active `traceId` and `spanId`, or `{}` | +| `setActiveSpanAttributes(attributes)` | Adds attributes to the active span | +| `initializeOTLP()` | Marks the compatibility wrapper initialized | +| `shutdownOTLP()` | Delegates shutdown to the extension lifecycle | +| `isOTLPEnabled()` | Reports whether `initializeOTLP()` was called, not exporter readiness | + +`WithSpanOptions.kind` accepts the exported numeric `SpanKind` values. + +## Metrics ### Configuration -```ts -interface AutoInstrumentConfig { - tracing: boolean; - metrics: boolean; - instruments?: ("http" | "fetch" | "react" | "error")[]; -} - -await initAutoInstrumentation({ - tracing: true, - metrics: true, - instruments: ["http", "fetch", "react", "error"], -}); -``` - -### HTTP Handler Instrumentation - -```ts -import { instrumentHttpHandler } from "#observability"; - -// Wrap your HTTP handler -const instrumentedHandler = instrumentHttpHandler( - async (req: Request) => { - return new Response("Hello World"); - }, - { - spanName: "http.request", - recordMetrics: true, - }, -); - -// Use with server -Deno.serve(instrumentedHandler); -``` - -### Fetch Instrumentation - -```ts -import { instrumentFetch } from "#observability"; - -// Instrument fetch globally -instrumentFetch(); - -// Now all fetch calls are automatically traced -const response = await fetch("https://api.example.com/users"); -// Creates span: "http.client.fetch" with method, url, status attributes -``` - -### React Render Instrumentation - -```ts -import { instrumentReactRender } from "#observability"; - -// Instrument React rendering -const instrumentedRender = instrumentReactRender( - async (element: React.ReactElement) => { - return await renderToString(element); - }, -); - -// Use instrumented render -const html = await instrumentedRender(); -// Creates span: "react.render" with component name and duration -``` - -### Error Instrumentation - -```ts -import { instrumentErrorHandler } from "#observability"; - -// Wrap error handler -const instrumentedErrorHandler = instrumentErrorHandler( - async (error: Error, req: Request) => { - console.error("Error:", error); - return new Response("Internal Server Error", { status: 500 }); - }, -); - -// Errors automatically create spans and record metrics -``` - -### Batch Instrumentation - -```ts -import { instrumentBatch } from "#observability"; - -// Instrument multiple functions at once -const operations = instrumentBatch({ - fetchUser: async (id: string) => {/* ... */}, - updateUser: async (id: string, data: any) => {/* ... */}, - deleteUser: async (id: string) => {/* ... */}, -}); - -// Each operation now creates spans automatically -await operations.fetchUser("123"); -``` - -## Observability Backends - -### Jaeger - -```ts -await initTracing({ - serviceName: "veryfront-app", - endpoint: "http://localhost:14268/api/traces", - exporterType: "jaeger", - enabled: true, -}); -``` - -### Zipkin - -```ts -await initTracing({ - serviceName: "veryfront-app", - endpoint: "http://localhost:9411/api/v2/spans", - exporterType: "otlp", - enabled: true, -}); -``` - -### Grafana Cloud (OTLP) - -```ts -await initTracing({ - serviceName: "veryfront-app", - endpoint: "https://otlp-gateway-prod-us-east-0.grafana.net/otlp", - enabled: true, - headers: { - "Authorization": `Basic ${btoa(`${instanceId}:${apiToken}`)}`, - }, -}); - -await initMetrics({ - serviceName: "veryfront-app", - endpoint: "https://otlp-gateway-prod-us-east-0.grafana.net/otlp", - enabled: true, - headers: { - "Authorization": `Basic ${btoa(`${instanceId}:${apiToken}`)}`, - }, -}); -``` - -### Honeycomb - -```ts -await initTracing({ - serviceName: "veryfront-app", - endpoint: "https://api.honeycomb.io", - enabled: true, - headers: { - "x-honeycomb-team": process.env.HONEYCOMB_API_KEY, - "x-honeycomb-dataset": "veryfront-traces", - }, -}); -``` - -## Best Practices - -1. **Initialize early** - Call init functions at application startup -2. **Use auto-instrumentation** - Enable for common patterns (HTTP, fetch, React) -3. **Manual spans for business logic** - Use `withSpan` for important operations -4. **Add meaningful attributes** - Include user IDs, request IDs, operation details -5. **Sample in production** - Use `sampleRate` to reduce overhead (e.g., 0.1 = 10%) -6. **Propagate context** - Always extract/inject context for distributed traces -7. **Record metrics consistently** - Use standard metric names and labels -8. **Handle shutdown gracefully** - Call `shutdownTracing()` and `shutdownMetrics()` - -## Performance Tips - -- Auto-instrumentation adds ~1-5ms overhead per operation -- Sampling reduces overhead proportionally (0.1 = 90% reduction) -- Use batch exports to reduce network calls -- Disable in development if not needed -- Use console exporter for debugging (no network overhead) - -## Monitoring Examples - -### SLI/SLO Tracking - -```ts -// Track service level indicators -recordHttpRequestComplete(requestId, { - statusCode: 200, - duration: 120, // < 200ms SLO -}); - -// Query metrics to calculate SLI -const successRate = successfulRequests / totalRequests; -const p95Latency = calculateP95(requestDurations); - -console.log(`Success Rate: ${successRate * 100}% (SLO: 99.9%)`); -console.log(`P95 Latency: ${p95Latency}ms (SLO: 200ms)`); -``` - -### Error Rate Monitoring - -```ts -// Automatic error tracking -instrumentErrorHandler(async (error) => { - // Error automatically recorded in metrics - return handleError(error); -}); - -// Query error rate -const errorRate = errorCount / totalRequests; -if (errorRate > 0.01) { // > 1% error rate - alert("High error rate detected!"); -} -``` - -## Related Modules - -- **#server** - Server implementation with observability hooks -- **#rendering** - SSR/RSC rendering with automatic tracing -- **#api** - API routes with HTTP metrics -- **#middleware** - Middleware pipeline with instrumentation - -## References - -- [OpenTelemetry Documentation](https://opentelemetry.io/docs/) -- [Jaeger Documentation](https://www.jaegertracing.io/docs/) -- [Grafana Cloud OTLP](https://grafana.com/docs/grafana-cloud/send-data/otlp/) -- [Honeycomb Documentation](https://docs.honeycomb.io/) +`initMetrics(config?, adapter?)` accepts a partial `MetricsConfig`: + +| Field | Type | Default | +| ----------------- | ------------------------------------- | -------------------- | +| `enabled` | `boolean` | `false` | +| `exporter` | `"prometheus" \| "otlp" \| "console"` | `"console"` | +| `endpoint` | `string` | unset | +| `prefix` | `string` | `"veryfront"` | +| `collectInterval` | `number` | `60000` milliseconds | +| `debug` | `boolean` | `false` | + +The runtime adapter or host environment can provide `VERYFRONT_OTEL`, +`OTEL_METRICS_ENABLED`, `OTEL_EXPORTER_OTLP_ENDPOINT`, +`OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, and `OTEL_METRICS_EXPORTER`. +The metrics-specific endpoint takes precedence over the generic endpoint. +Configuration values are type-checked; `collectInterval` must be a positive +integer within the portable JavaScript timer range. + +The core metrics manager requires a metrics API installed by an observability +extension. Without one, recorders update their in-process runtime state and +external instruments remain disabled. + +### Functions + +All duration arguments are milliseconds. Attributes are +`Record`. + +| Function | Signature summary | +| ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------- | +| `recordHttpRequest` | `(attributes?) => void` | +| `recordHttpRequestComplete` | `(durationMs, attributes?) => void` | +| `recordCacheGet` | `(hit, attributes?) => void` | +| `recordCacheSet` | `(attributes?) => void` | +| `recordCacheInvalidate` | `(count, attributes?) => void` | +| `setCacheSize` | `(size) => void` | +| `recordRender`, `recordRSCRender`, `recordRSCStream` | `(durationMs, attributes?) => void` | +| `recordRenderError`, `recordRSCError` | `(attributes?) => void` | +| `recordRSCRequest` | `("manifest" \| "page" \| "stream" \| "action", attributes?) => void` | +| `recordBuild`, `recordDataFetch` | `(durationMs, attributes?) => void` | +| `recordBundle` | `(sizeKb, attributes?) => void` | +| `recordDataFetchError`, `recordCorsRejection`, `recordSecurityHeaders`, `recordErrorCount` | `(attributes?) => void` | +| `getMetricsState()` | Returns initialization, cache-size, and active-request state | +| `isMetricsEnabled()` | Returns whether a real meter is installed | +| `shutdownMetrics()` | Signals core shutdown; exporter teardown remains extension-owned | + +Non-finite and negative measurements are normalized before recording. Active +request and cache-size state is clamped at zero. Instrument failures are +isolated from application work. + +## Instrumentation wrappers + +`initAutoInstrumentation(config?, adapter?)` initializes the configured tracing +and metrics managers. It does not replace global functions. Apply the exported +wrappers explicitly. + +`AutoInstrumentConfig` contains optional `tracing`, `metrics`, +`instrumentHttp`, `instrumentFetch`, `instrumentReact`, and `captureErrors` +fields. The four instrumentation flags are configuration metadata; wrapper +installation remains explicit. + +| Function | Contract | +| --------------------------------------------------- | -------------------------------------------------------------------------- | +| `instrumentHttpHandler(handler)` | Returns an async request handler with server-span tracing | +| `instrumentFetch(baseFetch?)` | Returns a fetch-compatible function; it does not mutate `globalThis.fetch` | +| `instrumentReactRender(renderFn, componentName)` | Traces one synchronous or asynchronous render | +| `instrumentErrorHandler(handler, captureToSpan?)` | Optionally captures an error before invoking the handler | +| `instrument(fn, spanName, options?)` | Wraps an async function and preserves its argument/result types | +| `instrumentSync(fn, spanName, options?)` | Synchronous form of `instrument` | +| `instrumentBatch(name, items, processor, options?)` | Processes sequential batches, with items in each batch run concurrently | +| `isAutoInstrumentEnabled()` | Reports whether the initializer has completed | + +`instrumentBatch` defaults to a batch size of 10 and rejects non-positive or +non-integer batch sizes. + +Initialization snapshots nested tracing and metrics configuration. Concurrent +callers share one readiness promise, and test lifecycle resets cannot be +overwritten by an obsolete initialization. + +## Service tracer adapter + +`createOpenTelemetryServiceTracer(options)` adapts injected OpenTelemetry trace +and context APIs to the service tracer contract. The returned object provides: + +- `tracer.startSpan`, `tracer.scope`, `tracer.wrap`, and `tracer.trace` +- `setActiveSpanAttributes(attributes)` +- `getTraceContext()` + +Async wrappers keep spans open until their returned promise settles while +preserving the exact returned promise or thenable object. Telemetry recording +failures do not replace completed application results or failures. + +## Application error reporting + +Core defines `ApplicationErrorReporterInitializer`, the active reporter +lifecycle, and the bounded capture/flush boundary. It contains no Sentry SDK, +configuration, environment-variable handling, or vendor loader. With no +explicitly composed initializer, application-error reporting is disabled. +Selected initializer and cleanup failures propagate to their lifecycle caller; +overlapping ownership transitions are serialized so a stale reporter cannot +dispose a newer one. + +`captureApplicationError(error, context)` ignores expected request +cancellation. Reporter failures, invalid reporter results, and hostile error or +context values do not replace application control flow. +`flushApplicationErrors(timeoutMs?)` has a strict deadline and returns `false` +for timeout, rejection, exceptions, or an invalid timeout; it never waits for a +non-cooperative reporter after the deadline. + +Concrete reporters are separate extension packages. Sentry configuration and +runtime setup are documented by `@veryfront/ext-observability-sentry`. + +## In-process metrics + +The `metrics` object exposes counters and bounded histogram snapshots for +framework-local diagnostics. Root-level convenience exports also include +`recordApiRequest`, `recordApiRetry`, `recordContentCacheHit`, and +`recordContentNetworkFetch`. + +`metrics.snapshot()` returns a detached snapshot. Histogram boundaries and +counts in returned snapshots are safe for callers to mutate. + +## Request profiling + +The root entry point exports: + +| Function | Contract | +| -------------------------------------------- | --------------------------------------------------------------------- | +| `profilePhase(name, asyncFn)` | Measures and accumulates an async phase in the active request profile | +| `profileSyncPhase(name, fn)` | Synchronous phase measurement | +| `markRequestProfilePhase(name, durationMs?)` | Adds an explicit phase duration | +| `snapshotRequestProfiles()` | Returns retained profile records and the latest sequence | + +Profiling uses async-local request state. The full internal profiler also uses +`VERYFRONT_ENABLE_PERF_PROFILING`, `VERYFRONT_ENABLE_SERVER_TIMING`, and +`VERYFRONT_DISABLE_SLOW_REQUEST_PROFILING`. +Each request retains at most 128 distinct phase names and can produce only one +final profile record. + +## Development diagnostics + +### `ErrorCollector` + +`ErrorCollector({ maxErrors? })` retains development errors by type and category. +`maxErrors` must be a non-negative safe integer; zero keeps notifications active +without retaining entries. Query methods return detached copies. Subscriber +failures do not interrupt collection. Retained messages and stacks are limited +to 1,000 characters; file and slug metadata is also bounded. + +### `LogBuffer` + +`LogBuffer({ maxSize? })` retains structured log entries. `maxSize` must be a +non-negative safe integer. `query`, `tail`, `getAll`, and `toJSON` return +detached copies. `interceptConsole(buffer, source?)` returns a function that +restores the original console methods. Retained messages are limited to 1,000 +characters and source names to 255 characters. + +### `FileLogSubscriber` + +`FileLogConfig` contains: + +| Field | Type | +| ---------- | ----------------------------------------------------- | +| `enabled` | `boolean` | +| `path` | non-empty `string`, at most 4,096 characters | +| `maxSize` | positive byte count or a string such as `"10mb"` | +| `maxFiles` | integer from 1 through 100, including the active file | +| `level` | `"debug" \| "info" \| "warn" \| "error"` | +| `format` | `"json" \| "text"` | + +`FileLogSubscriber` serializes writes, rotates files by size, and exposes +`flush()` and `close()`. Passive subscriber callbacks report and contain write +failures; explicit `flush()` and `close()` reject when writes, durability sync, +or file closure fails. The pending-write queue retains at most 256 entries. A +full queue drops the new entry, retains a bounded failure sample, and makes the +next explicit flush or close reject rather than hiding data loss. At most 16 +individual failures are retained; additional failures are represented by one +omission summary. Concurrent flush callers share the same durability attempt +and outcome. The subscriber requires the Deno file API. + +## Data safety and cardinality + +Telemetry attributes with credential-like keys are replaced with +`[REDACTED]`. Credentials embedded in URL userinfo or sensitive query +parameters are also removed from traced URLs, recorded errors, buffered logs, +and collected development errors. Structured log and error context is copied +and key-redacted before retention. + +Core applies the following limits before retaining values or invoking a +telemetry provider: + +| Surface | Limit | +| ------------------------ | -------------------------------------------- | +| Attributes per operation | 128 | +| Attribute key | 255 characters | +| Attribute string or item | 10,000 characters | +| Attribute array | 128 items; larger arrays become `[REDACTED]` | +| Span or event name | 1,000 characters | +| Structured context depth | 16 levels | +| One structured container | 1,024 entries | +| One structured snapshot | 4,096 visited nodes | +| Retained structured text | 1,000 characters | + +Oversized or hostile structured containers fail closed to `[REDACTED]`; +ordinary accepted values are detached from caller mutation. + +Exception telemetry never evaluates error-field accessors or a configured +`Error.prepareStackTrace` hook. Safe own string-valued data fields preserve a +bounded message, already-available stack, and built-in, aggregate, custom, or +framework error name. A captured platform compatibility check identifies +native errors without consulting mutable global constructors. DOMException +prototype accessors are never invoked: runtimes whose immutable brand check +recognizes DOMException report the conservative name `DOMException` and omit +its inherited message; older Node releases treat it as opaque. Older V8 +releases also omit stacks because requesting even their property descriptor +materializes the lazy stack. Proxies and accessor-backed fields fail closed; +telemetry handling never changes the value thrown back to application code. + +Redaction is defense in depth, not permission to attach secrets. Free-form +values that are not recognizable URLs may still contain sensitive data. Keep +attribute keys bounded and values low-cardinality. Prefer route templates, +operation kinds, and status classes over raw IDs, arbitrary paths, request +bodies, SQL statements, or user-provided text. diff --git a/src/observability/application-errors.test.ts b/src/observability/application-errors.test.ts index d67e8d1078..e285dfb57d 100644 --- a/src/observability/application-errors.test.ts +++ b/src/observability/application-errors.test.ts @@ -1,9 +1,15 @@ -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { + assertEquals, + assertRejects, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { it } from "#veryfront/testing/bdd.ts"; import { type ApplicationErrorContext, captureApplicationError, flushApplicationErrors, + initializeApplicationErrorReporter, setApplicationErrorReporter, } from "./application-errors.ts"; import type { ApplicationErrorContext as SharedApplicationErrorContext } from "./application-error-contract.ts"; @@ -19,11 +25,21 @@ it("application error reporter is optional", async () => { }); it("application error reporter receives unexpected failures and correlation context", async () => { - const captures: Array<{ error: unknown; boundary: string; traceId?: string }> = []; + const captures: Array<{ + error: unknown; + boundary: string; + processRole?: string; + traceId?: string; + }> = []; let flushTimeout: number | undefined; setApplicationErrorReporter({ capture(error, context) { - captures.push({ error, boundary: context.boundary, traceId: context.traceId }); + captures.push({ + error, + boundary: context.boundary, + processRole: context.processRole, + traceId: context.traceId, + }); return "event-id"; }, flush(timeoutMs) { @@ -36,11 +52,17 @@ it("application error reporter receives unexpected failures and correlation cont assertEquals( captureApplicationError(error, { boundary: "renderer.request", + processRole: "renderer", traceId: "trace-1", }), "event-id", ); - assertEquals(captures, [{ error, boundary: "renderer.request", traceId: "trace-1" }]); + assertEquals(captures, [{ + error, + boundary: "renderer.request", + processRole: "renderer", + traceId: "trace-1", + }]); assertEquals(await flushApplicationErrors(1_500), true); assertEquals(flushTimeout, 1_500); }); @@ -73,3 +95,288 @@ it("application error reporter ignores expected cancellation", () => { assertEquals(eventId, undefined); assertEquals(captured, false); }); + +it("application error capture failures never replace application control flow", () => { + const hostile = new Proxy({}, { + getPrototypeOf() { + throw new Error("prototype unavailable"); + }, + }); + setApplicationErrorReporter({ + capture() { + throw new Error("reporter unavailable"); + }, + flush: () => Promise.resolve(true), + }); + + assertEquals( + captureApplicationError(hostile, { boundary: "renderer.request" }), + undefined, + ); +}); + +it("application error flush is strictly bounded and fail-open", async () => { + setApplicationErrorReporter({ + capture: () => undefined, + flush: () => new Promise(() => {}), + }); + + assertEquals(await flushApplicationErrors(5), false); + + setApplicationErrorReporter({ + capture: () => undefined, + flush: () => Promise.reject(new Error("transport unavailable")), + }); + assertEquals(await flushApplicationErrors(5), false); + assertEquals(await flushApplicationErrors(-1), false); +}); + +it("application error initialization is explicitly disabled without a selected initializer", async () => { + const lifecycle = await initializeApplicationErrorReporter({ + serviceName: "test-service", + }); + + assertEquals(lifecycle.enabled, false); + assertEquals(await lifecycle.flush(), true); + await lifecycle.dispose(); +}); + +it("selected application error initializer failures propagate unchanged", async () => { + const initializationError = new Error("reporter initialization failed"); + const thrown = await assertRejects(() => + initializeApplicationErrorReporter({ + initializer: { + initialize: () => Promise.reject(initializationError), + }, + serviceName: "test-service", + }) + ); + + assertStrictEquals(thrown, initializationError); +}); + +it("application error initialization rejects invalid service identities and direct replacement races", async () => { + await assertRejects( + () => + initializeApplicationErrorReporter({ + serviceName: " invalid ", + }), + TypeError, + "canonical string", + ); + + let resolveInitialization: ((value: undefined) => void) | undefined; + let markInitializationStarted: (() => void) | undefined; + const initializationStarted = new Promise((resolve) => { + markInitializationStarted = resolve; + }); + const pending = initializeApplicationErrorReporter({ + initializer: { + initialize: () => + new Promise((resolve) => { + resolveInitialization = resolve; + markInitializationStarted?.(); + }), + }, + serviceName: "test-service", + }); + assertThrows( + () => setApplicationErrorReporter(undefined), + Error, + "in-flight application-error initialization", + ); + await initializationStarted; + resolveInitialization?.(undefined); + await pending; +}); + +it("application error lifecycle publishes, flushes, and disposes one owned reporter", async () => { + const captures: unknown[] = []; + const flushTimeouts: Array = []; + let disposeCalls = 0; + const lifecycle = await initializeApplicationErrorReporter({ + initializer: { + initialize: ({ serviceName }) => { + assertEquals(serviceName, "test-service"); + return { + reporter: { + capture(error) { + captures.push(error); + return "event-id"; + }, + flush(timeoutMs) { + flushTimeouts.push(timeoutMs); + return Promise.resolve(true); + }, + }, + dispose() { + disposeCalls++; + }, + }; + }, + }, + serviceName: "test-service", + }); + + const error = new Error("reported"); + assertEquals(lifecycle.capture(error, { boundary: "test" }), "event-id"); + assertEquals(await lifecycle.flush(50), true); + await lifecycle.dispose(); + await lifecycle.dispose(); + + assertEquals(captures, [error]); + assertEquals(flushTimeouts, [50]); + assertEquals(disposeCalls, 1); + assertEquals(lifecycle.capture(new Error("stale"), { boundary: "test" }), undefined); +}); + +it("direct reporter replacement hands over ownership from an active lifecycle", async () => { + let disposeCalls = 0; + const lifecycleCaptures: unknown[] = []; + const lifecycle = await initializeApplicationErrorReporter({ + initializer: { + initialize: () => ({ + reporter: { + capture(error) { + lifecycleCaptures.push(error); + return "lifecycle-event"; + }, + flush: () => Promise.resolve(true), + }, + dispose() { + disposeCalls++; + }, + }), + }, + serviceName: "test-service", + }); + + // Mirrors the sentry publish path (sentry.ts, node-sentry.ts) installing its + // reporter while a lifecycle owns the process reporter. + const directCaptures: unknown[] = []; + setApplicationErrorReporter({ + capture(error) { + directCaptures.push(error); + return "direct-event"; + }, + flush: () => Promise.resolve(true), + }); + + const error = new Error("reported after handover"); + assertEquals(captureApplicationError(error, { boundary: "test" }), "direct-event"); + assertEquals(directCaptures, [error]); + assertEquals(lifecycleCaptures, []); + assertEquals(lifecycle.capture(new Error("detached"), { boundary: "test" }), undefined); + assertEquals(await lifecycle.flush(50), true); + + // The detached lifecycle still disposes its own session and must not clear + // the reporter it no longer owns. + await lifecycle.dispose(); + assertEquals(disposeCalls, 1); + assertEquals(captureApplicationError(error, { boundary: "test" }), "direct-event"); + assertEquals(directCaptures.length, 2); +}); + +it("sentry teardown clears the reporter while a lifecycle is still active", async () => { + let disposeCalls = 0; + const lifecycle = await initializeApplicationErrorReporter({ + initializer: { + initialize: () => ({ + reporter: { + capture: () => "lifecycle-event", + flush: () => Promise.resolve(true), + }, + dispose() { + disposeCalls++; + }, + }), + }, + serviceName: "test-service", + }); + + // Mirrors resetSentryForTests() and the node agent service lifecycle reset(). + setApplicationErrorReporter(undefined); + + assertEquals(captureApplicationError(new Error("cleared"), { boundary: "test" }), undefined); + assertEquals(await flushApplicationErrors(50), true); + assertEquals(disposeCalls, 0); + + // A later initialization still disposes the detached session automatically. + const next = await initializeApplicationErrorReporter({ serviceName: "test-service" }); + assertEquals(disposeCalls, 1); + assertEquals(next.enabled, false); + await lifecycle.dispose(); + assertEquals(disposeCalls, 1); +}); + +it("superseded application error initialization disposes stale state before starting its replacement", async () => { + let resolveFirst: + | ((value: { + reporter: { capture(): string; flush(): Promise }; + dispose(): void; + }) => void) + | undefined; + let markFirstStarted: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + let finishStaleDisposal: (() => void) | undefined; + const staleDisposalFinished = new Promise((resolve) => { + finishStaleDisposal = resolve; + }); + let markStaleDisposalStarted: (() => void) | undefined; + const staleDisposalStarted = new Promise((resolve) => { + markStaleDisposalStarted = resolve; + }); + let staleDisposeCalls = 0; + let secondInitializeCalls = 0; + const first = initializeApplicationErrorReporter({ + initializer: { + initialize: () => + new Promise((resolve) => { + resolveFirst = resolve; + markFirstStarted?.(); + }), + }, + serviceName: "first", + }); + await firstStarted; + + const secondPending = initializeApplicationErrorReporter({ + initializer: { + initialize: () => { + secondInitializeCalls++; + return { + reporter: { + capture: () => "second", + flush: () => Promise.resolve(true), + }, + dispose: () => {}, + }; + }, + }, + serviceName: "second", + }); + resolveFirst?.({ + reporter: { + capture: () => "first", + flush: () => Promise.resolve(true), + }, + async dispose() { + staleDisposeCalls++; + markStaleDisposalStarted?.(); + await staleDisposalFinished; + }, + }); + + await staleDisposalStarted; + assertEquals(secondInitializeCalls, 0); + finishStaleDisposal?.(); + const firstLifecycle = await first; + const second = await secondPending; + assertEquals(firstLifecycle.enabled, false); + assertEquals(staleDisposeCalls, 1); + assertEquals(secondInitializeCalls, 1); + assertEquals(second.capture(new Error("current"), { boundary: "test" }), "second"); + await second.dispose(); +}); diff --git a/src/observability/application-errors.ts b/src/observability/application-errors.ts index b52727ea78..4fb8c2888d 100644 --- a/src/observability/application-errors.ts +++ b/src/observability/application-errors.ts @@ -1,20 +1,223 @@ +import { MAX_TIMER_DELAY_MS } from "#veryfront/utils/timer.ts"; +import { sanitizeTelemetryAttributes, sanitizeTelemetryText } from "./telemetry-error.ts"; +import { MAX_APPLICATION_ERROR_CONTEXT_VALUE_LENGTH } from "./limits.ts"; +import { + type ApplicationErrorReporterInitializer, + ApplicationErrorReporterInitializerName, + type ApplicationErrorReporterSession, +} from "#veryfront/extensions/observability/application-error-reporter.ts"; import type { ApplicationErrorContext, ApplicationErrorReporter, } from "./application-error-contract.ts"; +export type { + ApplicationErrorReporterInitializationContext, + ApplicationErrorReporterInitializer, + ApplicationErrorReporterSession, +} from "#veryfront/extensions/observability/application-error-reporter.ts"; export type { ApplicationErrorAttributeValue, ApplicationErrorContext, ApplicationErrorReporter, } from "./application-error-contract.ts"; +export { ApplicationErrorReporterInitializerName }; + +/** Active application-error reporter ownership. */ +export type ApplicationErrorReporterLifecycle = { + readonly enabled: boolean; + capture(error: unknown, context: ApplicationErrorContext): string | undefined; + flush(timeoutMs?: number): Promise; + dispose(): Promise; +}; + +const MAX_APPLICATION_ERROR_SERVICE_NAME_LENGTH = 255; let reporter: ApplicationErrorReporter | undefined; +let reporterOwner: symbol | undefined; +let initializationGeneration = 0; +let activeLifecycle: ApplicationErrorReporterLifecycle | undefined; +let initializationQueue: Promise = Promise.resolve(); +let pendingInitializations = 0; +/** + * Publish an unowned reporter directly, taking over from any active lifecycle. + * + * Direct replacement is a supported sequential handover: an active lifecycle + * keeps its session and stays responsible for disposing it, but stops owning + * the published reporter, so its capture/flush degrade to no-ops and its + * dispose leaves the newly published reporter in place. + * + * Replacement during an in-flight initialization is rejected instead, because + * that initialization publishes unconditionally once it settles and would + * silently discard the reporter installed here. + */ export function setApplicationErrorReporter( nextReporter: ApplicationErrorReporter | undefined, ): void { + if (pendingInitializations > 0) { + throw new Error( + "Wait for the in-flight application-error initialization to settle before replacing its reporter", + ); + } reporter = nextReporter; + reporterOwner = undefined; +} + +function disabledApplicationErrorReporterLifecycle(): ApplicationErrorReporterLifecycle { + return Object.freeze({ + enabled: false, + capture: () => undefined, + flush: () => Promise.resolve(true), + dispose: () => Promise.resolve(), + }); +} + +function captureReporterSession(value: unknown): ApplicationErrorReporterSession { + if (value === null || typeof value !== "object") { + throw new TypeError("Application-error reporter initializer must return a session object"); + } + const session = value as Partial; + const sessionReporter = session.reporter; + if ( + sessionReporter === null || typeof sessionReporter !== "object" || + typeof sessionReporter.capture !== "function" || + typeof sessionReporter.flush !== "function" + ) { + throw new TypeError("Application-error reporter session must contain a valid reporter"); + } + const capture = sessionReporter.capture; + const flush = sessionReporter.flush; + const dispose = session.dispose; + if (typeof dispose !== "function") { + throw new TypeError("Application-error reporter session must provide dispose()"); + } + + return Object.freeze({ + reporter: Object.freeze({ + capture: (error: unknown, context: ApplicationErrorContext) => { + const eventId = Reflect.apply(capture, sessionReporter, [error, context]); + return typeof eventId === "string" ? eventId : undefined; + }, + flush: (timeoutMs?: number) => + Promise.resolve(Reflect.apply(flush, sessionReporter, [timeoutMs])), + }), + dispose: () => Reflect.apply(dispose, session, []), + }); +} + +function enqueueInitialization(operation: () => Promise): Promise { + pendingInitializations++; + const result = initializationQueue.then(operation); + initializationQueue = result.then( + () => undefined, + () => undefined, + ); + return result.finally(() => { + pendingInitializations--; + }); +} + +function snapshotApplicationErrorServiceName(value: unknown): string { + if ( + typeof value !== "string" || value.length === 0 || + value.length > MAX_APPLICATION_ERROR_SERVICE_NAME_LENGTH || + value.trim() !== value || value.normalize("NFC") !== value + ) { + throw new TypeError( + `Application-error service name must be a canonical string of at most ${MAX_APPLICATION_ERROR_SERVICE_NAME_LENGTH} characters`, + ); + } + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code <= 31 || code === 127) { + throw new TypeError("Application-error service name cannot contain control characters"); + } + } + return value; +} + +/** + * Activate an explicitly selected reporter initializer. + * + * The latest initialization owns the process reporter. A superseded + * initialization disposes the session it created instead of publishing stale + * state. Provider initialization and disposal failures are not converted into + * disabled/no-op behavior. + */ +export function initializeApplicationErrorReporter(options: { + initializer?: ApplicationErrorReporterInitializer; + serviceName: string; +}): Promise { + const generation = ++initializationGeneration; + const initializer = options.initializer; + const requestedServiceName = options.serviceName; + return enqueueInitialization(async () => { + if (generation !== initializationGeneration) { + return disabledApplicationErrorReporterLifecycle(); + } + const serviceName = snapshotApplicationErrorServiceName(requestedServiceName); + const previousLifecycle = activeLifecycle; + if (previousLifecycle) await previousLifecycle.dispose(); + if (generation !== initializationGeneration) { + return disabledApplicationErrorReporterLifecycle(); + } + + if (!initializer) { + reporter = undefined; + reporterOwner = undefined; + return disabledApplicationErrorReporterLifecycle(); + } + + if (typeof initializer.initialize !== "function") { + throw new TypeError("Application-error reporter initializer must provide initialize()"); + } + const sessionValue = await Reflect.apply(initializer.initialize, initializer, [{ + serviceName, + }]); + if (sessionValue === undefined) { + reporter = undefined; + reporterOwner = undefined; + return disabledApplicationErrorReporterLifecycle(); + } + const session = captureReporterSession(sessionValue); + + if (generation !== initializationGeneration) { + await session.dispose(); + return disabledApplicationErrorReporterLifecycle(); + } + + const owner = Symbol("application-error-reporter"); + reporter = session.reporter; + reporterOwner = owner; + let disposal: Promise | undefined; + const lifecycle: ApplicationErrorReporterLifecycle = Object.freeze({ + enabled: true, + capture(error, context) { + if (reporterOwner !== owner) return undefined; + return captureApplicationError(error, context); + }, + flush(timeoutMs) { + if (reporterOwner !== owner) return Promise.resolve(true); + return flushApplicationErrors(timeoutMs); + }, + dispose() { + if (disposal) return disposal; + if (reporterOwner === owner) { + reporter = undefined; + reporterOwner = undefined; + } + disposal = Promise.resolve() + .then(() => session.dispose()) + .finally(() => { + if (activeLifecycle === lifecycle) activeLifecycle = undefined; + }); + return disposal; + }, + }); + activeLifecycle = lifecycle; + return lifecycle; + }); } export function captureApplicationError( @@ -22,21 +225,77 @@ export function captureApplicationError( context: ApplicationErrorContext, ): string | undefined { if (isExpectedApplicationError(error)) return undefined; + const currentReporter = reporter; + if (!currentReporter) return undefined; + try { - return reporter?.capture(error, context); + const snapshot = snapshotApplicationErrorContext(context); + return snapshot ? currentReporter.capture(error, snapshot) : undefined; } catch { + // Error reporting is diagnostic and must never replace the application + // failure or response that led to this capture attempt. return undefined; } } -export function flushApplicationErrors(timeoutMs = 2_000): Promise { +export async function flushApplicationErrors(timeoutMs = 2_000): Promise { + const currentReporter = reporter; + if (!currentReporter) return true; + if ( + !Number.isSafeInteger(timeoutMs) || timeoutMs < 0 || + timeoutMs > MAX_TIMER_DELAY_MS + ) { + return false; + } + + let timeout: ReturnType | undefined; + const deadline = new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), timeoutMs); + }); + const pending = Promise.resolve() + .then(() => currentReporter.flush(timeoutMs)) + .then((result) => result === true, () => false); + try { - return reporter?.flush(timeoutMs) ?? Promise.resolve(true); - } catch { - return Promise.resolve(false); + return await Promise.race([pending, deadline]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); } } export function isExpectedApplicationError(error: unknown): boolean { - return error instanceof DOMException && error.name === "AbortError"; + try { + return error instanceof DOMException && error.name === "AbortError"; + } catch { + return false; + } +} + +function snapshotApplicationErrorContext( + context: ApplicationErrorContext, +): ApplicationErrorContext | null { + if (context === null || typeof context !== "object") return null; + const boundary = normalizeContextValue(context.boundary); + if (!boundary) return null; + + const snapshot: ApplicationErrorContext = { boundary }; + for (const key of ["method", "processRole", "requestId", "spanId", "traceId"] as const) { + const value = context[key]; + if (value === undefined) continue; + const normalized = normalizeContextValue(value); + if (normalized) snapshot[key] = normalized; + } + const attributes = sanitizeTelemetryAttributes(context.attributes); + if (attributes && Object.keys(attributes).length > 0) { + snapshot.attributes = Object.freeze(attributes); + } + return Object.freeze(snapshot); +} + +function normalizeContextValue(value: unknown): string | null { + if (typeof value !== "string" || !value.trim()) return null; + return sanitizeTelemetryText( + value.trim(), + MAX_APPLICATION_ERROR_CONTEXT_VALUE_LENGTH, + ); } diff --git a/src/observability/auto-instrument.test.ts b/src/observability/auto-instrument.test.ts index 8e67ecc0d6..8774fef57e 100644 --- a/src/observability/auto-instrument.test.ts +++ b/src/observability/auto-instrument.test.ts @@ -13,7 +13,7 @@ import "#veryfront/schemas/_test-setup.ts"; * - Edge cases and error scenarios */ -import { assertEquals, assertExists } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertExists, assertStrictEquals } from "#veryfront/testing/assert.ts"; import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; import { delay } from "#std/async.ts"; import { scaleMs } from "#veryfront/testing/timing.ts"; @@ -30,6 +30,7 @@ import { isAutoInstrumentEnabled, } from "./auto-instrument/index.ts"; import { __resetAutoInstrumentForTests } from "./auto-instrument/orchestrator.ts"; +import { metricsManager } from "./metrics/manager.ts"; import { createResolvedFetch, createThrowingFetch, @@ -435,6 +436,34 @@ describe("Auto-Instrumentation", () => { }); describe("instrumentReactRender", () => { + it("records and preserves PromiseLike render rejections", async () => { + const recorder = metricsManager.getRecorder(); + assertExists(recorder); + const originalRecordRenderError = recorder.recordRenderError; + let recordedErrors = 0; + recorder.recordRenderError = () => { + recordedErrors++; + }; + const rejection = { reason: "suspended render failed" }; + const thenable = { + then(_resolve: (value: string) => void, reject: (error: unknown) => void): void { + reject(rejection); + }, + } as unknown as Promise; + let caught: unknown; + + try { + await instrumentReactRender(() => thenable, "ThenableComponent"); + } catch (error) { + caught = error; + } finally { + recorder.recordRenderError = originalRecordRenderError; + } + + assertStrictEquals(caught, rejection); + assertEquals(recordedErrors, 1); + }); + it("should instrument synchronous render function", async () => { const renderFn = (): string => "
Hello
"; const result = await instrumentReactRender(renderFn, "TestComponent"); @@ -511,6 +540,21 @@ describe("Auto-Instrumentation", () => { }); describe("instrumentErrorHandler", () => { + it("should invoke the handler when error capture itself fails", async () => { + const handler = (): Response => new Response("handled", { status: 500 }); + const instrumented = instrumentErrorHandler(handler); + const error = new Error("capture failure"); + Object.defineProperty(error, "stack", { + get() { + throw new Error("telemetry stack failure"); + }, + }); + + const response = await instrumented(error); + + assertEquals(await response.text(), "handled"); + }); + it("should instrument error handler with span capture", async () => { const handler = (error: Error): Response => new Response(error.message, { status: 500 }); const instrumented = instrumentErrorHandler(handler, true); @@ -589,7 +633,10 @@ describe("Auto-Instrumentation", () => { Promise.resolve({ userId, action }); const instrumented = instrument(fn, "user.action", { - attributes: ([userId, action]: unknown[]) => ({ userId, action }), + attributes: ([userId, action]: unknown[]) => ({ + userId: String(userId), + action: String(action), + }), }); const result = await instrumented("user-123", "login"); @@ -648,7 +695,7 @@ describe("Auto-Instrumentation", () => { it("should record custom attributes", () => { const fn = (name: string): string => `Hello, ${name}`; const instrumented = instrumentSync(fn, "greet", { - attributes: ([name]: unknown[]) => ({ name }), + attributes: ([name]: unknown[]) => ({ name: String(name) }), }); const result = instrumented("World"); diff --git a/src/observability/auto-instrument/configurator.test.ts b/src/observability/auto-instrument/configurator.test.ts index 801ae96ebc..325ddf96ac 100644 --- a/src/observability/auto-instrument/configurator.test.ts +++ b/src/observability/auto-instrument/configurator.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { DEFAULT_CONFIG, mergeConfig } from "./configurator.ts"; @@ -49,5 +49,32 @@ describe("observability/auto-instrument/configurator", () => { }, ); }); + + it("rejects malformed flags and nested configuration", () => { + assertThrows( + () => mergeConfig({ instrumentHttp: "yes" } as never), + TypeError, + "instrumentHttp", + ); + assertThrows( + () => mergeConfig({ tracing: { enabled: "yes" } } as never), + TypeError, + "tracing enabled", + ); + assertThrows( + () => mergeConfig({ metrics: null } as never), + TypeError, + "metrics config", + ); + }); + + it("returns detached nested configuration snapshots", () => { + const tracing = { enabled: true, exporter: "console" as const }; + const merged = mergeConfig({ tracing }); + + tracing.enabled = false; + + assertEquals(merged.tracing?.enabled, true); + }); }); }); diff --git a/src/observability/auto-instrument/configurator.ts b/src/observability/auto-instrument/configurator.ts index 830e74e193..0bff3f85db 100644 --- a/src/observability/auto-instrument/configurator.ts +++ b/src/observability/auto-instrument/configurator.ts @@ -1,12 +1,58 @@ -import type { AutoInstrumentConfig } from "./types.ts"; +import type { AutoInstrumentConfig, MetricsConfig, TracingConfig } from "./types.ts"; -export const DEFAULT_CONFIG: AutoInstrumentConfig = { +export const DEFAULT_CONFIG: Readonly = Object.freeze({ instrumentHttp: true, instrumentFetch: true, instrumentReact: true, captureErrors: true, -}; +}); export function mergeConfig(config: AutoInstrumentConfig = {}): AutoInstrumentConfig { - return { ...DEFAULT_CONFIG, ...config }; + if (config === null || typeof config !== "object" || Array.isArray(config)) { + throw new TypeError("Auto-instrumentation config must be an object"); + } + + const raw = config as Record; + const merged: AutoInstrumentConfig = {}; + for ( + const key of [ + "instrumentHttp", + "instrumentFetch", + "instrumentReact", + "captureErrors", + ] as const + ) { + const value = raw[key] === undefined ? DEFAULT_CONFIG[key] : raw[key]; + if (typeof value !== "boolean") { + throw new TypeError(`Auto-instrumentation ${key} must be a boolean`); + } + merged[key] = value; + } + + if (raw.tracing !== undefined) { + merged.tracing = snapshotNestedConfig( + raw.tracing, + "tracing", + ); + } + if (raw.metrics !== undefined) { + merged.metrics = snapshotNestedConfig( + raw.metrics, + "metrics", + ); + } + return merged; +} + +function snapshotNestedConfig(value: unknown, name: string): T { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`Auto-instrumentation ${name} config must be an object`); + } + const snapshot = { ...value } as Record; + if (typeof snapshot.enabled !== "boolean") { + throw new TypeError( + `Auto-instrumentation ${name} enabled must be a boolean`, + ); + } + return snapshot as T; } diff --git a/src/observability/auto-instrument/http-instrumentation.test.ts b/src/observability/auto-instrument/http-instrumentation.test.ts new file mode 100644 index 0000000000..0ec4e3659d --- /dev/null +++ b/src/observability/auto-instrument/http-instrumentation.test.ts @@ -0,0 +1,293 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { + _resetShimForTests, + type AttributeValue, + setGlobalContextAccessor, + setGlobalTracerProvider, + type Span, + type Tracer, +} from "../tracing/api-shim.ts"; +import { createInstrumentedFetch, instrumentHttpHandler } from "./http-instrumentation.ts"; + +type SpanFailure = "setAttributes" | "recordException" | "end"; +type ActiveSpanBehavior = "duplicate" | "omit" | "replace" | "throw-after"; + +function installTracer( + failure?: SpanFailure, + onStart?: (attributes: Record) => void, + observers: { + onAttributes?: (attributes: Record) => void; + onStatus?: (status: { code: number; message?: string }) => void; + onException?: (error: unknown) => void; + } = {}, +): void { + const span: Span = { + setAttribute() { + return span; + }, + setAttributes(attributes) { + if (failure === "setAttributes") throw new Error("telemetry setAttributes failed"); + observers.onAttributes?.(attributes); + return span; + }, + setStatus(status) { + observers.onStatus?.(status); + return span; + }, + recordException(error) { + if (failure === "recordException") throw new Error("telemetry recordException failed"); + observers.onException?.(error); + }, + addEvent() { + return span; + }, + end() { + if (failure === "end") throw new Error("telemetry end failed"); + }, + spanContext() { + return { traceId: "1".repeat(32), spanId: "1".repeat(16), traceFlags: 1 }; + }, + updateName() {}, + }; + + const tracer = { + startActiveSpan( + _name: string, + optionsOrFn: + | { attributes?: Record } + | ((span: Span) => unknown), + contextOrFn?: unknown, + fn?: (span: Span) => unknown, + ) { + const options = typeof optionsOrFn === "function" ? {} : optionsOrFn; + const callback = typeof optionsOrFn === "function" + ? optionsOrFn + : typeof contextOrFn === "function" + ? contextOrFn as (span: Span) => unknown + : fn!; + onStart?.(options.attributes ?? {}); + return callback(span); + }, + } as unknown as Tracer; + + setGlobalTracerProvider({ getTracer: () => tracer }); +} + +function installMaliciousTracer(behavior: ActiveSpanBehavior): void { + const span: Span = { + setAttribute() { + return span; + }, + setAttributes() { + return span; + }, + setStatus() { + return span; + }, + recordException() {}, + addEvent() { + return span; + }, + end() {}, + spanContext() { + return { traceId: "1".repeat(32), spanId: "1".repeat(16), traceFlags: 1 }; + }, + updateName() {}, + }; + const tracer = { + startActiveSpan(...args: unknown[]) { + const callback = args.at(-1) as (span: Span) => unknown; + if (behavior === "omit") return Promise.resolve(new Response("provider replacement")); + + const applicationResult = callback(span); + if (behavior === "duplicate") callback(span); + if (behavior === "throw-after") throw new Error("provider failed after callback"); + if (behavior === "replace") return Promise.resolve(new Response("provider replacement")); + return applicationResult; + }, + } as unknown as Tracer; + + setGlobalTracerProvider({ getTracer: () => tracer }); +} + +describe("observability/auto-instrument/http-instrumentation", () => { + afterEach(() => { + _resetShimForTests(); + }); + + it("preserves Request method and headers while adding tracing headers", async () => { + let spanAttributes: Record = {}; + installTracer(undefined, (attributes) => { + spanAttributes = attributes; + }); + + let received: Request | undefined; + const baseFetch = ((input: RequestInfo | URL, init?: RequestInit) => { + received = new Request(input, init); + return Promise.resolve(new Response("ok")); + }) as typeof fetch; + const instrumentedFetch = createInstrumentedFetch(baseFetch); + const request = new Request("https://example.com/items", { + method: "POST", + headers: { + authorization: "Bearer ", + "x-request-id": "request-1", + }, + }); + + await instrumentedFetch(request); + + assertEquals(received?.method, "POST"); + assertEquals(received?.headers.get("authorization"), "Bearer "); + assertEquals(received?.headers.get("x-request-id"), "request-1"); + assertEquals(spanAttributes["http.method"], "POST"); + }); + + it("runs HTTP handlers exactly once despite adversarial active-span providers", async () => { + for (const behavior of ["duplicate", "omit", "replace", "throw-after"] as const) { + _resetShimForTests(); + installMaliciousTracer(behavior); + let calls = 0; + const expected = new Response(`handler-${behavior}`); + const instrumentedHandler = instrumentHttpHandler(() => { + calls++; + return expected; + }); + + const result = await instrumentedHandler(new Request("https://example.com/items")); + + assertEquals(result, expected); + assertEquals(calls, 1); + } + }); + + it("runs base fetch exactly once despite adversarial active-span providers", async () => { + for (const behavior of ["duplicate", "omit", "replace", "throw-after"] as const) { + _resetShimForTests(); + installMaliciousTracer(behavior); + let calls = 0; + const expected = new Response(`fetch-${behavior}`); + const instrumentedFetch = createInstrumentedFetch( + (() => { + calls++; + return Promise.resolve(expected); + }) as typeof fetch, + ); + + const result = await instrumentedFetch("https://example.com/items"); + + assertEquals(result, expected); + assertEquals(calls, 1); + } + }); + + it("preserves the exact application rejection when a provider replaces its result", async () => { + installMaliciousTracer("replace"); + const applicationError = new Error("application rejection"); + let calls = 0; + const instrumentedHandler = instrumentHttpHandler(() => { + calls++; + throw applicationError; + }); + + let caught: unknown; + try { + await instrumentedHandler(new Request("https://example.com/items")); + } catch (error) { + caught = error; + } + + assertEquals(caught, applicationError); + assertEquals(calls, 1); + }); + + it("does not replace a successful fetch result when span recording fails", async () => { + installTracer("setAttributes"); + const instrumentedFetch = createInstrumentedFetch( + (() => Promise.resolve(new Response("application result"))) as typeof fetch, + ); + + const response = await instrumentedFetch("https://example.com/items"); + + assertEquals(await response.text(), "application result"); + }); + + it("does not replace a successful handler result when span finalization fails", async () => { + installTracer("end"); + const instrumentedHandler = instrumentHttpHandler(() => new Response("application result")); + + const response = await instrumentedHandler(new Request("https://example.com/items")); + + assertEquals(await response.text(), "application result"); + }); + + it("runs the handler when the context provider cannot return an active context", async () => { + installTracer(); + setGlobalContextAccessor({ + active: () => { + throw new Error("context provider failed"); + }, + with: (_context, fn) => fn(), + }); + let calls = 0; + const instrumentedHandler = instrumentHttpHandler(() => { + calls++; + return new Response("application result"); + }); + + const response = await instrumentedHandler(new Request("https://example.com/items")); + + assertEquals(await response.text(), "application result"); + assertEquals(calls, 1); + }); + + it("preserves the original handler failure when error telemetry also fails", async () => { + installTracer("recordException"); + const applicationError = new Error("application failed"); + const instrumentedHandler = instrumentHttpHandler(() => { + throw applicationError; + }); + + await assertRejects( + () => instrumentedHandler(new Request("https://example.com/items")), + Error, + "application failed", + ); + }); + + it("redacts URL credentials from recorded failures without changing the thrown error", async () => { + let recordedAttributes: Record = {}; + let recordedStatus: { code: number; message?: string } | undefined; + let recordedException: unknown; + installTracer(undefined, undefined, { + onAttributes: (attributes) => { + recordedAttributes = attributes; + }, + onStatus: (status) => { + recordedStatus = status; + }, + onException: (error) => { + recordedException = error; + }, + }); + const applicationError = new Error( + "failed https://user:password@example.test/path?access_token=secret", + ); + const instrumentedHandler = instrumentHttpHandler(() => { + throw applicationError; + }); + + try { + await instrumentedHandler(new Request("https://example.com/items")); + throw new Error("expected handler failure"); + } catch (error) { + assertEquals(error, applicationError); + } + + assertEquals(String(recordedAttributes["error.message"]).includes("secret"), false); + assertEquals(recordedStatus?.message?.includes("secret"), false); + assertEquals((recordedException as Error).message.includes("secret"), false); + }); +}); diff --git a/src/observability/auto-instrument/http-instrumentation.ts b/src/observability/auto-instrument/http-instrumentation.ts index 87bcd8627c..42f7d70029 100644 --- a/src/observability/auto-instrument/http-instrumentation.ts +++ b/src/observability/auto-instrument/http-instrumentation.ts @@ -1,6 +1,7 @@ import { serverLogger } from "#veryfront/utils"; import { sanitizeUrlForSpan } from "#veryfront/utils/logger/redact.ts"; import { + type Context, context as otContext, propagation, type Span, @@ -9,6 +10,8 @@ import { trace, } from "#veryfront/observability/tracing/api-shim.ts"; import type { ErrorAttributes, HttpAttributes } from "./types.ts"; +import { sanitizeErrorForTelemetry, sanitizeTelemetryAttributes } from "../telemetry-error.ts"; +import { runAsyncWithContextFallback } from "../tracing/context-callback.ts"; const logger = serverLogger.component("auto-instrument"); @@ -16,6 +19,50 @@ function getHttpTracer() { return trace.getTracer("veryfront-http"); } +function reportTelemetryFailure(failureMessage: string, error: unknown): void { + try { + logger.debug(failureMessage, error); + } catch (_) { + /* expected: telemetry and logging failures must not affect application work */ + } +} + +function runTelemetryOperation(operation: () => void, failureMessage: string): void { + try { + operation(); + } catch (error) { + reportTelemetryFailure(failureMessage, error); + } +} + +async function runWithActiveSpanFallback( + activate: (callback: (span: Span) => Promise) => Promise | T, + operation: (span: Span | null) => Promise, + failureMessage: string, +): Promise { + let selectedSpan: Span | null = null; + let spanSelected = false; + + return await runAsyncWithContextFallback( + async (invoke) => { + return await activate((candidate) => { + if (!spanSelected) { + spanSelected = true; + selectedSpan = candidate ?? null; + } else if (candidate && candidate !== selectedSpan) { + runTelemetryOperation( + () => candidate.end(), + "Failed to end duplicate active span", + ); + } + return invoke(); + }); + }, + () => operation(selectedSpan), + (error) => reportTelemetryFailure(failureMessage, error), + ); +} + const headersGetter = { keys(carrier: Headers): string[] { return [...carrier.keys()]; @@ -25,12 +72,12 @@ const headersGetter = { }, }; -function extractParentContext(headers: Headers) { +function extractParentContext(headers: Headers): Context | undefined { try { return propagation.extract(otContext.active(), headers, headersGetter); } catch (error) { - logger.debug("Failed to extract parent context", error); - return otContext.active(); + reportTelemetryFailure("Failed to extract parent context", error); + return undefined; } } @@ -41,43 +88,41 @@ export function instrumentHttpHandler( return async function instrumentedHttpHandler(request: Request): Promise { const startTime = performance.now(); const url = new URL(request.url); - const httpAttrs = buildHttpAttributes(request, url); + const httpAttrs = sanitizeTelemetryAttributes( + buildHttpAttributes(request, url), + ); const parentContext = extractParentContext(request.headers); - // Track whether the handler has been invoked to prevent double-execution. - // If startActiveSpan throws after the callback already called handler(), - // the outer catch must propagate the error rather than re-invoke handler. - let handlerInvoked = false; - - try { - return await getHttpTracer().startActiveSpan( - "http.server.request", - { kind: SpanKind.SERVER, attributes: httpAttrs }, - parentContext, - async (span) => { - try { - handlerInvoked = true; - const response = await handler(request); - recordResponseSuccess(span, response, performance.now() - startTime, httpAttrs); - return response; - } catch (error) { - recordResponseError(span, error, performance.now() - startTime, httpAttrs); - throw error; - } finally { - span.end(); - } - }, - ); - } catch (error) { - if (handlerInvoked) { - // Handler already ran — propagate its error without re-invoking. + const runHandler = async (span: Span | null): Promise => { + try { + const response = await handler(request); + runTelemetryOperation( + () => recordResponseSuccess(span, response, performance.now() - startTime, httpAttrs), + "Failed to record HTTP server response", + ); + return response; + } catch (error) { + runTelemetryOperation( + () => recordResponseError(span, error, performance.now() - startTime, httpAttrs), + "Failed to record HTTP server error", + ); throw error; + } finally { + if (span) { + runTelemetryOperation(() => span.end(), "Failed to end HTTP server span"); + } } - logger.debug( - "[auto-instrument] HTTP handler span failed, falling back to raw handler", - error, - ); - return await handler(request); - } + }; + const spanOptions = { kind: SpanKind.SERVER, attributes: httpAttrs }; + return await runWithActiveSpanFallback( + (callback) => { + const tracer = getHttpTracer(); + return parentContext + ? tracer.startActiveSpan("http.server.request", spanOptions, parentContext, callback) + : tracer.startActiveSpan("http.server.request", spanOptions, callback); + }, + runHandler, + "[auto-instrument] HTTP handler span failed, falling back to raw handler", + ); }; } /** Create a fetch implementation instrumented with observability spans. */ @@ -90,7 +135,7 @@ export function createInstrumentedFetch( ): Promise { const startTime = performance.now(); const urlString = extractFetchUrl(input); - const method = init?.method ?? "GET"; + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); const spanUrl = sanitizeUrlForSpan(urlString); const fetchAttrs: HttpAttributes = { @@ -109,41 +154,68 @@ export function createInstrumentedFetch( } catch (_) { /* expected: relative URLs cannot be parsed, leave defaults */ } + const sanitizedFetchAttrs = sanitizeTelemetryAttributes(fetchAttrs); - // Tracks whether baseFetch was invoked to prevent double-execution on span failure. - let fetchInvoked = false; + return await runWithActiveSpanFallback( + (callback) => + getHttpTracer().startActiveSpan( + "http.client.fetch", + { kind: SpanKind.CLIENT, attributes: sanitizedFetchAttrs }, + callback, + ), + async (span) => { + try { + let effectiveInit = init; + if (span) { + try { + const headers = new Headers( + init?.headers ?? (input instanceof Request ? input.headers : undefined), + ); + runTelemetryOperation( + () => + propagation.inject(otContext.active(), headers, { + set: (h, k, v) => h.set(k, v), + }), + "Failed to inject fetch trace context", + ); + effectiveInit = { ...init, headers }; + } catch (error) { + reportTelemetryFailure("Failed to prepare fetch trace context", error); + } + } - try { - return await getHttpTracer().startActiveSpan( - "http.client.fetch", - { kind: SpanKind.CLIENT, attributes: fetchAttrs }, - async (span) => { - try { - const headers = new Headers(init?.headers); - propagation.inject(otContext.active(), headers, { - set: (h, k, v) => h.set(k, v), - }); - - fetchInvoked = true; - const response = await baseFetch(input, { ...init, headers }); - recordResponseSuccess(span, response, performance.now() - startTime, fetchAttrs); - return response; - } catch (error) { - recordResponseError(span, error, performance.now() - startTime, fetchAttrs); - throw error; - } finally { - span.end(); + const response = await baseFetch(input, effectiveInit); + runTelemetryOperation( + () => + recordResponseSuccess( + span, + response, + performance.now() - startTime, + sanitizedFetchAttrs, + ), + "Failed to record HTTP client response", + ); + return response; + } catch (error) { + runTelemetryOperation( + () => + recordResponseError( + span, + error, + performance.now() - startTime, + sanitizedFetchAttrs, + ), + "Failed to record HTTP client error", + ); + throw error; + } finally { + if (span) { + runTelemetryOperation(() => span.end(), "Failed to end HTTP client span"); } - }, - ); - } catch (error) { - if (fetchInvoked) { - // baseFetch already ran — propagate its error without re-invoking. - throw error; - } - logger.debug("Fetch span failed, falling back to base fetch", error); - return await baseFetch(input, init); - } + } + }, + "Fetch span failed, falling back to base fetch", + ); }; } @@ -165,25 +237,32 @@ function recordResponseSuccess( ): void { if (!span) return; - span.setAttributes({ - "http.status_code": response.status, - "http.response.size": Number(response.headers.get("content-length") ?? 0), - "http.duration_ms": Math.round(duration), - }); - - if (response.status >= 500) { - span.setStatus({ code: SpanStatusCode.ERROR }); - } else if (response.status >= 400) { - span.setStatus({ code: SpanStatusCode.UNSET, message: `HTTP ${response.status}` }); - } else { - span.setStatus({ code: SpanStatusCode.OK }); - } + const contentLength = Number(response.headers.get("content-length") ?? 0); + runTelemetryOperation( + () => + span.setAttributes( + sanitizeTelemetryAttributes({ + "http.status_code": response.status, + "http.response.size": Number.isFinite(contentLength) && contentLength >= 0 + ? contentLength + : 0, + "http.duration_ms": Math.max(0, Math.round(duration)), + "http.method": httpAttrs["http.method"], + "http.target": httpAttrs["http.target"], + }), + ), + "Failed to record HTTP response attributes", + ); - // Preserve original request method/path for downstream analysis - span.setAttributes({ - "http.method": httpAttrs["http.method"], - "http.target": httpAttrs["http.target"], - }); + runTelemetryOperation(() => { + if (response.status >= 500) { + span.setStatus({ code: SpanStatusCode.ERROR }); + } else if (response.status >= 400) { + span.setStatus({ code: SpanStatusCode.UNSET, message: `HTTP ${response.status}` }); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + }, "Failed to record HTTP response status"); } function recordResponseError( @@ -194,17 +273,32 @@ function recordResponseError( ): void { if (!span) return; - span.recordException(error instanceof Error ? error : new Error(String(error))); - span.setAttributes({ - ...buildErrorAttributes(error), - "http.duration_ms": Math.round(duration), - "http.method": httpAttrs["http.method"], - "http.target": httpAttrs["http.target"], - }); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error instanceof Error ? error.message : String(error), - }); + const telemetryError = sanitizeErrorForTelemetry(error); + + runTelemetryOperation( + () => span.recordException(telemetryError), + "Failed to record HTTP exception", + ); + runTelemetryOperation( + () => + span.setAttributes( + sanitizeTelemetryAttributes({ + ...buildErrorAttributes(error), + "http.duration_ms": Math.max(0, Math.round(duration)), + "http.method": httpAttrs["http.method"], + "http.target": httpAttrs["http.target"], + }), + ), + "Failed to record HTTP error attributes", + ); + runTelemetryOperation( + () => + span.setStatus({ + code: SpanStatusCode.ERROR, + message: telemetryError.message, + }), + "Failed to record HTTP error status", + ); } function extractFetchUrl(input: RequestInfo | URL): string { @@ -214,17 +308,10 @@ function extractFetchUrl(input: RequestInfo | URL): string { } function buildErrorAttributes(error: unknown): ErrorAttributes { - if (error instanceof Error) { - return { - error: "true", - "error.type": error.constructor.name, - "error.message": error.message, - }; - } - + const telemetryError = sanitizeErrorForTelemetry(error); return { error: "true", - "error.type": "Unknown", - "error.message": String(error), + "error.type": telemetryError.name, + "error.message": telemetryError.message, }; } diff --git a/src/observability/auto-instrument/orchestrator.test.ts b/src/observability/auto-instrument/orchestrator.test.ts index 7eea4f7f98..543e5a32a0 100644 --- a/src/observability/auto-instrument/orchestrator.test.ts +++ b/src/observability/auto-instrument/orchestrator.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertStrictEquals } from "#veryfront/testing/assert.ts"; import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; import { __resetAutoInstrumentForTests, @@ -65,9 +65,33 @@ describe("observability/auto-instrument/orchestrator", () => { }); assertEquals(isAutoInstrumentEnabled(), true); }); + + it("shares one readiness promise across concurrent initialization", async () => { + const first = initAutoInstrumentation({ + tracing: { enabled: true, exporter: "console" }, + }); + const second = initAutoInstrumentation({ + tracing: { enabled: false }, + }); + + assertStrictEquals(second, first); + await first; + assertEquals(isAutoInstrumentEnabled(), true); + }); }); describe("__resetAutoInstrumentForTests", () => { + it("prevents an in-flight initialization from restoring stale state", async () => { + const initializing = initAutoInstrumentation({ + tracing: { enabled: true, exporter: "console" }, + }); + __resetAutoInstrumentForTests(); + + await initializing; + + assertEquals(isAutoInstrumentEnabled(), false); + }); + it("should reset initialization state", async () => { await initAutoInstrumentation(); assertEquals(isAutoInstrumentEnabled(), true); diff --git a/src/observability/auto-instrument/orchestrator.ts b/src/observability/auto-instrument/orchestrator.ts index 174c701a3b..d7264743e8 100644 --- a/src/observability/auto-instrument/orchestrator.ts +++ b/src/observability/auto-instrument/orchestrator.ts @@ -8,34 +8,48 @@ import { mergeConfig } from "./configurator.ts"; const logger = serverLogger.component("auto-instrument"); let initialized = false; +let initializationPromise: Promise | null = null; +let lifecycleGeneration = 0; /** Initialize automatic instrumentation wrappers. */ -export async function initAutoInstrumentation( +export function initAutoInstrumentation( config: AutoInstrumentConfig = {}, adapter?: RuntimeAdapter, ): Promise { + if (initializationPromise) return initializationPromise; if (initialized) { logger.debug("Already initialized"); - return; + return Promise.resolve(); } const finalConfig = mergeConfig(config); + const generation = lifecycleGeneration; - try { - if (finalConfig.tracing?.enabled) { - await initTracing(finalConfig.tracing, adapter); - } + const attempt = (async (): Promise => { + try { + if (finalConfig.tracing?.enabled) { + await initTracing(finalConfig.tracing, adapter); + } + + if (finalConfig.metrics?.enabled) { + await initMetrics(finalConfig.metrics, adapter); + } - if (finalConfig.metrics?.enabled) { - await initMetrics(finalConfig.metrics, adapter); + if (generation === lifecycleGeneration) logInitialization(finalConfig); + } catch (error) { + if (generation === lifecycleGeneration) { + logger.warn("Failed to initialize auto-instrumentation", error); + } + } finally { + if (generation === lifecycleGeneration) initialized = true; } + })(); - logInitialization(finalConfig); - } catch (error) { - logger.warn("Failed to initialize auto-instrumentation", error); - } finally { - initialized = true; - } + const tracked = attempt.finally(() => { + if (initializationPromise === tracked) initializationPromise = null; + }); + initializationPromise = tracked; + return tracked; } /** Check whether auto instrumentation is enabled. */ @@ -48,7 +62,9 @@ export function isAutoInstrumentEnabled(): boolean { * @internal */ export function __resetAutoInstrumentForTests(): void { + lifecycleGeneration++; initialized = false; + initializationPromise = null; } function logInitialization(config: AutoInstrumentConfig): void { diff --git a/src/observability/auto-instrument/react-instrumentation.ts b/src/observability/auto-instrument/react-instrumentation.ts index b80bf27fe2..a4497e7ab0 100644 --- a/src/observability/auto-instrument/react-instrumentation.ts +++ b/src/observability/auto-instrument/react-instrumentation.ts @@ -1,10 +1,12 @@ import { type Span, SpanStatusCode } from "#veryfront/observability/tracing/api-shim.ts"; +import { sanitizeUrlForSpan } from "#veryfront/utils/logger/redact.ts"; import { endSpan, setSpanAttributes, SpanNames, startSpan, withSpan } from "../tracing/index.ts"; import { recordRenderError } from "../metrics/index.ts"; +import { sanitizeErrorForTelemetry } from "../telemetry-error.ts"; /** Instrument a React render operation. */ export function instrumentReactRender( - renderFn: () => Promise | T, + renderFn: () => PromiseLike | T, componentName: string, ): Promise { return withSpan( @@ -13,8 +15,7 @@ export function instrumentReactRender( const startTime = performance.now(); try { - const result = renderFn(); - const resolved = result instanceof Promise ? await result : result; + const resolved = await Promise.resolve(renderFn()); recordRenderDuration(span, startTime); return resolved; @@ -36,7 +37,13 @@ export function instrumentErrorHandler( captureToSpan = true, ): (error: Error, request?: Request) => Promise | Response { return (error: Error, request?: Request): Promise | Response => { - if (captureToSpan) captureErrorToSpan(error, request); + if (captureToSpan) { + try { + captureErrorToSpan(error, request); + } catch (_) { + /* expected: telemetry failures must not prevent error handling */ + } + } return handler(error, request); }; } @@ -48,8 +55,17 @@ function handleRenderError(span: Span | null, error: unknown, componentName: str // but we need to record the exception and status if (!span) return; - span.recordException(error as Error); - span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + const telemetryError = sanitizeErrorForTelemetry(error); + try { + span.recordException(telemetryError); + } catch (_) { + /* expected: telemetry failures must not replace render failures */ + } + try { + span.setStatus({ code: SpanStatusCode.ERROR, message: telemetryError.message }); + } catch (_) { + /* expected: telemetry failures must not replace render failures */ + } } function recordRenderDuration(span: Span | null, startTime: number): void { @@ -58,12 +74,13 @@ function recordRenderDuration(span: Span | null, startTime: number): void { } function captureErrorToSpan(error: Error, request?: Request): void { + const telemetryError = sanitizeErrorForTelemetry(error); const span = startSpan("error.handler", { kind: "internal", attributes: { - "error.type": error.constructor.name, - "error.message": error.message, - "error.stack": error.stack ?? "", + "error.type": telemetryError.name, + "error.message": telemetryError.message, + "error.stack": telemetryError.stack ?? "", }, }); @@ -71,10 +88,10 @@ function captureErrorToSpan(error: Error, request?: Request): void { const url = new URL(request.url); setSpanAttributes(span, { "http.method": request.method, - "http.url": request.url, + "http.url": sanitizeUrlForSpan(request.url), "http.path": url.pathname, }); } - endSpan(span, error); + endSpan(span, telemetryError); } diff --git a/src/observability/auto-instrument/wrappers.test.ts b/src/observability/auto-instrument/wrappers.test.ts index 9b4da8483c..792faaf7fb 100644 --- a/src/observability/auto-instrument/wrappers.test.ts +++ b/src/observability/auto-instrument/wrappers.test.ts @@ -5,6 +5,16 @@ import { instrument, instrumentBatch, instrumentSync } from "./wrappers.ts"; describe("observability/auto-instrument/wrappers", () => { describe("instrument (async wrapper)", () => { + it("does not block the wrapped operation when attribute collection fails", async () => { + const wrapped = instrument(() => Promise.resolve("application result"), "test.attributes", { + attributes: () => { + throw new Error("telemetry attributes failed"); + }, + }); + + assertEquals(await wrapped(), "application result"); + }); + it("should wrap an async function and preserve its result", async () => { const fn = (x: number): Promise => Promise.resolve(x * 2); const wrapped = instrument(fn, "test.double"); @@ -59,6 +69,16 @@ describe("observability/auto-instrument/wrappers", () => { }); describe("instrumentSync (sync wrapper)", () => { + it("does not block the wrapped operation when attribute collection fails", () => { + const wrapped = instrumentSync(() => "application result", "test.attributes", { + attributes: () => { + throw new Error("telemetry attributes failed"); + }, + }); + + assertEquals(wrapped(), "application result"); + }); + it("should wrap a sync function and preserve its result", () => { const fn = (x: number): number => x * 3; const wrapped = instrumentSync(fn, "test.triple"); @@ -104,6 +124,14 @@ describe("observability/auto-instrument/wrappers", () => { }); describe("instrumentBatch", () => { + it("rejects invalid batch sizes instead of silently skipping work", async () => { + await assertRejects( + () => instrumentBatch("test.invalid-size", [1], async () => {}, { batchSize: -1 }), + RangeError, + "batchSize", + ); + }); + it("should process all items", async () => { const results: number[] = []; // deno-lint-ignore require-await diff --git a/src/observability/auto-instrument/wrappers.ts b/src/observability/auto-instrument/wrappers.ts index 5d2138908f..6ba563969c 100644 --- a/src/observability/auto-instrument/wrappers.ts +++ b/src/observability/auto-instrument/wrappers.ts @@ -3,12 +3,12 @@ import { endSpan, setSpanAttributes, type SpanOptions, startSpan } from "../trac import type { BatchOptions, InstrumentOptions } from "./types.ts"; /** Instrument an async operation. */ -export function instrument Promise>( - fn: T, +export function instrument( + fn: (...args: TArgs) => Promise, spanName: string, options?: InstrumentOptions, -): T { - return (async (...args: Parameters): Promise> => { +): (...args: TArgs) => Promise { + return async (...args: TArgs): Promise => { const span = createSpan(spanName, args, options); const startTime = performance.now(); @@ -16,21 +16,21 @@ export function instrument Promise>( const result = await fn(...args); recordDuration(span, startTime); endSpan(span); - return result as ReturnType; + return result; } catch (error) { - endSpan(span, error as Error); + endSpan(span, error); throw error; } - }) as T; + }; } /** Instrument a synchronous operation. */ -export function instrumentSync unknown>( - fn: T, +export function instrumentSync( + fn: (...args: TArgs) => TResult, spanName: string, options?: InstrumentOptions, -): T { - return ((...args: Parameters): ReturnType => { +): (...args: TArgs) => TResult { + return (...args: TArgs): TResult => { const span = createSpan(spanName, args, options); const startTime = performance.now(); @@ -38,12 +38,12 @@ export function instrumentSync unknown>( const result = fn(...args); recordDuration(span, startTime); endSpan(span); - return result as ReturnType; + return result; } catch (error) { - endSpan(span, error as Error); + endSpan(span, error); throw error; } - }) as T; + }; } /** Instrument a batch operation. */ @@ -54,6 +54,9 @@ export async function instrumentBatch( options?: BatchOptions, ): Promise { const batchSize = options?.batchSize ?? 10; + if (!Number.isSafeInteger(batchSize) || batchSize <= 0) { + throw new RangeError("instrumentBatch batchSize must be a positive integer"); + } const totalBatches = Math.ceil(items.length / batchSize); const batchSpan = startSpan(operationName, { @@ -84,22 +87,29 @@ export async function instrumentBatch( await Promise.all(batch.map((item, index) => processor(item, start + index))); endSpan(batchItemSpan); } catch (error) { - endSpan(batchItemSpan, error as Error); + endSpan(batchItemSpan, error); throw error; } } endSpan(batchSpan); } catch (error) { - endSpan(batchSpan, error as Error); + endSpan(batchSpan, error); throw error; } } function createSpan(spanName: string, args: unknown[], options?: InstrumentOptions): Span | null { + let attributes: SpanOptions["attributes"] = {}; + try { + attributes = options?.attributes?.(args) ?? {}; + } catch (_) { + /* expected: instrumentation metadata must not block the wrapped operation */ + } + const spanOptions: SpanOptions = { kind: options?.kind ?? "internal", - attributes: options?.attributes?.(args) ?? {}, + attributes, }; return startSpan(spanName, spanOptions); diff --git a/src/observability/error-collector.test.ts b/src/observability/error-collector.test.ts index e25b329366..d815d21fd5 100644 --- a/src/observability/error-collector.test.ts +++ b/src/observability/error-collector.test.ts @@ -1,10 +1,33 @@ import "#veryfront/schemas/_test-setup.ts"; import { assertEquals, assertExists, assertThrows } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; import { ErrorCollector, parseCompileError } from "./error-collector.ts"; describe("cli/mc./error-collector", () => { describe("ErrorCollector", () => { + it("should reject invalid maxErrors values", () => { + assertThrows(() => new ErrorCollector({ maxErrors: -1 }), RangeError, "maxErrors"); + assertThrows(() => new ErrorCollector({ maxErrors: 1.5 }), RangeError, "maxErrors"); + assertThrows( + () => new ErrorCollector({ maxErrors: Number.NaN }), + RangeError, + "maxErrors", + ); + }); + + it("should support a zero-sized collector without retaining errors", () => { + const ec = new ErrorCollector({ maxErrors: 0 }); + const received: string[] = []; + ec.subscribe((error) => received.push(error.message)); + + const error = ec.addRuntimeError("reported"); + + assertEquals(error.message, "reported"); + assertEquals(received, ["reported"]); + assertEquals(ec.count, 0); + }); + it("should add and retrieve errors", () => { const ec = new ErrorCollector(); ec.add({ type: "compile", category: "BUILD", message: "fail" }); @@ -128,6 +151,101 @@ describe("cli/mc./error-collector", () => { assertEquals(ec.getAll({ file: /^src\// }).length, 1); }); + it("should filter deterministically with stateful file regex patterns", () => { + const ec = new ErrorCollector(); + ec.addCompileError("a", "src/a.ts"); + ec.addCompileError("b", "src/b.ts"); + const pattern = /^src\//g; + + assertEquals(ec.getAll({ file: pattern }).length, 2); + assertEquals(ec.getAll({ file: pattern }).length, 2); + }); + + it("should redact retained error details without mutating caller context", () => { + const ec = new ErrorCollector(); + const context = { apiKey: "secret", safe: "value" }; + + const error = ec.addRuntimeError( + "failed https://user:password@example.test/path?access_token=secret", + "at https://example.test/path?token=secret", + context, + ); + + assertEquals(error.message.includes("secret"), false); + assertEquals(error.stack?.includes("secret"), false); + assertEquals(error.context, { apiKey: "[REDACTED]", safe: "value" }); + assertEquals(context.apiKey, "secret"); + }); + + it("bounds retained error messages and stacks", () => { + const ec = new ErrorCollector(); + + const error = ec.addRuntimeError( + "m".repeat(MAX_STRING_DISPLAY_LENGTH + 100), + "s".repeat(MAX_STRING_DISPLAY_LENGTH + 100), + ); + + assertEquals(error.message.length, MAX_STRING_DISPLAY_LENGTH); + assertEquals(error.stack?.length, MAX_STRING_DISPLAY_LENGTH); + assertEquals(ec.get(error.id)?.message.length, MAX_STRING_DISPLAY_LENGTH); + }); + + it("retains only declared error fields", () => { + const ec = new ErrorCollector(); + + const error = ec.add({ + type: "runtime", + category: "RUNTIME", + message: "failure", + undeclared: "must not be retained", + } as never); + + assertEquals(Object.hasOwn(error, "undeclared"), false); + assertEquals(Object.hasOwn(ec.get(error.id) ?? {}, "undeclared"), false); + }); + + it("should not expose retained errors to caller or subscriber mutation", () => { + const ec = new ErrorCollector(); + ec.subscribe((error) => { + error.message = "subscriber mutation"; + if (error.context) error.context.value = "subscriber mutation"; + }); + + const returned = ec.addRuntimeError("original", undefined, { value: "original" }); + returned.message = "caller mutation"; + if (returned.context) returned.context.value = "caller mutation"; + + const retained = ec.get(returned.id); + assertExists(retained); + assertEquals(retained.message, "original"); + assertEquals(retained.context?.value, "original"); + + retained.message = "query mutation"; + assertEquals(ec.get(returned.id)?.message, "original"); + }); + + it("detaches structured Date and URL context for every observer", () => { + const ec = new ErrorCollector(); + const date = new Date("2025-01-02T03:04:05.000Z"); + const url = new URL("https://user:password@example.test/path?token=secret"); + let subscriberDate: Date | undefined; + ec.subscribe((error) => { + subscriberDate = error.context?.date as Date; + subscriberDate.setUTCFullYear(2030); + }); + + const returned = ec.addRuntimeError("structured", undefined, { date, url }); + (returned.context?.date as Date).setUTCFullYear(2040); + (returned.context?.url as URL).pathname = "/mutated"; + + const retained = ec.get(returned.id)?.context; + assertEquals((retained?.date as Date).getUTCFullYear(), 2025); + assertEquals((retained?.url as URL).pathname, "/path"); + assertEquals((retained?.url as URL).href.includes("secret"), false); + assertEquals(date.getUTCFullYear(), 2025); + assertEquals(subscriberDate?.getUTCFullYear(), 2030); + }); + it("should get by id", () => { const ec = new ErrorCollector(); const err = ec.add({ type: "compile", category: "BUILD", message: "test" }); @@ -245,6 +363,41 @@ describe("cli/mc./error-collector", () => { "mismatched type/category", ); }); + + it("rejects unknown runtime types before category lookup", () => { + const ec = new ErrorCollector(); + + assertThrows( + () => + ec.add({ + category: undefined, + type: "__proto__", + message: "invalid", + } as never), + Error, + "invalid error type", + ); + + assertEquals(ec.count, 0); + }); + + it("does not create NaN buckets from legacy malformed entries", () => { + const ec = new ErrorCollector(); + const internal = ec as unknown as { + errors: Map; + }; + internal.errors.set("legacy-invalid", { + type: "unknown", + category: "unknown", + }); + + const typeCounts = ec.countByType(); + const categoryCounts = ec.countByCategory(); + assertEquals(Object.values(typeCounts).every(Number.isFinite), true); + assertEquals(Object.values(categoryCounts).every(Number.isFinite), true); + assertEquals(Object.hasOwn(typeCounts, "unknown"), false); + assertEquals(Object.hasOwn(categoryCounts, "unknown"), false); + }); }); describe("parseCompileError", () => { diff --git a/src/observability/error-collector.ts b/src/observability/error-collector.ts index 6fb5202104..29d2c19c20 100644 --- a/src/observability/error-collector.ts +++ b/src/observability/error-collector.ts @@ -6,7 +6,10 @@ **************************/ import { type ErrorCategory, INVALID_ARGUMENT } from "#veryfront/errors"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; import { createSubscriberSet } from "#veryfront/utils/subscriber-set.ts"; +import { MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, MAX_OBSERVABILITY_NAME_LENGTH } from "./limits.ts"; +import { sanitizeStructuredTelemetryData, sanitizeTelemetryText } from "./telemetry-error.ts"; /** Public API contract for error type. */ export type ErrorType = "compile" | "runtime" | "bundle" | "hmr" | "module"; @@ -22,6 +25,10 @@ const ERROR_TYPE_TO_CATEGORY: Record = { module: "MODULE", }; +function isErrorType(value: unknown): value is ErrorType { + return typeof value === "string" && Object.hasOwn(ERROR_TYPE_TO_CATEGORY, value); +} + /** Error shape for dev. */ export interface DevError { /** Unique error identifier */ @@ -63,6 +70,23 @@ export interface ErrorFilter { /** Public API contract for error subscriber. */ export type ErrorSubscriber = (error: DevError) => void; +function snapshotError(error: DevError): DevError { + return { + ...error, + context: error.context ? sanitizeStructuredTelemetryData(error.context) : error.context, + }; +} + +function matchesPattern(pattern: RegExp, value: string): boolean { + const initialLastIndex = pattern.lastIndex; + try { + pattern.lastIndex = 0; + return pattern.test(value); + } finally { + pattern.lastIndex = initialLastIndex; + } +} + /** Implement error collector. */ export class ErrorCollector { private errors = new Map(); @@ -71,7 +95,11 @@ export class ErrorCollector { private maxErrors: number; constructor(options: { maxErrors?: number } = {}) { - this.maxErrors = options.maxErrors ?? 100; + const maxErrors = options.maxErrors ?? 100; + if (!Number.isSafeInteger(maxErrors) || maxErrors < 0) { + throw new RangeError("ErrorCollector maxErrors must be a non-negative integer"); + } + this.maxErrors = maxErrors; } private generateId(): string { @@ -79,30 +107,64 @@ export class ErrorCollector { } add(error: Omit): DevError { - const expectedCategory = ERROR_TYPE_TO_CATEGORY[error.type]; - if (error.category !== expectedCategory) { + const type: unknown = error.type; + if (!isErrorType(type)) { + throw INVALID_ARGUMENT.create({ + detail: `ErrorCollector.add() received invalid error type: ${String(type)}`, + }); + } + + const category: unknown = error.category; + const expectedCategory = ERROR_TYPE_TO_CATEGORY[type]; + if (category !== expectedCategory) { throw INVALID_ARGUMENT.create({ detail: - `ErrorCollector.add() received mismatched type/category: ${error.type} must use ${expectedCategory}, got ${error.category}`, + `ErrorCollector.add() received mismatched type/category: ${type} must use ${expectedCategory}, got ${ + String(category) + }`, + }); + } + if (typeof error.message !== "string") { + throw INVALID_ARGUMENT.create({ + detail: "ErrorCollector.add() requires a string message", }); } const fullError: DevError = { - ...error, id: this.generateId(), + category: expectedCategory, + type, + slug: typeof error.slug === "string" + ? sanitizeTelemetryText(error.slug, MAX_OBSERVABILITY_NAME_LENGTH) + : undefined, + message: sanitizeTelemetryText(error.message, MAX_STRING_DISPLAY_LENGTH), + file: typeof error.file === "string" + ? sanitizeTelemetryText( + error.file, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ) + : undefined, + line: error.line, + column: error.column, + stack: typeof error.stack === "string" + ? sanitizeTelemetryText(error.stack, MAX_STRING_DISPLAY_LENGTH) + : undefined, timestamp: Date.now(), + context: error.context ? sanitizeStructuredTelemetryData(error.context) : error.context, }; - if (this.errors.size >= this.maxErrors) { - const oldestId = this.errors.keys().next().value; - if (oldestId) this.errors.delete(oldestId); - } + if (this.maxErrors > 0) { + if (this.errors.size >= this.maxErrors) { + const oldestId = this.errors.keys().next().value; + if (oldestId) this.errors.delete(oldestId); + } - this.errors.set(fullError.id, fullError); + this.errors.set(fullError.id, fullError); + } this.subscribers.notify(fullError); - return fullError; + return snapshotError(fullError); } private addTypedError( @@ -209,7 +271,7 @@ export class ErrorCollector { getAll(filter?: ErrorFilter): DevError[] { const errors = Array.from(this.errors.values()); - if (!filter) return errors; + if (!filter) return errors.map(snapshotError); const { type, category, slug, file, since } = filter; @@ -235,7 +297,7 @@ export class ErrorCollector { if (file) { if (typeof file === "string") { if (e.file !== file) return false; - } else if (!e.file || !file.test(e.file)) { + } else if (!e.file || !matchesPattern(file, e.file)) { return false; } } @@ -243,11 +305,12 @@ export class ErrorCollector { if (since && e.timestamp < since) return false; return true; - }); + }).map(snapshotError); } get(id: string): DevError | undefined { - return this.errors.get(id); + const error = this.errors.get(id); + return error ? snapshotError(error) : undefined; } clearFile(file: string): number { @@ -283,7 +346,7 @@ export class ErrorCollector { }; for (const { type } of this.errors.values()) { - counts[type]++; + if (isErrorType(type)) counts[type] += 1; } return counts; @@ -308,14 +371,16 @@ export class ErrorCollector { }; for (const { category } of this.errors.values()) { - counts[category]++; + if (typeof category === "string" && Object.hasOwn(counts, category)) { + counts[category as ErrorCategory] += 1; + } } return counts; } subscribe(callback: ErrorSubscriber): () => void { - return this.subscribers.subscribe(callback); + return this.subscribers.subscribe((error) => callback(snapshotError(error))); } toJSON(): DevError[] { diff --git a/src/observability/file-log-subscriber.test.ts b/src/observability/file-log-subscriber.test.ts index 3881fe91a8..747468ef51 100644 --- a/src/observability/file-log-subscriber.test.ts +++ b/src/observability/file-log-subscriber.test.ts @@ -1,8 +1,20 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { + assertEquals, + assertRejects, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; -import { FileLogSubscriber, parseMaxSize } from "./file-log-subscriber.ts"; -import { LogBuffer } from "./log-buffer.ts"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { MAX_FILE_LOG_FILES } from "#veryfront/utils/config-resource-limits.ts"; +import { FileLogSubscriber, parseMaxSize, writeAll } from "./file-log-subscriber.ts"; +import { + MAX_FILE_LOG_PENDING_WRITES, + MAX_FILE_LOG_RETAINED_FAILURES, + MAX_OBSERVABILITY_NAME_LENGTH, +} from "./limits.ts"; +import { LogBuffer, type LogEntry } from "./log-buffer.ts"; import type { FileLogConfig } from "./file-log-subscriber.ts"; function makeConfig(overrides: Partial & { path: string }): FileLogConfig { @@ -63,9 +75,84 @@ describe("observability/file-log-subscriber", () => { assertEquals((err as Error).message.includes("Invalid maxSize"), true); } }); + + it("should reject non-positive and non-finite sizes", () => { + assertThrows(() => parseMaxSize(0), RangeError, "maxSize"); + assertThrows(() => parseMaxSize(-1), RangeError, "maxSize"); + assertThrows(() => parseMaxSize(Number.POSITIVE_INFINITY), RangeError, "maxSize"); + }); + }); + + describe("writeAll", () => { + it("retries partial writes until every byte is persisted", async () => { + const writes: number[] = []; + const writer = { + write(bytes: Uint8Array): Promise { + const written = Math.min(2, bytes.length); + writes.push(written); + return Promise.resolve(written); + }, + }; + + await writeAll(writer, new Uint8Array([1, 2, 3, 4, 5])); + + assertEquals(writes, [2, 2, 1]); + }); + + it("rejects a zero-progress write instead of looping forever", async () => { + await assertRejects( + () => writeAll({ write: () => Promise.resolve(0) }, new Uint8Array([1])), + Error, + "zero bytes", + ); + }); }); describe("FileLogSubscriber", () => { + it("should reject invalid rotation counts and empty paths", () => { + assertThrows( + () => new FileLogSubscriber(makeConfig({ path: "test.log", maxFiles: 0 })), + RangeError, + "maxFiles", + ); + assertThrows( + () => + new FileLogSubscriber({ + ...makeConfig({ path: "test.log" }), + maxFiles: MAX_FILE_LOG_FILES + 1, + }), + RangeError, + "maxFiles", + ); + assertThrows( + () => new FileLogSubscriber(makeConfig({ path: " " })), + TypeError, + "path", + ); + assertThrows( + () => + new FileLogSubscriber({ + ...makeConfig({ path: "test.log" }), + level: "toString" as never, + }), + TypeError, + "level", + ); + }); + + it("should not create or write a file when disabled", async () => { + const dir = await makeTempDir(); + const logPath = `${dir}/disabled.log`; + const sub = new FileLogSubscriber(makeConfig({ path: logPath, enabled: false })); + const buf = new LogBuffer(); + buf.subscribe(sub.getSubscriber()); + + buf.info("must not be written"); + await sub.close(); + + assertEquals(await fileExists(logPath), false); + }); + it("should write log entries as JSON", async () => { const dir = await makeTempDir(); const logPath = `${dir}/test.log`; @@ -86,6 +173,62 @@ describe("observability/file-log-subscriber", () => { await sub.close(); }); + it("should redact direct subscriber entries before writing", async () => { + const dir = await makeTempDir(); + const logPath = `${dir}/direct.log`; + const sub = new FileLogSubscriber(makeConfig({ path: logPath, format: "json" })); + + sub.getSubscriber()({ + id: "direct", + level: "error" as const, + message: "failed https://example.test/path?token=secret", + data: { apiKey: "secret", safe: "value" }, + timestamp: 1, + source: "test", + }); + await sub.flush(); + + const content = await Deno.readTextFile(logPath); + assertEquals(content.includes("secret"), false); + assertEquals(content.includes("[REDACTED]"), true); + + await sub.close(); + }); + + it("bounds and projects direct entries before queueing them", async () => { + const dir = await makeTempDir(); + const logPath = `${dir}/bounded-direct.log`; + const sub = new FileLogSubscriber(makeConfig({ path: logPath })); + + sub.getSubscriber()( + { + id: "i".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + level: "info", + message: "m".repeat(MAX_STRING_DISPLAY_LENGTH + 100), + timestamp: Date.now(), + source: "s".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + undeclared: "must not be retained", + } as LogEntry & { undeclared: string }, + ); + await sub.flush(); + + const entry = JSON.parse( + (await Deno.readTextFile(logPath)).trim(), + ) as Record; + assertEquals((entry.id as string).length, MAX_OBSERVABILITY_NAME_LENGTH); + assertEquals( + (entry.message as string).length, + MAX_STRING_DISPLAY_LENGTH, + ); + assertEquals( + (entry.source as string).length, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + assertEquals(Object.hasOwn(entry, "undeclared"), false); + + await sub.close(); + }); + it("should write log entries as text", async () => { const dir = await makeTempDir(); const logPath = `${dir}/test.log`; @@ -250,7 +393,7 @@ describe("observability/file-log-subscriber", () => { await sub.close(); }); - it("should log non-permission write queue failures", async () => { + it("reports non-permission write failures to both diagnostics and flush callers", async () => { const dir = await makeTempDir(); const sub = new FileLogSubscriber(makeConfig({ path: dir })); const originalError = console.error; @@ -264,7 +407,7 @@ describe("observability/file-log-subscriber", () => { buf.subscribe(sub.getSubscriber()); buf.info("cannot write to a directory", "test"); - await sub.flush(); + await assertRejects(() => sub.flush(), Error); } finally { console.error = originalError; } @@ -277,6 +420,386 @@ describe("observability/file-log-subscriber", () => { true, ); }); + + for (const stage of ["directory", "open/stat", "rotation", "write"] as const) { + it(`atomically disables file logging after a ${stage} permission denial`, async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/permission.log` })); + const internals = sub as unknown as { + file: { + write(bytes: Uint8Array): Promise; + truncate(length: number): Promise; + seek(offset: number, whence: number): Promise; + sync(): Promise; + close(): void; + } | null; + currentSize: number; + maxSizeBytes: number; + ensureDir(): Promise; + openFile(): Promise; + rotate(): Promise; + }; + const denial = new Deno.errors.PermissionDenied(`${stage} denied`); + let filesystemCalls = 0; + const fakeFile = { + write(bytes: Uint8Array): Promise { + filesystemCalls++; + return stage === "write" ? Promise.reject(denial) : Promise.resolve(bytes.length); + }, + truncate(): Promise { + filesystemCalls++; + return Promise.resolve(); + }, + seek(offset: number): Promise { + filesystemCalls++; + return Promise.resolve(offset); + }, + sync(): Promise { + filesystemCalls++; + return Promise.resolve(); + }, + close(): void { + filesystemCalls++; + }, + }; + + if (stage === "directory") { + internals.ensureDir = () => { + filesystemCalls++; + return Promise.reject(denial); + }; + } else if (stage === "open/stat") { + internals.openFile = () => { + filesystemCalls++; + return Promise.reject(denial); + }; + } else { + internals.file = fakeFile; + if (stage === "rotation") { + internals.currentSize = 1; + internals.maxSizeBytes = 1; + internals.rotate = () => { + filesystemCalls++; + return Promise.reject(denial); + }; + } + } + + const originalError = console.error; + console.error = () => {}; + try { + const subscriber = sub.getSubscriber(); + subscriber({ + id: "denied", + level: "error", + message: "first", + timestamp: Date.now(), + source: "test", + }); + await assertRejects(() => sub.flush(), Deno.errors.PermissionDenied); + const callsAfterDenial = filesystemCalls; + + subscriber({ + id: "ignored", + level: "error", + message: "second", + timestamp: Date.now(), + source: "test", + }); + await sub.flush(); + + assertEquals(filesystemCalls, callsAfterDenial); + } finally { + console.error = originalError; + await sub.close(); + } + }); + } + + it("truncates a partial record and keeps a recovered handle retryable", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/partial.log` })); + let writeCalls = 0; + let closeCalls = 0; + const truncations: number[] = []; + let fail = true; + const internals = sub as unknown as { + file: { + write(bytes: Uint8Array): Promise; + truncate(length: number): Promise; + seek(offset: number, whence: number): Promise; + sync(): Promise; + close(): void; + } | null; + currentSize: number; + }; + internals.file = { + write(bytes) { + writeCalls++; + if (fail && writeCalls === 1) return Promise.resolve(Math.min(1, bytes.length)); + if (fail) { + fail = false; + return Promise.reject(new Error("device unavailable")); + } + return Promise.resolve(bytes.length); + }, + truncate(length) { + truncations.push(length); + return Promise.resolve(); + }, + seek(offset) { + return Promise.resolve(offset); + }, + sync() { + return Promise.resolve(); + }, + close() { + closeCalls++; + }, + }; + internals.currentSize = 100; + const originalError = console.error; + console.error = () => {}; + + try { + sub.getSubscriber()({ + id: "partial", + level: "error", + message: "partial write", + timestamp: Date.now(), + source: "test", + }); + await assertRejects(() => sub.flush(), Error, "device unavailable"); + sub.getSubscriber()({ + id: "recovered", + level: "error", + message: "complete record", + timestamp: Date.now(), + source: "test", + }); + await sub.flush(); + } finally { + console.error = originalError; + await sub.close(); + } + + assertEquals(writeCalls, 3); + assertEquals(truncations, [100]); + assertEquals(closeCalls, 1); + assertEquals(internals.file, null); + assertEquals(internals.currentSize > 100, true); + }); + + it("preserves the write failure when failure reporting itself throws", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: dir })); + const originalError = console.error; + console.error = () => { + throw new Error("console unavailable"); + }; + + try { + sub.getSubscriber()({ + id: "failure", + level: "error", + message: "cannot write", + timestamp: Date.now(), + source: "test", + }); + await assertRejects(() => sub.flush(), Error); + } finally { + console.error = originalError; + await sub.close(); + } + }); + + it("bounds queued entries when the filesystem stops making progress", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/bounded.log` })); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const internals = sub as unknown as { + pendingWrites: number; + writeEntry(entry: unknown): Promise; + }; + internals.writeEntry = () => gate; + const subscriber = sub.getSubscriber(); + const entry = { + id: "queued", + level: "info" as const, + message: "queued", + timestamp: Date.now(), + source: "test", + }; + + for (let index = 0; index <= MAX_FILE_LOG_PENDING_WRITES; index++) { + subscriber({ ...entry, id: `queued-${index}` }); + } + + assertEquals(internals.pendingWrites, MAX_FILE_LOG_PENDING_WRITES); + release(); + await assertRejects(() => sub.flush(), Error, "capacity"); + await sub.close(); + }); + + it("retains only a bounded sample of repeated file-log failures", () => { + const sub = new FileLogSubscriber(makeConfig({ path: "ignored.log" })); + const internals = sub as unknown as { + pendingFailures: unknown[]; + omittedFailureCount: number; + recordFailure(error: unknown): void; + }; + + for (let index = 0; index < MAX_FILE_LOG_RETAINED_FAILURES + 10; index++) { + internals.recordFailure(new Error(`failure-${index}`)); + } + + assertEquals(internals.pendingFailures.length, MAX_FILE_LOG_RETAINED_FAILURES); + assertEquals(internals.omittedFailureCount, 10); + }); + + it("shares one recorded failure across concurrent flush callers", async () => { + const sub = new FileLogSubscriber(makeConfig({ path: "ignored.log" })); + const internals = sub as unknown as { + recordFailure(error: unknown): void; + }; + internals.recordFailure(new Error("shared failure")); + + const first = sub.flush(); + const second = sub.flush(); + + assertStrictEquals(second, first); + const results = await Promise.allSettled([first, second]); + assertEquals( + results.map((result) => result.status), + ["rejected", "rejected"], + ); + await sub.close(); + }); + + it("closes and clears the file even when flushing rejects", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/close.log` })); + let closeCalls = 0; + const internals = sub as unknown as { + file: { close(): void } | null; + writeQueue: Promise; + }; + internals.file = { close: () => closeCalls++ }; + internals.writeQueue = Promise.reject(new Error("flush failed")); + + await assertRejects(() => sub.close(), Error, "flush failed"); + + assertEquals(closeCalls, 1); + assertEquals(internals.file, null); + }); + + it("surfaces durability sync failures from flush", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/sync.log` })); + const internals = sub as unknown as { + file: { sync(): Promise; close(): void } | null; + }; + internals.file = { + sync: () => Promise.reject(new Error("sync unavailable")), + close: () => {}, + }; + + await assertRejects(() => sub.flush(), Error, "sync unavailable"); + await assertRejects(() => sub.close(), Error, "sync unavailable"); + }); + + it("shares a transient close failure, then retries cleanup exactly once", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/close-error.log` })); + let closeCalls = 0; + const internals = sub as unknown as { + file: { sync(): Promise; close(): void } | null; + }; + const file = { + sync: () => Promise.resolve(), + close: () => { + closeCalls++; + if (closeCalls === 1) throw new Error("close temporarily unavailable"); + }, + }; + internals.file = file; + + const concurrentResults = await Promise.allSettled([ + sub.close(), + sub.close(), + ]); + + assertEquals( + concurrentResults.map((result) => result.status), + ["rejected", "rejected"], + ); + assertEquals(closeCalls, 1); + assertEquals(internals.file, file); + + await Promise.all([sub.close(), sub.close()]); + + assertEquals(closeCalls, 2); + assertEquals(internals.file, null); + + await sub.close(); + assertEquals(closeCalls, 2); + }); + + it("shares one successful close attempt across concurrent callers", async () => { + const dir = await makeTempDir(); + const sub = new FileLogSubscriber(makeConfig({ path: `${dir}/close-once.log` })); + let closeCalls = 0; + let signalSyncStarted!: () => void; + let releaseSync!: () => void; + const syncStarted = new Promise((resolve) => { + signalSyncStarted = resolve; + }); + const syncGate = new Promise((resolve) => { + releaseSync = resolve; + }); + const internals = sub as unknown as { + file: { sync(): Promise; close(): void } | null; + }; + internals.file = { + sync: () => { + signalSyncStarted(); + return syncGate; + }, + close: () => { + closeCalls++; + }, + }; + + const firstClose = sub.close(); + await syncStarted; + const secondClose = sub.close(); + releaseSync(); + await Promise.all([firstClose, secondClose]); + + assertEquals(closeCalls, 1); + assertEquals(internals.file, null); + + await sub.close(); + assertEquals(closeCalls, 1); + }); + + it("keeps the passive subscriber callback fail-open for hostile entries", () => { + const sub = new FileLogSubscriber(makeConfig({ path: "ignored.log" })); + const entry = { + id: "hostile", + level: "error" as const, + get message(): string { + throw new Error("message unavailable"); + }, + timestamp: Date.now(), + source: "test", + }; + + sub.getSubscriber()(entry); + }); }); }); diff --git a/src/observability/file-log-subscriber.ts b/src/observability/file-log-subscriber.ts index b6cbebdd02..2be0a4cf66 100644 --- a/src/observability/file-log-subscriber.ts +++ b/src/observability/file-log-subscriber.ts @@ -1,11 +1,23 @@ +import { dirname } from "veryfront/platform/path"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { MAX_FILE_LOG_FILES } from "#veryfront/utils/config-resource-limits.ts"; +import { sanitizeUrlCredentials } from "#veryfront/utils/logger/redact.ts"; import { serverLogger } from "#veryfront/utils/logger/logger.ts"; import type { LogEntry, LogLevel, LogSubscriber } from "./log-buffer.ts"; +import { + MAX_FILE_LOG_PENDING_WRITES, + MAX_FILE_LOG_RETAINED_FAILURES, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + MAX_OBSERVABILITY_NAME_LENGTH, +} from "./limits.ts"; +import { sanitizeStructuredTelemetryData, sanitizeTelemetryText } from "./telemetry-error.ts"; /** Configuration used by file log. */ export interface FileLogConfig { enabled: boolean; path: string; maxSize: number | string; + /** Total retained files, including the active log file. */ maxFiles: number; level: LogLevel; format: "json" | "text"; @@ -27,7 +39,21 @@ const SIZE_UNITS: Record = { /** Parses max size. */ export function parseMaxSize(value: number | string): number { - if (typeof value === "number") return value; + if (typeof value === "number") { + const bytes = Math.floor(value); + if (!Number.isFinite(value) || bytes <= 0) { + throw new RangeError("File log maxSize must be a positive finite number"); + } + return bytes; + } + if (typeof value !== "string") { + throw new TypeError("File log maxSize must be a number or string"); + } + if (value.length > MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH) { + throw new RangeError( + `File log maxSize text must not exceed ${MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH} characters`, + ); + } const match = value.trim().toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(b|kb|mb|gb)?$/); if (!match?.[1]) { @@ -36,18 +62,106 @@ export function parseMaxSize(value: number | string): number { const num = parseFloat(match[1]); const unit = match[2] ?? "b"; - return Math.floor(num * (SIZE_UNITS[unit] ?? 1)); + const bytes = Math.floor(num * (SIZE_UNITS[unit] ?? 1)); + if (!Number.isFinite(bytes) || bytes <= 0) { + throw new RangeError("File log maxSize must be a positive finite number"); + } + return bytes; } function formatEntryText(entry: LogEntry): string { const time = new Date(entry.timestamp).toISOString(); const level = entry.level.toUpperCase().padEnd(5); - const data = entry.data ? ` ${JSON.stringify(entry.data)}` : ""; + const data = entry.data ? ` ${safeJsonStringify(entry.data)}` : ""; return `${time} ${level} [${entry.source}] ${entry.message}${data}`; } function formatEntryJson(entry: LogEntry): string { - return JSON.stringify(entry); + return safeJsonStringify(entry); +} + +function safeJsonStringify(value: unknown): string { + return JSON.stringify( + value, + (_key, child) => typeof child === "bigint" ? child.toString() : child, + ) ?? ""; +} + +function sanitizeEntry(entry: LogEntry): LogEntry { + if (entry === null || typeof entry !== "object") { + throw new TypeError("File log entry must be an object"); + } + if (!Object.hasOwn(LOG_LEVEL_PRIORITY, entry.level)) { + throw new TypeError(`Invalid file log entry level: ${String(entry.level)}`); + } + if ( + typeof entry.id !== "string" || typeof entry.message !== "string" || + typeof entry.source !== "string" + ) { + throw new TypeError("File log entry id, message, and source must be strings"); + } + if ( + typeof entry.timestamp !== "number" || + !Number.isFinite(new Date(entry.timestamp).getTime()) + ) { + throw new RangeError("File log entry timestamp must be a valid date"); + } + + return { + id: sanitizeTelemetryText(entry.id, MAX_OBSERVABILITY_NAME_LENGTH), + level: entry.level, + message: sanitizeTelemetryText(entry.message, MAX_STRING_DISPLAY_LENGTH), + data: entry.data === undefined ? undefined : sanitizeStructuredTelemetryData(entry.data), + timestamp: entry.timestamp, + source: sanitizeTelemetryText( + entry.source, + MAX_OBSERVABILITY_NAME_LENGTH, + ), + }; +} + +function isPermissionDenied(error: unknown): boolean { + try { + if (error instanceof Deno.errors.PermissionDenied) return true; + if (error instanceof AggregateError) { + return error.errors.some((failure) => isPermissionDenied(failure)); + } + return false; + } catch (_) { + return false; + } +} + +function describeFailure(error: unknown): string { + try { + if (error instanceof Error && typeof error.message === "string") return error.message; + } catch (_) { + // Fall through to the guarded string conversion. + } + try { + return String(error); + } catch (_) { + return "Unknown file logging failure"; + } +} + +/** Writes every byte, including when the underlying writer makes partial progress. */ +export async function writeAll( + writer: { write(bytes: Uint8Array): Promise }, + bytes: Uint8Array, +): Promise { + let offset = 0; + while (offset < bytes.length) { + const remaining = bytes.subarray(offset); + const written = await writer.write(remaining); + if (!Number.isSafeInteger(written) || written <= 0 || written > remaining.length) { + if (written === 0) { + throw new Error("File write made zero bytes of progress"); + } + throw new Error(`File write returned an invalid byte count: ${written}`); + } + offset += written; + } } /** Implement file log subscriber. */ @@ -55,40 +169,127 @@ export class FileLogSubscriber { private file: Deno.FsFile | null = null; private currentSize = 0; private writeQueue: Promise = Promise.resolve(); + private pendingFailures: unknown[] = []; + private pendingWrites = 0; + private omittedFailureCount = 0; + private flushPromise: Promise | null = null; + private closePromise: Promise | null = null; private maxSizeBytes: number; private minLevel: number; private formatter: (entry: LogEntry) => string; private closed = false; private permissionFailed = false; + private reportingFailure = false; private config: FileLogConfig; private readonly encoder = new TextEncoder(); constructor(config: FileLogConfig) { - this.config = config; - this.maxSizeBytes = parseMaxSize(config.maxSize); - this.minLevel = LOG_LEVEL_PRIORITY[config.level]; - this.formatter = config.format === "json" ? formatEntryJson : formatEntryText; + if (config === null || typeof config !== "object") { + throw new TypeError("File log config must be an object"); + } + if ( + typeof config.path !== "string" || !config.path.trim() || + config.path.length > MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH + ) { + throw new TypeError( + `File log path must contain between 1 and ${MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH} characters`, + ); + } + if (typeof config.enabled !== "boolean") { + throw new TypeError("File log enabled must be a boolean"); + } + if ( + !Number.isSafeInteger(config.maxFiles) || + config.maxFiles <= 0 || + config.maxFiles > MAX_FILE_LOG_FILES + ) { + throw new RangeError( + `File log maxFiles must be an integer between 1 and ${MAX_FILE_LOG_FILES}`, + ); + } + if (!Object.hasOwn(LOG_LEVEL_PRIORITY, config.level)) { + throw new TypeError(`Invalid file log level: ${String(config.level)}`); + } + if (config.format !== "json" && config.format !== "text") { + throw new TypeError(`Invalid file log format: ${String(config.format)}`); + } + + this.config = { + enabled: config.enabled, + path: config.path, + maxSize: config.maxSize, + maxFiles: config.maxFiles, + level: config.level, + format: config.format, + }; + this.maxSizeBytes = parseMaxSize(this.config.maxSize); + this.minLevel = LOG_LEVEL_PRIORITY[this.config.level]; + this.formatter = this.config.format === "json" ? formatEntryJson : formatEntryText; } getSubscriber(): LogSubscriber { return (entry: LogEntry) => { - if (this.closed || this.permissionFailed) return; - if (LOG_LEVEL_PRIORITY[entry.level] < this.minLevel) return; - this.enqueue(entry); + try { + if ( + !this.config.enabled || this.closed || this.permissionFailed || this.reportingFailure + ) return; + if (LOG_LEVEL_PRIORITY[entry.level] < this.minLevel) return; + this.enqueue(sanitizeEntry(entry)); + } catch (error) { + this.reportFailure( + `[FileLogSubscriber] Failed to accept a log entry for ${this.config.path}.`, + error, + ); + } }; } private enqueue(entry: LogEntry): void { - this.writeQueue = this.writeQueue.then(() => this.writeEntry(entry)).catch((error) => { - serverLogger.error( - `FileLogSubscriber: failed writing to ${this.config.path}, file logging will continue`, - { error: error instanceof Error ? error : new Error(String(error)) }, + if (this.pendingWrites >= MAX_FILE_LOG_PENDING_WRITES) { + const error = new RangeError( + `File log pending-write capacity of ${MAX_FILE_LOG_PENDING_WRITES} was reached`, ); - }); + this.recordFailure(error); + this.reportFailure( + `FileLogSubscriber: dropped an entry for ${this.config.path} because the write queue reached capacity`, + error, + ); + return; + } + + this.pendingWrites++; + this.writeQueue = this.writeQueue + .then(() => this.writeEntry(entry)) + .catch((error) => { + this.recordFailure(error); + if (this.disableForPermissionFailure(error)) { + this.reportFailure( + `FileLogSubscriber: permission denied writing to ${this.config.path}, file logging disabled`, + error, + ); + } else { + this.reportFailure( + `FileLogSubscriber: failed writing to ${this.config.path}, file logging will continue`, + error, + ); + } + }) + .finally(() => { + this.pendingWrites--; + }); + } + + private recordFailure(error: unknown): void { + if (this.pendingFailures.length < MAX_FILE_LOG_RETAINED_FAILURES) { + this.pendingFailures.push(error); + } else { + this.omittedFailureCount++; + } } private async writeEntry(entry: LogEntry): Promise { try { + if (this.permissionFailed) return; if (!this.file) await this.openFile(); const line = this.formatter(entry) + "\n"; @@ -98,17 +299,50 @@ export class FileLogSubscriber { await this.rotate(); } - await this.file!.write(bytes); - this.currentSize += bytes.length; - } catch (err) { - if (err instanceof Deno.errors.PermissionDenied) { - this.permissionFailed = true; - serverLogger.error( - `FileLogSubscriber: permission denied writing to ${this.config.path}, file logging disabled`, - ); - return; + const file = this.file!; + const recordStart = this.currentSize; + try { + await writeAll(file, bytes); + this.currentSize += bytes.length; + } catch (error) { + this.disableForPermissionFailure(error); + const recoveryFailure = await this.rollbackPartialRecord(file, recordStart); + if (recoveryFailure !== undefined) { + throw new AggregateError( + [error, recoveryFailure], + "File write failed and its partial record could not be rolled back", + ); + } + throw error; + } + } catch (error) { + if (this.disableForPermissionFailure(error)) { + this.closeCurrentFileQuietly(); + this.currentSize = 0; } - throw err; + throw error; + } + } + + private disableForPermissionFailure(error: unknown): boolean { + if (!isPermissionDenied(error)) return false; + this.permissionFailed = true; + return true; + } + + private async rollbackPartialRecord( + file: Deno.FsFile, + recordStart: number, + ): Promise { + try { + await file.truncate(recordStart); + await file.seek(0, Deno.SeekMode.End); + this.currentSize = recordStart; + return undefined; + } catch (error) { + this.closeCurrentFileQuietly(); + this.currentSize = 0; + return error; } } @@ -122,23 +356,64 @@ export class FileLogSubscriber { try { const stat = await this.file.stat(); this.currentSize = stat.size; - } catch { - this.currentSize = 0; + } catch (error) { + this.closeCurrentFileQuietly(); + throw error; } } private async ensureDir(): Promise { - const dir = this.config.path.substring(0, this.config.path.lastIndexOf("/")); - if (dir) { + const dir = dirname(this.config.path); + if (dir !== ".") { await Deno.mkdir(dir, { recursive: true }); } } - private async rotate(): Promise { - if (this.file) { - this.file.close(); - this.file = null; + private reportFailure(message: string, error?: unknown): void { + if (this.reportingFailure) return; + this.reportingFailure = true; + try { + const normalizedError = error === undefined + ? undefined + : error instanceof Error + ? error + : new Error(describeFailure(error)); + serverLogger.error( + sanitizeUrlCredentials(message), + ...(normalizedError ? [{ error: normalizedError }] : []), + ); + } catch { + // Diagnostics must never break the logging queue or application code. + } finally { + this.reportingFailure = false; + } + } + + private closeCurrentFile(): void { + const file = this.file; + this.file = null; + if (!file) return; + file.close(); + } + + private closeCurrentFileQuietly(): void { + try { + this.closeCurrentFile(); + } catch (_) { + /* expected: recovery retains the primary I/O failure */ } + } + + private closeCurrentFileForShutdown(): void { + const file = this.file; + if (!file) return; + // A failed shutdown close is retryable, so retain ownership until close succeeds. + file.close(); + if (this.file === file) this.file = null; + } + + private async rotate(): Promise { + this.closeCurrentFile(); for (let i = this.config.maxFiles - 1; i >= 1; i--) { const from = i === 1 ? this.config.path : `${this.config.path}.${i - 1}`; @@ -166,28 +441,75 @@ export class FileLogSubscriber { this.currentSize = 0; } - async flush(): Promise { - await this.writeQueue; + flush(): Promise { + if (this.flushPromise) return this.flushPromise; + + const attempt = this.performFlush(); + const tracked = attempt.finally(() => { + if (this.flushPromise === tracked) this.flushPromise = null; + }); + this.flushPromise = tracked; + return tracked; + } + + private async performFlush(): Promise { + const failures: unknown[] = []; + try { + await this.writeQueue; + } catch (error) { + // Defensive compatibility for an already-rejected queue created by an + // older owner or an injected adapter. + failures.push(error); + } + failures.push(...this.pendingFailures.splice(0)); + if (this.omittedFailureCount > 0) { + failures.push( + new Error( + `${this.omittedFailureCount} additional file-log failures were omitted`, + ), + ); + this.omittedFailureCount = 0; + } if (this.file) { try { await this.file.sync(); - } catch { - // file may already be closed + } catch (error) { + this.disableForPermissionFailure(error); + failures.push(error); } } + this.throwFailures(failures, "File log flush failed"); } async close(): Promise { + if (this.closePromise) return await this.closePromise; this.closed = true; - await this.flush(); - if (this.file) { + const closeAttempt = (async () => { + const failures: unknown[] = []; try { - this.file.close(); - } catch { - // already closed + await this.flush(); + } catch (error) { + failures.push(error); } - this.file = null; - } + try { + this.closeCurrentFileForShutdown(); + } catch (error) { + failures.push(error); + } + this.throwFailures(failures, "File log close failed"); + })(); + const trackedClose = closeAttempt.catch((error) => { + this.closePromise = null; + throw error; + }); + this.closePromise = trackedClose; + return await trackedClose; + } + + private throwFailures(failures: readonly unknown[], message: string): void { + if (failures.length === 0) return; + if (failures.length === 1) throw failures[0]; + throw new AggregateError(failures, message); } } diff --git a/src/observability/index.test.ts b/src/observability/index.test.ts index 7084b64f70..26f6e8bb68 100644 --- a/src/observability/index.test.ts +++ b/src/observability/index.test.ts @@ -1,9 +1,104 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertStrictEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import * as localObservability from "./index.ts"; import * as observability from "veryfront/observability"; +const expectedRuntimeExports = [ + "ApplicationErrorReporterInitializerName", + "ErrorCollector", + "FileLogSubscriber", + "LogBuffer", + "SpanKind", + "SpanNames", + "SpanStatusCode", + "addSpanEvent", + "createChildSpan", + "createFileLogSubscriber", + "createOpenTelemetryServiceTracer", + "captureApplicationError", + "endSpan", + "extractContext", + "getActiveContext", + "getErrorCollector", + "getGlobalMetricsAPI", + "getHostTelemetryEnv", + "getLogBuffer", + "getMetricsState", + "getTraceContext", + "flushApplicationErrors", + "initAutoInstrumentation", + "initMetrics", + "initTracing", + "initializeOTLP", + "initializeApplicationErrorReporter", + "injectContext", + "instrument", + "instrumentBatch", + "instrumentErrorHandler", + "instrumentFetch", + "instrumentHttpHandler", + "instrumentReactRender", + "instrumentSync", + "interceptConsole", + "isAutoInstrumentEnabled", + "isMetricsEnabled", + "isOTLPEnabled", + "isReservedSharedRuntimeTelemetryEnvKey", + "isTracingDegraded", + "isTracingEnabled", + "markRequestProfilePhase", + "metrics", + "parseCompileError", + "parseMaxSize", + "profilePhase", + "profileSyncPhase", + "recordApiRequest", + "recordApiRetry", + "recordBuild", + "recordBundle", + "recordCacheGet", + "recordCacheInvalidate", + "recordCacheSet", + "recordContentCacheHit", + "recordContentNetworkFetch", + "recordCorsRejection", + "recordDataFetch", + "recordDataFetchError", + "recordErrorCount", + "recordHttpRequest", + "recordHttpRequestComplete", + "recordRSCError", + "recordRSCRender", + "recordRSCRequest", + "recordRSCStream", + "recordRender", + "recordRenderError", + "recordSecurityHeaders", + "resetErrorCollector", + "resetLogBuffer", + "setActiveSpanAttributes", + "setCacheSize", + "setSpanAttributes", + "shutdownMetrics", + "shutdownOTLP", + "shutdownTracing", + "snapshotRequestProfiles", + "startSpan", + "trace", + "withActiveSpan", + "withSpan", + "withSpanSync", +].sort(); + describe("veryfront/observability public export surface", () => { + it("preserves the exact runtime surface and package mapping", () => { + assertEquals(Object.keys(localObservability).sort(), expectedRuntimeExports); + assertEquals(Object.keys(observability).sort(), expectedRuntimeExports); + assertStrictEquals(observability.withSpan, localObservability.withSpan); + assertStrictEquals(observability.metrics, localObservability.metrics); + }); + it("does not expose test resets or mutable metrics state", () => { assertEquals("_resetShimForTests" in observability, false); assertEquals("resetMetrics" in observability, false); diff --git a/src/observability/index.ts b/src/observability/index.ts index 42e0b862b8..ace63add05 100644 --- a/src/observability/index.ts +++ b/src/observability/index.ts @@ -166,3 +166,16 @@ export { FileLogSubscriber, parseMaxSize, } from "./file-log-subscriber.ts"; + +export { + type ApplicationErrorContext, + type ApplicationErrorReporter, + type ApplicationErrorReporterInitializationContext, + type ApplicationErrorReporterInitializer, + ApplicationErrorReporterInitializerName, + type ApplicationErrorReporterLifecycle, + type ApplicationErrorReporterSession, + captureApplicationError, + flushApplicationErrors, + initializeApplicationErrorReporter, +} from "./application-errors.ts"; diff --git a/src/observability/instruments/cache-instruments.ts b/src/observability/instruments/cache-instruments.ts index 7bc244e7ba..b01196be71 100644 --- a/src/observability/instruments/cache-instruments.ts +++ b/src/observability/instruments/cache-instruments.ts @@ -5,6 +5,7 @@ import type { ObservableResult, } from "#veryfront/observability/tracing/api-shim.ts"; import type { MetricsConfig, RuntimeState } from "../metrics/types.ts"; +import type { ObservableCallbackBinding } from "./observable-callbacks.ts"; export interface CacheInstruments { cacheGetCounter: Counter | null; @@ -18,7 +19,6 @@ export interface CacheInstruments { export function createCacheInstruments( meter: Meter, config: MetricsConfig, - runtimeState: RuntimeState, ): CacheInstruments { const prefix = `${config.prefix}.cache`; @@ -52,8 +52,6 @@ export function createCacheInstruments( unit: "entries", }); - cacheSizeGauge.addCallback((result: ObservableResult) => result.observe(runtimeState.cacheSize)); - return { cacheGetCounter, cacheHitCounter, @@ -63,3 +61,14 @@ export function createCacheInstruments( cacheSizeGauge, }; } + +export function createCacheObservableBindings( + instruments: CacheInstruments, + runtimeState: RuntimeState, +): ObservableCallbackBinding[] { + if (!instruments.cacheSizeGauge) return []; + return [{ + instrument: instruments.cacheSizeGauge, + callback: (result: ObservableResult) => result.observe(runtimeState.cacheSize), + }]; +} diff --git a/src/observability/instruments/error-instruments.test.ts b/src/observability/instruments/error-instruments.test.ts index 7c70edb661..699ffe1bcb 100644 --- a/src/observability/instruments/error-instruments.test.ts +++ b/src/observability/instruments/error-instruments.test.ts @@ -4,7 +4,7 @@ import "#veryfront/schemas/_test-setup.ts"; */ import { describe, it } from "#veryfront/testing/bdd"; -import { assertEquals } from "#veryfront/testing/assert"; +import { assertEquals, assertExists } from "#veryfront/testing/assert"; import { recordError } from "./error-instruments.ts"; import { CONFIG_NOT_FOUND, RENDER_ERROR } from "#veryfront/errors/error-registry.ts"; import type { Counter } from "#veryfront/observability/tracing/api-shim.ts"; @@ -27,10 +27,12 @@ describe("error-instruments", () => { recordError(error, mockCounter); assertEquals(calls.length, 1); - assertEquals(calls[0].value, 1); - assertEquals(calls[0].attributes.slug, "config-not-found"); - assertEquals(calls[0].attributes.category, "CONFIG"); - assertEquals(calls[0].attributes.status, "404"); + const call = calls[0]; + assertExists(call); + assertEquals(call.value, 1); + assertEquals(call.attributes.slug, "config-not-found"); + assertEquals(call.attributes.category, "CONFIG"); + assertEquals(call.attributes.status, "404"); }); it("should handle different error types", () => { @@ -47,9 +49,11 @@ describe("error-instruments", () => { recordError(error, mockCounter); assertEquals(calls.length, 1); - assertEquals(calls[0].attributes.slug, "render-error"); - assertEquals(calls[0].attributes.category, "RUNTIME"); - assertEquals(calls[0].attributes.status, "500"); + const call = calls[0]; + assertExists(call); + assertEquals(call.attributes.slug, "render-error"); + assertEquals(call.attributes.category, "RUNTIME"); + assertEquals(call.attributes.status, "500"); }); it("should do nothing when counter is null", () => { @@ -66,6 +70,17 @@ describe("error-instruments", () => { recordError(error, undefined); }); + it("does not let a metrics backend failure escape", () => { + const error = CONFIG_NOT_FOUND.create(); + const counter = { + add() { + throw new Error("metrics backend unavailable"); + }, + } as Counter; + + recordError(error, counter); + }); + it("should convert status to string", () => { const calls: Array<{ value: number; attributes: Record }> = []; @@ -79,8 +94,10 @@ describe("error-instruments", () => { recordError(error, mockCounter); - assertEquals(typeof calls[0].attributes.status, "string"); - assertEquals(calls[0].attributes.status, "404"); + const call = calls[0]; + assertExists(call); + assertEquals(typeof call.attributes.status, "string"); + assertEquals(call.attributes.status, "404"); }); }); }); diff --git a/src/observability/instruments/error-instruments.ts b/src/observability/instruments/error-instruments.ts index a71be7e5c6..b49fc5da51 100644 --- a/src/observability/instruments/error-instruments.ts +++ b/src/observability/instruments/error-instruments.ts @@ -46,9 +46,13 @@ export function recordError( return; } - errorCounter.add(1, { - slug: error.slug, - category: error.category, - status: String(error.status), - }); + try { + errorCounter.add(1, { + slug: error.slug, + category: error.category, + status: String(error.status), + }); + } catch (_) { + /* expected: metrics failures must never affect error handling */ + } } diff --git a/src/observability/instruments/instruments-factory.test.ts b/src/observability/instruments/instruments-factory.test.ts new file mode 100644 index 0000000000..874ae100c1 --- /dev/null +++ b/src/observability/instruments/instruments-factory.test.ts @@ -0,0 +1,45 @@ +import { assertEquals, assertNotEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import type { Meter, ObservableResult } from "#veryfront/observability/tracing/api-shim.ts"; +import { disposeInstruments, initializeInstruments } from "./instruments-factory.ts"; + +describe("observability/instruments/instruments-factory", () => { + it("does not leak observable callbacks from a partial initialization and remains retryable", () => { + const callbacks = new Set<(result: ObservableResult) => void>(); + let failRenderFamily = true; + const writableInstrument = { add() {}, record() {} }; + const meter: Meter = { + createCounter: () => writableInstrument, + createUpDownCounter: () => writableInstrument, + createHistogram: (name) => { + if (failRenderFamily && name.includes(".render.duration")) { + throw new Error("render histogram unavailable"); + } + return writableInstrument; + }, + createObservableGauge: () => ({ + addCallback: (callback) => callbacks.add(callback), + removeCallback: (callback) => callbacks.delete(callback), + }), + }; + const config = { + enabled: true, + exporter: "console" as const, + prefix: "test", + }; + const runtimeState = { cacheSize: 0, activeRequests: 0 }; + + const failed = initializeInstruments(meter, config, runtimeState); + + assertEquals(failed.cacheSizeGauge, null); + assertEquals(callbacks.size, 0); + + failRenderFamily = false; + const initialized = initializeInstruments(meter, config, runtimeState); + assertNotEquals(initialized.cacheSizeGauge, null); + assertEquals(callbacks.size, 5); + + disposeInstruments(initialized); + assertEquals(callbacks.size, 0); + }); +}); diff --git a/src/observability/instruments/instruments-factory.ts b/src/observability/instruments/instruments-factory.ts index de8706a1f1..fa5598952b 100644 --- a/src/observability/instruments/instruments-factory.ts +++ b/src/observability/instruments/instruments-factory.ts @@ -2,24 +2,23 @@ import type { Meter } from "#veryfront/observability/tracing/api-shim.ts"; import { serverLogger } from "#veryfront/utils/logger/logger.ts"; import type { MetricsConfig, MetricsInstruments, RuntimeState } from "../metrics/types.ts"; import { createBuildInstruments } from "./build-instruments.ts"; -import { createCacheInstruments } from "./cache-instruments.ts"; +import { createCacheInstruments, createCacheObservableBindings } from "./cache-instruments.ts"; import { createDataInstruments } from "./data-instruments.ts"; import { createErrorInstruments } from "./error-instruments.ts"; import { createHttpInstruments } from "./http-instruments.ts"; -import { createMemoryInstruments } from "./memory-instruments.ts"; +import { createMemoryInstruments, createMemoryObservableBindings } from "./memory-instruments.ts"; import { createModelCallContextInstruments } from "./model-call-context-instruments.ts"; +import { installObservableCallbacks } from "./observable-callbacks.ts"; import { createRenderInstruments } from "./render-instruments.ts"; import { createRscInstruments } from "./rsc-instruments.ts"; import { createStreamLifecycleInstruments } from "./stream-lifecycle-instruments.ts"; const logger = serverLogger.component("metrics"); +const instrumentDisposers = new WeakMap void>(); +const initializedInstrumentSets = new WeakSet(); -export function initializeInstruments( - meter: Meter, - config: MetricsConfig, - runtimeState: RuntimeState, -): MetricsInstruments { - const emptyInstruments: MetricsInstruments = { +export function createEmptyInstruments(): MetricsInstruments { + return { httpRequestCounter: null, httpRequestDuration: null, httpActiveRequests: null, @@ -69,12 +68,30 @@ export function initializeInstruments( modelCallContextAppendRequestCount: null, modelCallContextRecorderBarrierDuration: null, }; +} + +export function disposeInstruments(instruments: MetricsInstruments): void { + const dispose = instrumentDisposers.get(instruments); + instrumentDisposers.delete(instruments); + dispose?.(); +} + +export function isInitializedInstrumentSet(instruments: MetricsInstruments): boolean { + return initializedInstrumentSets.has(instruments); +} + +export function initializeInstruments( + meter: Meter, + config: MetricsConfig, + runtimeState: RuntimeState, +): MetricsInstruments { + const emptyInstruments = createEmptyInstruments(); try { - return { + const instruments: MetricsInstruments = { ...emptyInstruments, ...createHttpInstruments(meter, config), - ...createCacheInstruments(meter, config, runtimeState), + ...createCacheInstruments(meter, config), ...createRenderInstruments(meter, config), ...createRscInstruments(meter, config), ...createBuildInstruments(meter, config), @@ -84,6 +101,13 @@ export function initializeInstruments( ...createStreamLifecycleInstruments(meter, config), ...createModelCallContextInstruments(meter, config), }; + const dispose = installObservableCallbacks([ + ...createCacheObservableBindings(instruments, runtimeState), + ...createMemoryObservableBindings(instruments), + ]); + instrumentDisposers.set(instruments, dispose); + initializedInstrumentSets.add(instruments); + return instruments; } catch (error) { logger.warn("Failed to initialize metric instruments", error); return emptyInstruments; diff --git a/src/observability/instruments/memory-instruments.test.ts b/src/observability/instruments/memory-instruments.test.ts new file mode 100644 index 0000000000..8ec8766710 --- /dev/null +++ b/src/observability/instruments/memory-instruments.test.ts @@ -0,0 +1,16 @@ +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { parseV8HeapLimitBytes } from "./memory-instruments.ts"; + +describe("observability/instruments/memory-instruments", () => { + it("parses explicit V8 heap limits without deployment-specific defaults", () => { + assertEquals( + parseV8HeapLimitBytes("--max-old-space-size=4096"), + 4096 * 1024 * 1024, + ); + assertEquals(parseV8HeapLimitBytes("--max_old_space_size 2048"), 2048 * 1024 * 1024); + assertEquals(parseV8HeapLimitBytes(""), undefined); + assertEquals(parseV8HeapLimitBytes("--max-old-space-size=0"), undefined); + assertEquals(parseV8HeapLimitBytes("--max-old-space-size=invalid"), undefined); + }); +}); diff --git a/src/observability/instruments/memory-instruments.ts b/src/observability/instruments/memory-instruments.ts index ab4e2e03e4..b80b0318b1 100644 --- a/src/observability/instruments/memory-instruments.ts +++ b/src/observability/instruments/memory-instruments.ts @@ -6,14 +6,54 @@ import type { import { getV8FlagsEnv } from "#veryfront/config/env.ts"; import { getMemoryUsage } from "../metrics/config.ts"; import type { MetricsConfig } from "../metrics/types.ts"; +import type { ObservableCallbackBinding } from "./observable-callbacks.ts"; -let _v8HeapLimitMB: number | undefined; -function getV8HeapLimitMB(): number { - if (_v8HeapLimitMB !== undefined) return _v8HeapLimitMB; - const match = getV8FlagsEnv().match(/--max-old-space-size=(\d+)/); - const value = match?.[1]; - _v8HeapLimitMB = value ? parseInt(value, 10) : 5120; // Default from values.yaml - return _v8HeapLimitMB; +const BYTES_PER_MEBIBYTE = 1024 * 1024; + +/** Parse the configured V8 old-space limit, when present. */ +export function parseV8HeapLimitBytes(flags: string): number | undefined { + const value = flags.match(/--max[-_]old[-_]space[-_]size(?:=|\s+)(\d+)/)?.[1]; + if (!value) return undefined; + + const megabytes = Number(value); + const bytes = megabytes * BYTES_PER_MEBIBYTE; + return Number.isSafeInteger(megabytes) && megabytes > 0 && Number.isSafeInteger(bytes) + ? bytes + : undefined; +} + +let cachedV8HeapLimitBytes: number | null | undefined; +let pendingV8HeapLimitBytes: Promise | undefined; + +async function getV8HeapLimitBytes(): Promise { + if (cachedV8HeapLimitBytes !== undefined) return cachedV8HeapLimitBytes ?? undefined; + + let configuredLimit: number | undefined; + try { + configuredLimit = parseV8HeapLimitBytes(getV8FlagsEnv()); + } catch (_) { + // Host environment access is optional for metrics collection. + } + if (configuredLimit !== undefined) { + cachedV8HeapLimitBytes = configuredLimit; + return configuredLimit; + } + + if (!pendingV8HeapLimitBytes) { + pendingV8HeapLimitBytes = (async () => { + try { + const { getHeapStatistics } = await import("node:v8"); + const runtimeLimit = getHeapStatistics().heap_size_limit; + cachedV8HeapLimitBytes = Number.isFinite(runtimeLimit) && runtimeLimit > 0 + ? runtimeLimit + : null; + } catch (_) { + cachedV8HeapLimitBytes = null; + } + return cachedV8HeapLimitBytes ?? undefined; + })(); + } + return await pendingV8HeapLimitBytes; } export interface MemoryInstruments { @@ -23,18 +63,17 @@ export interface MemoryInstruments { heapPercentGauge: ObservableGauge | null; } -function addMemoryCallback( - gauge: ObservableGauge, +function createMemoryCallback( observe: ( result: ObservableResult, memoryUsage: NonNullable>, ) => void, -): void { - gauge.addCallback((result: ObservableResult) => { +): (result: ObservableResult) => void { + return (result: ObservableResult) => { const memoryUsage = getMemoryUsage(); if (!memoryUsage) return; observe(result, memoryUsage); - }); + }; } export function createMemoryInstruments(meter: Meter, config: MetricsConfig): MemoryInstruments { @@ -42,31 +81,59 @@ export function createMemoryInstruments(meter: Meter, config: MetricsConfig): Me description: "Memory usage (RSS)", unit: "bytes", }); - addMemoryCallback(memoryUsageGauge, (result, memoryUsage) => result.observe(memoryUsage.rss)); - const heapUsageGauge = meter.createObservableGauge(`${config.prefix}.memory.heap`, { description: "V8 heap memory used", unit: "bytes", }); - addMemoryCallback(heapUsageGauge, (result, memoryUsage) => result.observe(memoryUsage.heapUsed)); - const heapTotalGauge = meter.createObservableGauge(`${config.prefix}.memory.heap_total`, { description: "V8 heap memory allocated", unit: "bytes", }); - addMemoryCallback(heapTotalGauge, (result, memoryUsage) => result.observe(memoryUsage.heapTotal)); - // Heap utilization as percentage of configured limit // This is the key metric for autoscaling decisions const heapPercentGauge = meter.createObservableGauge(`${config.prefix}.memory.heap_percent`, { description: "V8 heap usage as percentage of configured limit", unit: "percent", }); - addMemoryCallback(heapPercentGauge, (result, memoryUsage) => { - const heapUsedMB = memoryUsage.heapUsed / (1024 * 1024); - const percent = (heapUsedMB / getV8HeapLimitMB()) * 100; - result.observe(Math.round(percent * 100) / 100); - }); - return { memoryUsageGauge, heapUsageGauge, heapTotalGauge, heapPercentGauge }; } + +export function createMemoryObservableBindings( + instruments: MemoryInstruments, +): ObservableCallbackBinding[] { + const bindings: ObservableCallbackBinding[] = []; + if (instruments.memoryUsageGauge) { + bindings.push({ + instrument: instruments.memoryUsageGauge, + callback: createMemoryCallback((result, memoryUsage) => result.observe(memoryUsage.rss)), + }); + } + if (instruments.heapUsageGauge) { + bindings.push({ + instrument: instruments.heapUsageGauge, + callback: createMemoryCallback((result, memoryUsage) => result.observe(memoryUsage.heapUsed)), + }); + } + if (instruments.heapTotalGauge) { + bindings.push({ + instrument: instruments.heapTotalGauge, + callback: createMemoryCallback((result, memoryUsage) => + result.observe(memoryUsage.heapTotal) + ), + }); + } + if (instruments.heapPercentGauge) { + bindings.push({ + instrument: instruments.heapPercentGauge, + callback: async (result) => { + const heapLimitBytes = await getV8HeapLimitBytes(); + if (heapLimitBytes === undefined) return; + const memoryUsage = getMemoryUsage(); + if (!memoryUsage) return; + const percent = (memoryUsage.heapUsed / heapLimitBytes) * 100; + result.observe(Math.round(percent * 100) / 100); + }, + }); + } + return bindings; +} diff --git a/src/observability/instruments/observable-callbacks.ts b/src/observability/instruments/observable-callbacks.ts new file mode 100644 index 0000000000..4f51193509 --- /dev/null +++ b/src/observability/instruments/observable-callbacks.ts @@ -0,0 +1,63 @@ +import type { + ObservableGauge, + ObservableResult, +} from "#veryfront/observability/tracing/api-shim.ts"; + +export type ObservableCallback = (result: ObservableResult) => void; + +export interface ObservableCallbackBinding { + instrument: ObservableGauge; + callback: ObservableCallback; +} + +/** + * Install callbacks as one reversible transaction. + * + * OpenTelemetry observable instruments support `removeCallback`. Requiring it + * before the first registration prevents a partially compatible provider from + * leaving callbacks behind when a later registration fails. + */ +export function installObservableCallbacks( + bindings: readonly ObservableCallbackBinding[], +): () => void { + for (const { instrument } of bindings) { + if (typeof instrument.addCallback !== "function") { + throw new TypeError("Observable instrument must implement addCallback()"); + } + if (typeof instrument.removeCallback !== "function") { + throw new TypeError("Observable instrument must implement removeCallback()"); + } + } + + const installed: ObservableCallbackBinding[] = []; + try { + for (const binding of bindings) { + binding.instrument.addCallback(binding.callback); + installed.push(binding); + } + } catch (error) { + for (let index = installed.length - 1; index >= 0; index--) { + const binding = installed[index]!; + try { + binding.instrument.removeCallback?.(binding.callback); + } catch (_) { + /* expected: continue rolling back every provider callback */ + } + } + throw error; + } + + let active = true; + return () => { + if (!active) return; + active = false; + for (let index = installed.length - 1; index >= 0; index--) { + const binding = installed[index]!; + try { + binding.instrument.removeCallback?.(binding.callback); + } catch (_) { + /* expected: shutdown remains fail-open if a provider rejects cleanup */ + } + } + }; +} diff --git a/src/observability/limits.ts b/src/observability/limits.ts new file mode 100644 index 0000000000..cde88ffb96 --- /dev/null +++ b/src/observability/limits.ts @@ -0,0 +1,22 @@ +/** + * Resource limits owned by the observability boundary. + * + * OpenTelemetry's stable default attribute-count limit is 128. The remaining + * limits keep core-owned snapshots and asynchronous file-log work bounded + * before an exporter or filesystem adapter receives them. + */ +export const MAX_TELEMETRY_ATTRIBUTE_COUNT = 128; +export const MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH = 255; +export const MAX_TELEMETRY_ATTRIBUTE_ARRAY_LENGTH = 128; + +export const MAX_STRUCTURED_TELEMETRY_DEPTH = 16; +export const MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES = 1_024; +export const MAX_STRUCTURED_TELEMETRY_NODES = 4_096; + +export const MAX_FILE_LOG_PENDING_WRITES = 256; +export const MAX_FILE_LOG_RETAINED_FAILURES = 16; + +export const MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH = 4_096; +export const MAX_OBSERVABILITY_NAME_LENGTH = 255; +export const MAX_APPLICATION_ERROR_CONTEXT_VALUE_LENGTH = 1_000; +export const MAX_REQUEST_PROFILE_PHASES = 128; diff --git a/src/observability/log-buffer.test.ts b/src/observability/log-buffer.test.ts index dda28a4439..c606c9a5e0 100644 --- a/src/observability/log-buffer.test.ts +++ b/src/observability/log-buffer.test.ts @@ -1,10 +1,22 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals, assertExists } from "#veryfront/testing/assert.ts"; +import { + assertEquals, + assertExists, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { MAX_OBSERVABILITY_NAME_LENGTH } from "./limits.ts"; import { interceptConsole, LogBuffer } from "./log-buffer.ts"; describe("observability/log-buffer", () => { describe("LogBuffer", () => { + it("should reject invalid maxSize values", () => { + assertThrows(() => new LogBuffer({ maxSize: -1 }), RangeError, "maxSize"); + assertThrows(() => new LogBuffer({ maxSize: Number.NaN }), RangeError, "maxSize"); + }); + it("should append log entries", () => { const buf = new LogBuffer(); buf.info("hello", "test"); @@ -30,10 +42,90 @@ describe("observability/log-buffer", () => { assertEquals(entry.data?.apiKey, "[REDACTED]"); assertEquals(entry.data?.userId, "u-1"); // Subscribers (incl. the file writer) only ever see the redacted copy. - assertEquals(seen[0].apiKey, "[REDACTED]"); + const subscriberData = seen[0]; + assertExists(subscriberData); + assertEquals(subscriberData.apiKey, "[REDACTED]"); assertEquals(JSON.stringify(entry).includes("sk-secret"), false); }); + it("redacts credentials embedded in log message URLs", () => { + const buf = new LogBuffer(); + + const entry = buf.error( + "request failed: https://user:password@example.test/path?access_token=secret", + ); + + assertEquals(entry.message.includes("password"), false); + assertEquals(entry.message.includes("secret"), false); + assertEquals(entry.message.includes("[REDACTED]"), true); + }); + + it("bounds retained message and source strings", () => { + const buf = new LogBuffer(); + + const entry = buf.info( + "m".repeat(MAX_STRING_DISPLAY_LENGTH + 100), + "s".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + ); + + assertEquals(entry.message.length, MAX_STRING_DISPLAY_LENGTH); + assertEquals(entry.source.length, MAX_OBSERVABILITY_NAME_LENGTH); + assertEquals(buf.getAll()[0]?.message.length, MAX_STRING_DISPLAY_LENGTH); + }); + + it("retains only declared entry fields", () => { + const buf = new LogBuffer(); + + const entry = buf.append({ + level: "info", + message: "message", + source: "test", + undeclared: "must not be retained", + } as never); + + assertEquals(Object.hasOwn(entry, "undeclared"), false); + assertEquals(Object.hasOwn(buf.getAll()[0] ?? {}, "undeclared"), false); + }); + + it("does not expose retained entries to caller or subscriber mutation", () => { + const buf = new LogBuffer(); + buf.subscribe((entry) => { + entry.message = "subscriber mutation"; + if (entry.data) entry.data.value = "subscriber mutation"; + }); + + const returned = buf.info("original", "test", { value: "original" }); + returned.message = "caller mutation"; + if (returned.data) returned.data.value = "caller mutation"; + + const retained = buf.getAll()[0]; + assertExists(retained); + assertEquals(retained.message, "original"); + assertEquals(retained.data?.value, "original"); + + retained.message = "query mutation"; + assertEquals(buf.getAll()[0]?.message, "original"); + }); + + it("detaches Date and URL values across returned and retained snapshots", () => { + const buf = new LogBuffer(); + const date = new Date("2025-01-02T03:04:05.000Z"); + const url = new URL("https://user:password@example.test/path?token=secret"); + + const returned = buf.info("structured", "test", { date, url }); + const returnedDate = returned.data?.date as Date; + const returnedUrl = returned.data?.url as URL; + returnedDate.setUTCFullYear(2030); + returnedUrl.pathname = "/mutated"; + + const retained = buf.getAll()[0]?.data; + assertEquals((retained?.date as Date).getUTCFullYear(), 2025); + assertEquals((retained?.url as URL).pathname, "/path"); + assertEquals((retained?.url as URL).href.includes("secret"), false); + assertEquals(date.getUTCFullYear(), 2025); + assertEquals(url.pathname, "/path"); + }); + it("redacts object args captured via interceptConsole (#1989)", () => { const buf = new LogBuffer(); const restore = interceptConsole(buf); @@ -43,12 +135,51 @@ describe("observability/log-buffer", () => { restore(); } - const message = buf.tail(1)[0].message; + const captured = buf.tail(1)[0]; + assertExists(captured); + const message = captured.message; assertEquals(message.includes("sk-secret"), false); assertEquals(message.includes("[REDACTED]"), true); assertEquals(message.includes("u-1"), true); }); + it("restores nested console interceptions by identity without resurrecting stale layers", () => { + const original = console.log; + const firstBuffer = new LogBuffer(); + const secondBuffer = new LogBuffer(); + const restoreFirst = interceptConsole(firstBuffer, "first"); + const restoreSecond = interceptConsole(secondBuffer, "second"); + + try { + restoreFirst(); + console.log("captured once"); + + assertEquals(firstBuffer.count, 0); + assertEquals(secondBuffer.count, 1); + + restoreSecond(); + restoreSecond(); + restoreFirst(); + assertStrictEquals(console.log, original); + } finally { + console.log = original; + } + }); + + it("does not overwrite a console method replaced by another owner", () => { + const original = console.log; + const restore = interceptConsole(new LogBuffer()); + const external = () => {}; + console.log = external; + + try { + restore(); + assertStrictEquals(console.log, external); + } finally { + console.log = original; + } + }); + it("should support all log levels", () => { const buf = new LogBuffer(); buf.debug("d"); @@ -117,6 +248,15 @@ describe("observability/log-buffer", () => { assertEquals(results.length, 1); }); + it("should query deterministically with stateful regex patterns", () => { + const buf = new LogBuffer(); + buf.info("error one"); + buf.info("error two"); + + assertEquals(buf.query({ pattern: /error/g }).length, 2); + assertEquals(buf.query({ pattern: /error/g }).length, 2); + }); + it("should query with limit", () => { const buf = new LogBuffer(); buf.info("1"); @@ -129,6 +269,14 @@ describe("observability/log-buffer", () => { assertEquals(first.message, "2"); }); + it("should return no entries for a zero limit or tail count", () => { + const buf = new LogBuffer(); + buf.info("1"); + + assertEquals(buf.query({ limit: 0 }), []); + assertEquals(buf.tail(0), []); + }); + it("should tail entries", () => { const buf = new LogBuffer(); buf.info("1"); diff --git a/src/observability/log-buffer.ts b/src/observability/log-buffer.ts index a247aa0c7f..19baa9cd53 100644 --- a/src/observability/log-buffer.ts +++ b/src/observability/log-buffer.ts @@ -1,5 +1,7 @@ -import { redactSensitive } from "#veryfront/utils/logger/redact.ts"; import { createSubscriberSet } from "#veryfront/utils/subscriber-set.ts"; +import { MAX_STRING_DISPLAY_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { MAX_OBSERVABILITY_NAME_LENGTH } from "./limits.ts"; +import { sanitizeStructuredTelemetryData, sanitizeTelemetryText } from "./telemetry-error.ts"; /** Public API contract for log level. */ export type LogLevel = "debug" | "info" | "warn" | "error"; @@ -25,6 +27,13 @@ export interface LogFilter { /** Public API contract for log subscriber. */ export type LogSubscriber = (entry: LogEntry) => void; +function snapshotEntry(entry: LogEntry): LogEntry { + return { + ...entry, + data: entry.data ? sanitizeStructuredTelemetryData(entry.data) : entry.data, + }; +} + /** Implement log buffer. */ export class LogBuffer { private entries: LogEntry[] = []; @@ -33,7 +42,11 @@ export class LogBuffer { private maxSize: number; constructor(options: { maxSize?: number } = {}) { - this.maxSize = options.maxSize ?? 1000; + const maxSize = options.maxSize ?? 1000; + if (!Number.isSafeInteger(maxSize) || maxSize < 0) { + throw new RangeError("LogBuffer maxSize must be a non-negative integer"); + } + this.maxSize = maxSize; } private generateId(): string { @@ -41,13 +54,25 @@ export class LogBuffer { } append(entry: Omit): LogEntry { + if ( + entry.level !== "debug" && entry.level !== "info" && + entry.level !== "warn" && entry.level !== "error" + ) { + throw new TypeError(`Invalid log level: ${String(entry.level)}`); + } + if (typeof entry.message !== "string" || typeof entry.source !== "string") { + throw new TypeError("Log message and source must be strings"); + } + const fullEntry: LogEntry = { - ...entry, + id: this.generateId(), + level: entry.level, + message: sanitizeTelemetryText(entry.message, MAX_STRING_DISPLAY_LENGTH), // Redact credential-like keys before the entry is buffered, surfaced to // subscribers, or written to disk by the file subscriber (#1989). - data: entry.data ? redactSensitive(entry.data) : entry.data, - id: this.generateId(), + data: entry.data ? sanitizeStructuredTelemetryData(entry.data) : entry.data, timestamp: Date.now(), + source: sanitizeTelemetryText(entry.source, MAX_OBSERVABILITY_NAME_LENGTH), }; this.entries.push(fullEntry); @@ -58,7 +83,7 @@ export class LogBuffer { this.subscribers.notify(fullEntry); - return fullEntry; + return snapshotEntry(fullEntry); } debug(message: string, source = "server", data?: Record): LogEntry { @@ -78,7 +103,7 @@ export class LogBuffer { } query(filter?: LogFilter): LogEntry[] { - if (!filter) return [...this.entries]; + if (!filter) return this.getAll(); let results = [...this.entries]; @@ -99,7 +124,15 @@ export class LogBuffer { const lower = pattern.toLowerCase(); results = results.filter((e) => e.message.toLowerCase().includes(lower)); } else { - results = results.filter((e) => pattern.test(e.message)); + const initialLastIndex = pattern.lastIndex; + try { + results = results.filter((e) => { + pattern.lastIndex = 0; + return pattern.test(e.message); + }); + } finally { + pattern.lastIndex = initialLastIndex; + } } } @@ -108,18 +141,20 @@ export class LogBuffer { } if (filter.limit != null) { - results = results.slice(-filter.limit); + const limit = Number.isFinite(filter.limit) ? Math.max(0, Math.floor(filter.limit)) : 0; + results = limit === 0 ? [] : results.slice(-limit); } - return results; + return results.map(snapshotEntry); } tail(count = 50): LogEntry[] { - return this.entries.slice(-count); + const normalizedCount = Number.isFinite(count) ? Math.max(0, Math.floor(count)) : 0; + return normalizedCount === 0 ? [] : this.entries.slice(-normalizedCount).map(snapshotEntry); } getAll(): LogEntry[] { - return [...this.entries]; + return this.entries.map(snapshotEntry); } clear(): void { @@ -141,7 +176,7 @@ export class LogBuffer { } subscribe(callback: LogSubscriber): () => void { - return this.subscribers.subscribe(callback); + return this.subscribers.subscribe((entry) => callback(snapshotEntry(entry))); } toJSON(): LogEntry[] { @@ -164,6 +199,34 @@ export class LogBuffer { let globalBuffer: LogBuffer | null = null; +type ConsoleMethod = "log" | "info" | "warn" | "error" | "debug"; +type ConsoleFunction = (...args: unknown[]) => void; + +interface ConsoleInterceptOwner { + readonly generation: number; + active: boolean; +} + +interface ConsoleWrapperMetadata { + readonly owner: ConsoleInterceptOwner; + readonly previous: ConsoleFunction; +} + +let consoleInterceptGeneration = 0; +const consoleWrapperMetadata = new WeakMap(); + +function resolveLiveConsoleFunction(candidate: ConsoleFunction): ConsoleFunction { + let current = candidate; + const seen = new Set(); + while (!seen.has(current)) { + seen.add(current); + const metadata = consoleWrapperMetadata.get(current); + if (!metadata || metadata.owner.active) return current; + current = metadata.previous; + } + return candidate; +} + /** Return log buffer. */ export function getLogBuffer(): LogBuffer { globalBuffer ??= new LogBuffer(); @@ -178,13 +241,17 @@ export function resetLogBuffer(): void { /** Capture console output in the log buffer. */ export function interceptConsole(buffer: LogBuffer, source = "console"): () => void { - const original = { + const previous: Record = { log: console.log, info: console.info, warn: console.warn, error: console.error, debug: console.debug, }; + const owner: ConsoleInterceptOwner = { + generation: ++consoleInterceptGeneration, + active: true, + }; function formatArgs(...args: unknown[]): string { return args @@ -194,32 +261,57 @@ export function interceptConsole(buffer: LogBuffer, source = "console"): () => v try { // Redact object args before they are folded into the message string, // where the per-entry data redaction can no longer reach them (#1989). - return JSON.stringify(redactSensitive(a)); + return JSON.stringify(sanitizeStructuredTelemetryData(a)); } catch (_) { /* expected: circular references or non-serializable values */ - return String(a); + try { + return String(a); + } catch (_) { + return "[Unserializable]"; + } } }) .join(" "); } function wrap( - method: keyof typeof original, + method: ConsoleMethod, log: (message: string, source: string) => LogEntry, - ): (...args: unknown[]) => void { - return (...args: unknown[]) => { - log(formatArgs(...args), source); - original[method].apply(console, args); + ): ConsoleFunction { + const wrapper: ConsoleFunction = (...args: unknown[]) => { + if (owner.active) { + try { + log(formatArgs(...args), source); + } catch (_) { + /* expected: interception must never block the underlying console */ + } + } + Reflect.apply(resolveLiveConsoleFunction(previous[method]), console, args); }; + consoleWrapperMetadata.set(wrapper, { owner, previous: previous[method] }); + return wrapper; } - console.log = wrap("log", buffer.info.bind(buffer)); - console.info = wrap("info", buffer.info.bind(buffer)); - console.warn = wrap("warn", buffer.warn.bind(buffer)); - console.error = wrap("error", buffer.error.bind(buffer)); - console.debug = wrap("debug", buffer.debug.bind(buffer)); + const wrappers: Record = { + log: wrap("log", buffer.info.bind(buffer)), + info: wrap("info", buffer.info.bind(buffer)), + warn: wrap("warn", buffer.warn.bind(buffer)), + error: wrap("error", buffer.error.bind(buffer)), + debug: wrap("debug", buffer.debug.bind(buffer)), + }; + + console.log = wrappers.log; + console.info = wrappers.info; + console.warn = wrappers.warn; + console.error = wrappers.error; + console.debug = wrappers.debug; return () => { - Object.assign(console, original); + if (!owner.active) return; + owner.active = false; + for (const method of Object.keys(wrappers) as ConsoleMethod[]) { + if (console[method] !== wrappers[method]) continue; + console[method] = resolveLiveConsoleFunction(previous[method]); + } }; } diff --git a/src/observability/metrics/config.test.ts b/src/observability/metrics/config.test.ts index d7b5af34e4..8b2b97fd12 100644 --- a/src/observability/metrics/config.test.ts +++ b/src/observability/metrics/config.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { DEFAULT_CONFIG, loadConfig } from "./config.ts"; @@ -85,9 +85,44 @@ describe("observability/metrics/config", () => { }; const result = loadConfig({}, adapterWithEnv(mockEnv)); - // Both are provided; the general endpoint is applied first, - // then metrics endpoint overrides if truthy - assertEquals(result.endpoint !== undefined, true); + assertEquals(result.endpoint, "http://metrics:4318"); + }); + + it("rejects malformed caller configuration instead of retaining invalid values", () => { + assertThrows( + () => loadConfig({ enabled: "yes" } as never, emptyEnvAdapter), + TypeError, + "enabled", + ); + assertThrows( + () => loadConfig({ exporter: "invalid" } as never, emptyEnvAdapter), + TypeError, + "exporter", + ); + assertThrows( + () => loadConfig({ prefix: " " }, emptyEnvAdapter), + TypeError, + "prefix", + ); + assertThrows( + () => loadConfig({ collectInterval: 0 }, emptyEnvAdapter), + RangeError, + "collectInterval", + ); + }); + + it("contains adapter environment failures without consulting another environment", () => { + const result = loadConfig( + { enabled: false, prefix: "configured" }, + adapterWithEnv({ + get() { + throw new Error("environment unavailable"); + }, + }), + ); + + assertEquals(result.enabled, false); + assertEquals(result.prefix, "configured"); }); }); }); diff --git a/src/observability/metrics/config.ts b/src/observability/metrics/config.ts index b8d523e814..f798ff15a3 100644 --- a/src/observability/metrics/config.ts +++ b/src/observability/metrics/config.ts @@ -2,28 +2,18 @@ import type { RuntimeAdapter } from "#veryfront/platform/adapters/base.ts"; import type { MetricsConfig } from "./types.ts"; import { memoryUsage as platformMemoryUsage } from "#veryfront/platform/compat/process.ts"; import { getHostTelemetryEnv } from "#veryfront/observability/tracing/telemetry-env.ts"; +import { MAX_TIMER_DELAY_MS } from "#veryfront/utils/timer.ts"; +import { MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, MAX_OBSERVABILITY_NAME_LENGTH } from "../limits.ts"; const DEFAULT_METRICS_COLLECT_INTERVAL_MS = 60000; -export const DEFAULT_CONFIG: MetricsConfig = { +export const DEFAULT_CONFIG: Readonly = Object.freeze({ enabled: false, exporter: "console", prefix: "veryfront", collectInterval: DEFAULT_METRICS_COLLECT_INTERVAL_MS, debug: false, -}; - -function getEnvVar(env: unknown, key: string): string | undefined { - if (env == null || typeof env !== "object") return undefined; - - const envObj = env as Record; - if (typeof envObj.get === "function") { - return envObj.get(key) as string | undefined; - } - - const value = envObj[key]; - return typeof value === "string" ? value : undefined; -} +}); function isValidExporter( exporter: unknown, @@ -35,39 +25,27 @@ export function loadConfig( config: Partial, adapter?: RuntimeAdapter, ): MetricsConfig { - const finalConfig: MetricsConfig = { ...DEFAULT_CONFIG, ...config }; - - function applyEnvConfig(opts: { - enabledFlag?: string; - veryfrontFlag?: string; - endpoint?: string; - metricsEndpoint?: string; - exporter?: unknown; - }): void { - finalConfig.enabled = opts.enabledFlag === "true" || opts.veryfrontFlag === "1" || - finalConfig.enabled; - - finalConfig.endpoint = opts.endpoint || opts.metricsEndpoint || finalConfig.endpoint; - - if (isValidExporter(opts.exporter)) { - finalConfig.exporter = opts.exporter; - } - } + const finalConfig = normalizeConfig(config); const env = adapter?.env; if (env) { - applyEnvConfig({ - enabledFlag: getEnvVar(env, "OTEL_METRICS_ENABLED"), - veryfrontFlag: getEnvVar(env, "VERYFRONT_OTEL"), - endpoint: getEnvVar(env, "OTEL_EXPORTER_OTLP_ENDPOINT"), - metricsEndpoint: getEnvVar(env, "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"), - exporter: getEnvVar(env, "OTEL_METRICS_EXPORTER"), - }); + try { + applyEnvConfig(finalConfig, { + enabledFlag: env.get("OTEL_METRICS_ENABLED"), + veryfrontFlag: env.get("VERYFRONT_OTEL"), + endpoint: env.get("OTEL_EXPORTER_OTLP_ENDPOINT"), + metricsEndpoint: env.get("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"), + exporter: env.get("OTEL_METRICS_EXPORTER"), + }); + } catch { + // Preserve caller configuration when the explicit adapter environment + // is unavailable; do not cross into the host environment. + } return finalConfig; } try { - applyEnvConfig({ + applyEnvConfig(finalConfig, { enabledFlag: getHostTelemetryEnv("OTEL_METRICS_ENABLED"), veryfrontFlag: getHostTelemetryEnv("VERYFRONT_OTEL"), endpoint: getHostTelemetryEnv("OTEL_EXPORTER_OTLP_ENDPOINT"), @@ -81,6 +59,105 @@ export function loadConfig( return finalConfig; } +function normalizeConfig(config: Partial): MetricsConfig { + if (config === null || typeof config !== "object" || Array.isArray(config)) { + throw new TypeError("Metrics config must be an object"); + } + const raw = config as Record; + const enabled = raw.enabled ?? DEFAULT_CONFIG.enabled; + if (typeof enabled !== "boolean") { + throw new TypeError("Metrics enabled must be a boolean"); + } + const exporter = raw.exporter ?? DEFAULT_CONFIG.exporter; + if (!isValidExporter(exporter)) { + throw new TypeError("Metrics exporter is invalid"); + } + const prefix = requireText( + raw.prefix ?? DEFAULT_CONFIG.prefix, + "prefix", + MAX_OBSERVABILITY_NAME_LENGTH, + ); + const endpoint = raw.endpoint === undefined ? undefined : requireText( + raw.endpoint, + "endpoint", + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + const collectInterval = raw.collectInterval ?? DEFAULT_CONFIG.collectInterval; + if ( + !Number.isSafeInteger(collectInterval) || (collectInterval as number) <= 0 || + (collectInterval as number) > MAX_TIMER_DELAY_MS + ) { + throw new RangeError( + `Metrics collectInterval must be an integer between 1 and ${MAX_TIMER_DELAY_MS}`, + ); + } + const debug = raw.debug ?? DEFAULT_CONFIG.debug; + if (typeof debug !== "boolean") { + throw new TypeError("Metrics debug must be a boolean"); + } + + return { + enabled, + exporter, + prefix, + collectInterval: collectInterval as number, + debug, + ...(endpoint ? { endpoint } : {}), + }; +} + +function applyEnvConfig( + config: MetricsConfig, + values: { + enabledFlag?: unknown; + veryfrontFlag?: unknown; + endpoint?: unknown; + metricsEndpoint?: unknown; + exporter?: unknown; + }, +): void { + config.enabled = isTrueEnvironmentValue(values.enabledFlag) || + values.veryfrontFlag === "1" || config.enabled; + + const signalEndpoint = normalizeEnvironmentText( + values.metricsEndpoint, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + const endpoint = normalizeEnvironmentText( + values.endpoint, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + config.endpoint = signalEndpoint || endpoint || config.endpoint; + + const exporter = typeof values.exporter === "string" + ? values.exporter.trim().toLowerCase() + : undefined; + if (isValidExporter(exporter)) config.exporter = exporter; +} + +function isTrueEnvironmentValue(value: unknown): boolean { + return typeof value === "string" && value.trim().toLowerCase() === "true"; +} + +function requireText(value: unknown, name: string, maxLength: number): string { + if (typeof value !== "string") { + throw new TypeError(`Metrics ${name} must be a string`); + } + const normalized = value.trim(); + if (!normalized || normalized.length > maxLength) { + throw new TypeError( + `Metrics ${name} must contain between 1 and ${maxLength} characters`, + ); + } + return normalized; +} + +function normalizeEnvironmentText(value: unknown, maxLength: number): string | undefined { + if (typeof value !== "string") return undefined; + const normalized = value.trim(); + return normalized && normalized.length <= maxLength ? normalized : undefined; +} + export function getMemoryUsage(): { rss: number; heapUsed: number; diff --git a/src/observability/metrics/manager.test.ts b/src/observability/metrics/manager.test.ts index 767185809c..ef44698e6f 100644 --- a/src/observability/metrics/manager.test.ts +++ b/src/observability/metrics/manager.test.ts @@ -1,8 +1,36 @@ import "#veryfront/schemas/_test-setup.ts"; import { assertEquals, assertExists } from "#veryfront/testing/assert.ts"; -import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { afterEach, beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { + _resetShimForTests, + installGlobalTelemetryAPI, + type Meter, + type MetricsAPI, + type ObservableResult, +} from "#veryfront/observability/tracing/api-shim.ts"; import { MetricsManager } from "./manager.ts"; +function createMetricsApi(label: string, calls: string[]): MetricsAPI { + const instrument = (name: string) => ({ + add: () => calls.push(`${label}:${name}`), + record: () => calls.push(`${label}:${name}`), + }); + const meter: Meter = { + createCounter: (name) => instrument(name), + createUpDownCounter: (name) => instrument(name), + createHistogram: (name) => instrument(name), + createObservableGauge: () => { + const callbacks = new Set<(result: ObservableResult) => void>(); + return { + addCallback: (callback: (result: ObservableResult) => void) => callbacks.add(callback), + removeCallback: (callback: (result: ObservableResult) => void) => + callbacks.delete(callback), + }; + }, + }; + return { getMeter: () => meter }; +} + describe("observability/metrics/manager", () => { let manager: MetricsManager; @@ -10,6 +38,11 @@ describe("observability/metrics/manager", () => { manager = new MetricsManager(); }); + afterEach(() => { + manager.shutdown(); + _resetShimForTests(); + }); + describe("initial state", () => { it("should not be enabled before initialization", () => { assertEquals(manager.isEnabled(), false); @@ -29,6 +62,27 @@ describe("observability/metrics/manager", () => { }); describe("initialize", () => { + it("follows metrics provider A to B to none without using stale instruments", async () => { + const calls: string[] = []; + const providerA = installGlobalTelemetryAPI({ metricsApi: createMetricsApi("A", calls) }); + await manager.initialize({ enabled: true, prefix: "test" }); + manager.getRecorder()?.recordHttpRequest(); + + const providerB = installGlobalTelemetryAPI({ metricsApi: createMetricsApi("B", calls) }); + manager.getRecorder()?.recordHttpRequest(); + assertEquals(providerA.dispose(), false); + assertEquals(providerB.dispose(), true); + manager.getRecorder()?.recordHttpRequest(); + + assertEquals(manager.isEnabled(), false); + assertEquals(calls, [ + "A:test.http.requests", + "A:test.http.requests.active", + "B:test.http.requests", + "B:test.http.requests.active", + ]); + }); + it("should initialize with disabled config", async () => { await manager.initialize({ enabled: false }); @@ -67,6 +121,25 @@ describe("observability/metrics/manager", () => { }); describe("shutdown", () => { + it("releases the meter and instruments and permits reinitialization", async () => { + const calls: string[] = []; + installGlobalTelemetryAPI({ metricsApi: createMetricsApi("A", calls) }); + await manager.initialize({ enabled: true, prefix: "test" }); + manager.shutdown(); + + assertEquals(manager.getState(), { + initialized: false, + cacheSize: 0, + activeRequests: 0, + }); + assertEquals(manager.isEnabled(), false); + + installGlobalTelemetryAPI({ metricsApi: createMetricsApi("B", calls) }); + await manager.initialize({ enabled: true, prefix: "test" }); + manager.getRecorder()?.recordHttpRequest(); + assertEquals(calls, ["B:test.http.requests", "B:test.http.requests.active"]); + }); + it("should not throw when not initialized", () => { manager.shutdown(); }); diff --git a/src/observability/metrics/manager.ts b/src/observability/metrics/manager.ts index c2c75cf1ed..70b0f0e67f 100644 --- a/src/observability/metrics/manager.ts +++ b/src/observability/metrics/manager.ts @@ -4,11 +4,16 @@ */ import type { Meter } from "#veryfront/observability/tracing/api-shim.ts"; -import { getGlobalMetricsAPI } from "#veryfront/observability/tracing/api-shim.ts"; +import { getGlobalTelemetryAPISnapshot } from "#veryfront/observability/tracing/api-shim.ts"; import { serverLogger } from "#veryfront/utils/logger/logger.ts"; import type { RuntimeAdapter } from "#veryfront/platform/adapters/base.ts"; import { loadConfig } from "./config.ts"; -import { initializeInstruments } from "../instruments/index.ts"; +import { + createEmptyInstruments, + disposeInstruments, + initializeInstruments, + isInitializedInstrumentSet, +} from "../instruments/instruments-factory.ts"; import { MetricsRecorder } from "./recorder.ts"; import type { MetricsConfig, MetricsInstruments, OpenTelemetryAPI, RuntimeState } from "./types.ts"; import { RUNTIME_VERSION } from "#veryfront/utils/version.ts"; @@ -23,62 +28,11 @@ export class MetricsManager { private initialized = false; private meter: Meter | null = null; private api: OpenTelemetryAPI | null = null; - private instruments: MetricsInstruments = this.createEmptyInstruments(); + private instruments: MetricsInstruments = createEmptyInstruments(); private runtimeState: RuntimeState = { cacheSize: 0, activeRequests: 0 }; private recorder = new MetricsRecorder(this.instruments, this.runtimeState); - - private createEmptyInstruments(): MetricsInstruments { - return { - httpRequestCounter: null, - httpRequestDuration: null, - httpActiveRequests: null, - cacheGetCounter: null, - cacheHitCounter: null, - cacheMissCounter: null, - cacheSetCounter: null, - cacheInvalidateCounter: null, - cacheSizeGauge: null, - renderDuration: null, - renderCounter: null, - renderErrorCounter: null, - rscRenderDuration: null, - rscStreamDuration: null, - rscManifestCounter: null, - rscPageCounter: null, - rscStreamCounter: null, - rscActionCounter: null, - rscErrorCounter: null, - buildDuration: null, - bundleSizeHistogram: null, - bundleCounter: null, - dataFetchDuration: null, - dataFetchCounter: null, - dataFetchErrorCounter: null, - corsRejectionCounter: null, - securityHeadersCounter: null, - memoryUsageGauge: null, - heapUsageGauge: null, - heapTotalGauge: null, - heapPercentGauge: null, - errorCounter: null, - streamLifecycleOutcomeCounter: null, - streamLifecycleDeadlineCounter: null, - streamLifecycleTelemetryCounter: null, - streamLifecycleRepairCounter: null, - streamLifecycleShadowDivergenceCounter: null, - streamLifecycleAttemptDuration: null, - streamLifecycleFirstProgressDuration: null, - streamLifecycleSemanticIdleDuration: null, - streamLifecycleToolInputDuration: null, - streamLifecycleToolExecutionDuration: null, - modelCallContextWriterOutcomeCounter: null, - modelCallContextBarrierOutcomeCounter: null, - modelCallContextLogicalByteLength: null, - modelCallContextPartCount: null, - modelCallContextAppendRequestCount: null, - modelCallContextRecorderBarrierDuration: null, - }; - } + private config: MetricsConfig | null = null; + private providerRevision = -1; async initialize(config: Partial = {}, adapter?: RuntimeAdapter): Promise { if (this.initialized) { @@ -89,45 +43,79 @@ export class MetricsManager { const finalConfig = loadConfig(config, adapter); this.initialized = true; + this.config = finalConfig; if (!finalConfig.enabled) { logger.debug("Metrics collection disabled"); return; } - try { - // The metrics API is injected by ext-observability-opentelemetry via setGlobalMetricsAPI(). - // When the extension is not active, metrics collection is disabled. - const metricsApi = getGlobalMetricsAPI(); - if (!metricsApi) { - logger.debug("No metrics API available — metrics collection disabled"); - return; - } - this.api = { metrics: metricsApi } as OpenTelemetryAPI; - this.meter = metricsApi.getMeter(finalConfig.prefix, RUNTIME_VERSION); - - this.instruments = initializeInstruments(this.meter, finalConfig, this.runtimeState); - this.recorder.instruments = this.instruments; - + this.refreshProvider(true); + if (this.meter) { logger.info("OpenTelemetry metrics initialized", { exporter: finalConfig.exporter, endpoint: finalConfig.endpoint, prefix: finalConfig.prefix, }); + } + } + + private clearProviderState(): void { + disposeInstruments(this.instruments); + this.instruments = createEmptyInstruments(); + this.recorder.instruments = this.instruments; + this.api = null; + this.meter = null; + } + + private refreshProvider(force = false): void { + if (!this.initialized || !this.config?.enabled) return; + + const snapshot = getGlobalTelemetryAPISnapshot(); + if (!force && snapshot.metricsApiRevision === this.providerRevision) return; + + this.clearProviderState(); + if (!snapshot.metricsApi) { + this.providerRevision = snapshot.metricsApiRevision; + return; + } + + try { + const api = { metrics: snapshot.metricsApi } as OpenTelemetryAPI; + const meter = snapshot.metricsApi.getMeter(this.config.prefix, RUNTIME_VERSION); + const instruments = initializeInstruments(meter, this.config, this.runtimeState); + if (!isInitializedInstrumentSet(instruments)) { + // The factory already reported the backend failure. Do not mark this + // revision complete: a later operation may retry after a transient + // provider initialization failure. + return; + } + this.api = api; + this.meter = meter; + this.instruments = instruments; + this.recorder.instruments = instruments; + this.providerRevision = snapshot.metricsApiRevision; } catch (error) { - logger.warn("Failed to initialize OpenTelemetry metrics", error); + try { + logger.warn("Failed to initialize OpenTelemetry metrics", error); + } catch (_) { + /* expected: metrics lifecycle remains fail-open */ + } } } isEnabled(): boolean { + this.refreshProvider(); return this.initialized && this.meter !== null; } getRecorder(): MetricsRecorder | null { + this.refreshProvider(); return this.recorder; } getState(): { initialized: boolean; cacheSize: number; activeRequests: number } { + this.refreshProvider(); return { initialized: this.initialized, cacheSize: this.runtimeState.cacheSize, @@ -143,6 +131,12 @@ export class MetricsManager { } catch (error) { logger.warn("Error during metrics shutdown", error); } + this.clearProviderState(); + this.initialized = false; + this.config = null; + this.providerRevision = -1; + this.runtimeState.cacheSize = 0; + this.runtimeState.activeRequests = 0; } } diff --git a/src/observability/metrics/numeric.test.ts b/src/observability/metrics/numeric.test.ts new file mode 100644 index 0000000000..256d12006f --- /dev/null +++ b/src/observability/metrics/numeric.test.ts @@ -0,0 +1,29 @@ +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { + nonNegativeFiniteMeasure, + nonNegativeSafeInteger, + saturatingAdd, + saturatingAddMeasure, +} from "./numeric.ts"; + +describe("observability/metrics/numeric", () => { + it("normalizes all metric values to finite non-negative safe bounds", () => { + assertEquals(nonNegativeSafeInteger(Number.MAX_VALUE), Number.MAX_SAFE_INTEGER); + assertEquals(nonNegativeSafeInteger(Number.NaN), 0); + assertEquals(nonNegativeSafeInteger(-1), 0); + assertEquals(nonNegativeFiniteMeasure(Number.MAX_VALUE), Number.MAX_SAFE_INTEGER); + assertEquals(nonNegativeFiniteMeasure(Number.POSITIVE_INFINITY), 0); + assertEquals(nonNegativeFiniteMeasure(-0.5), 0); + }); + + it("saturates integer and fractional accumulation without overflow", () => { + assertEquals(saturatingAdd(Number.MAX_SAFE_INTEGER - 1, 10), Number.MAX_SAFE_INTEGER); + assertEquals(saturatingAdd(Number.NaN, 2.9), 2); + assertEquals( + saturatingAddMeasure(Number.MAX_SAFE_INTEGER - 0.5, 1), + Number.MAX_SAFE_INTEGER, + ); + assertEquals(saturatingAddMeasure(Number.NaN, 2.5), 2.5); + }); +}); diff --git a/src/observability/metrics/numeric.ts b/src/observability/metrics/numeric.ts new file mode 100644 index 0000000000..b80a3da58e --- /dev/null +++ b/src/observability/metrics/numeric.ts @@ -0,0 +1,34 @@ +/** Largest exact integer accepted by observability counters and durations. */ +const MAX_SAFE_METRIC_VALUE = Number.MAX_SAFE_INTEGER; + +/** Normalize a counter input to a finite, non-negative safe integer. */ +export function nonNegativeSafeInteger(value: number): number { + if (!Number.isFinite(value) || value <= 0) return 0; + return Math.min(MAX_SAFE_METRIC_VALUE, Math.floor(value)); +} + +/** Normalize a measurement to a finite, non-negative, exactly bounded value. */ +export function nonNegativeFiniteMeasure(value: number): number { + if (!Number.isFinite(value) || value <= 0) return 0; + return Math.min(MAX_SAFE_METRIC_VALUE, value); +} + +/** Add counter values without crossing JavaScript's exact-integer boundary. */ +export function saturatingAdd(current: number, increment = 1): number { + const normalizedCurrent = nonNegativeSafeInteger(current); + const normalizedIncrement = nonNegativeSafeInteger(increment); + if (normalizedIncrement >= MAX_SAFE_METRIC_VALUE - normalizedCurrent) { + return MAX_SAFE_METRIC_VALUE; + } + return normalizedCurrent + normalizedIncrement; +} + +/** Add measurements without producing Infinity or an unsafe finite result. */ +export function saturatingAddMeasure(current: number, increment: number): number { + const normalizedCurrent = nonNegativeFiniteMeasure(current); + const normalizedIncrement = nonNegativeFiniteMeasure(increment); + if (normalizedIncrement >= MAX_SAFE_METRIC_VALUE - normalizedCurrent) { + return MAX_SAFE_METRIC_VALUE; + } + return normalizedCurrent + normalizedIncrement; +} diff --git a/src/observability/metrics/recorder.test.ts b/src/observability/metrics/recorder.test.ts index 9a01010459..6ffff8c627 100644 --- a/src/observability/metrics/recorder.test.ts +++ b/src/observability/metrics/recorder.test.ts @@ -121,6 +121,7 @@ function createMockInstruments(): MetricsInstruments & { dataFetchErrorCounter: dataFetchErrorCounter as never, corsRejectionCounter: corsRejectionCounter as never, securityHeadersCounter: securityHeadersCounter as never, + errorCounter: null, memoryUsageGauge: null, heapUsageGauge: null, heapTotalGauge: null, @@ -135,6 +136,12 @@ function createMockInstruments(): MetricsInstruments & { streamLifecycleSemanticIdleDuration: null, streamLifecycleToolInputDuration: null, streamLifecycleToolExecutionDuration: null, + modelCallContextWriterOutcomeCounter: null, + modelCallContextBarrierOutcomeCounter: null, + modelCallContextLogicalByteLength: null, + modelCallContextPartCount: null, + modelCallContextAppendRequestCount: null, + modelCallContextRecorderBarrierDuration: null, _httpRequestCounter: httpRequestCounter, _httpRequestDuration: httpRequestDuration, @@ -185,6 +192,51 @@ describe("observability/metrics/recorder", () => { }); describe("recordHttpRequest", () => { + it("repairs poisoned state and saturates active request counters", () => { + runtimeState.activeRequests = Number.NaN; + recorder.recordHttpRequest(); + assertEquals(runtimeState.activeRequests, 1); + + runtimeState.activeRequests = Number.MAX_SAFE_INTEGER; + recorder.recordHttpRequest(); + assertEquals(runtimeState.activeRequests, Number.MAX_SAFE_INTEGER); + }); + + it("updates state when attribute enumeration and getters are hostile", () => { + const getterAttributes: Record = {}; + Object.defineProperty(getterAttributes, "route", { + enumerable: true, + get() { + throw new Error("hostile attribute getter"); + }, + }); + + recorder.recordHttpRequest(getterAttributes); + recorder.recordHttpRequest( + new Proxy({}, { + ownKeys() { + throw new Error("hostile attribute enumeration"); + }, + }), + ); + + assertEquals(runtimeState.activeRequests, 2); + assertEquals(instruments._httpRequestCounter._value, 2); + }); + + it("isolates instrument failures and still updates runtime state", () => { + instruments.httpRequestCounter = { + add() { + throw new Error("telemetry counter failed"); + }, + } as never; + + recorder.recordHttpRequest(); + + assertEquals(runtimeState.activeRequests, 1); + assertEquals(instruments._httpActiveRequests._value, 1); + }); + it("should increment http request counter and active requests", () => { recorder.recordHttpRequest(); assertEquals(instruments._httpRequestCounter._value, 1); @@ -198,6 +250,18 @@ describe("observability/metrics/recorder", () => { assertEquals(instruments._httpRequestCounter._lastAttributes, attrs); }); + it("redacts sensitive and URL credential attributes", () => { + recorder.recordHttpRequest({ + apiKey: "secret", + endpoint: "https://example.test/path?token=secret", + }); + + assertEquals(instruments._httpRequestCounter._lastAttributes, { + apiKey: "[REDACTED]", + endpoint: "https://example.test/path?token=[REDACTED]", + }); + }); + it("should accumulate multiple requests", () => { recorder.recordHttpRequest(); recorder.recordHttpRequest(); @@ -208,6 +272,13 @@ describe("observability/metrics/recorder", () => { }); describe("recordHttpRequestComplete", () => { + it("does not make active request state or gauges negative", () => { + recorder.recordHttpRequestComplete(100); + + assertEquals(runtimeState.activeRequests, 0); + assertEquals(instruments._httpActiveRequests._value, 0); + }); + it("should record duration and decrement active requests", () => { runtimeState.activeRequests = 1; recorder.recordHttpRequestComplete(150); @@ -240,6 +311,16 @@ describe("observability/metrics/recorder", () => { }); describe("recordCacheSet", () => { + it("repairs poisoned state and saturates cache size", () => { + runtimeState.cacheSize = Number.NaN; + recorder.recordCacheSet(); + assertEquals(runtimeState.cacheSize, 1); + + runtimeState.cacheSize = Number.MAX_SAFE_INTEGER; + recorder.recordCacheSet(); + assertEquals(runtimeState.cacheSize, Number.MAX_SAFE_INTEGER); + }); + it("should increment set counter and cache size", () => { recorder.recordCacheSet(); assertEquals(instruments._cacheSetCounter._value, 1); @@ -255,6 +336,15 @@ describe("observability/metrics/recorder", () => { }); describe("recordCacheInvalidate", () => { + it("ignores negative invalidation counts", () => { + runtimeState.cacheSize = 4; + + recorder.recordCacheInvalidate(-2); + + assertEquals(runtimeState.cacheSize, 4); + assertEquals(instruments._cacheInvalidateCounter._value, 0); + }); + it("should increment invalidation counter and reduce cache size", () => { runtimeState.cacheSize = 10; recorder.recordCacheInvalidate(3); @@ -270,6 +360,14 @@ describe("observability/metrics/recorder", () => { }); describe("setCacheSize", () => { + it("normalizes negative and non-finite cache sizes", () => { + recorder.setCacheSize(-2); + assertEquals(runtimeState.cacheSize, 0); + + recorder.setCacheSize(Number.POSITIVE_INFINITY); + assertEquals(runtimeState.cacheSize, 0); + }); + it("should set cache size directly", () => { recorder.setCacheSize(42); assertEquals(runtimeState.cacheSize, 42); @@ -283,6 +381,14 @@ describe("observability/metrics/recorder", () => { }); describe("recordRender", () => { + it("never sends non-finite or unsafe durations to a metrics backend", () => { + recorder.recordRender(Number.POSITIVE_INFINITY); + assertEquals(instruments._renderDuration._value, 0); + + recorder.recordRender(Number.MAX_VALUE); + assertEquals(instruments._renderDuration._value, Number.MAX_SAFE_INTEGER); + }); + it("should record render duration and increment counter", () => { recorder.recordRender(200); assertEquals(instruments._renderDuration._value, 200); @@ -426,6 +532,7 @@ describe("observability/metrics/recorder", () => { dataFetchErrorCounter: null, corsRejectionCounter: null, securityHeadersCounter: null, + errorCounter: null, memoryUsageGauge: null, heapUsageGauge: null, heapTotalGauge: null, @@ -440,6 +547,12 @@ describe("observability/metrics/recorder", () => { streamLifecycleSemanticIdleDuration: null, streamLifecycleToolInputDuration: null, streamLifecycleToolExecutionDuration: null, + modelCallContextWriterOutcomeCounter: null, + modelCallContextBarrierOutcomeCounter: null, + modelCallContextLogicalByteLength: null, + modelCallContextPartCount: null, + modelCallContextAppendRequestCount: null, + modelCallContextRecorderBarrierDuration: null, }; const nullRecorder = new MetricsRecorder(nullInstruments, runtimeState); diff --git a/src/observability/metrics/recorder.ts b/src/observability/metrics/recorder.ts index c7f39ccfab..9b1fd040bf 100644 --- a/src/observability/metrics/recorder.ts +++ b/src/observability/metrics/recorder.ts @@ -4,6 +4,16 @@ import type { ModelCallContextWriterOutcome, RuntimeState, } from "./types.ts"; +import { sanitizeTelemetryAttributes } from "../telemetry-error.ts"; +import { nonNegativeFiniteMeasure, nonNegativeSafeInteger, saturatingAdd } from "./numeric.ts"; + +function safelyRecord(operation: () => void): void { + try { + operation(); + } catch (_) { + /* expected: a telemetry backend failure must not affect application work */ + } +} export class MetricsRecorder { constructor( @@ -21,125 +31,179 @@ export class MetricsRecorder { } recordHttpRequest(attributes?: Record): void { - this.instruments.httpRequestCounter?.add(1, attributes); - this.instruments.httpActiveRequests?.add(1, attributes); - this.runtimeState.activeRequests++; + attributes = sanitizeTelemetryAttributes(attributes); + this.runtimeState.activeRequests = saturatingAdd(this.runtimeState.activeRequests); + safelyRecord(() => this.instruments.httpRequestCounter?.add(1, attributes)); + safelyRecord(() => this.instruments.httpActiveRequests?.add(1, attributes)); } recordHttpRequestComplete( durationMs: number, attributes?: Record, ): void { - this.instruments.httpRequestDuration?.record(durationMs, attributes); - this.instruments.httpActiveRequests?.add(-1, attributes); - this.runtimeState.activeRequests--; + attributes = sanitizeTelemetryAttributes(attributes); + const activeRequests = nonNegativeSafeInteger(this.runtimeState.activeRequests); + const hadActiveRequest = activeRequests > 0; + this.runtimeState.activeRequests = hadActiveRequest ? activeRequests - 1 : 0; + safelyRecord(() => + this.instruments.httpRequestDuration?.record( + nonNegativeFiniteMeasure(durationMs), + attributes, + ) + ); + if (hadActiveRequest) { + safelyRecord(() => this.instruments.httpActiveRequests?.add(-1, attributes)); + } } recordCacheGet(hit: boolean, attributes?: Record): void { - this.instruments.cacheGetCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.cacheGetCounter?.add(1, attributes)); if (hit) { - this.instruments.cacheHitCounter?.add(1, attributes); + safelyRecord(() => this.instruments.cacheHitCounter?.add(1, attributes)); } else { - this.instruments.cacheMissCounter?.add(1, attributes); + safelyRecord(() => this.instruments.cacheMissCounter?.add(1, attributes)); } } recordCacheSet(attributes?: Record): void { - this.instruments.cacheSetCounter?.add(1, attributes); - this.runtimeState.cacheSize++; + attributes = sanitizeTelemetryAttributes(attributes); + this.runtimeState.cacheSize = saturatingAdd(this.runtimeState.cacheSize); + safelyRecord(() => this.instruments.cacheSetCounter?.add(1, attributes)); } recordCacheInvalidate( count: number, attributes?: Record, ): void { - this.instruments.cacheInvalidateCounter?.add(count, attributes); - this.runtimeState.cacheSize = Math.max(0, this.runtimeState.cacheSize - count); + attributes = sanitizeTelemetryAttributes(attributes); + const normalizedCount = nonNegativeSafeInteger(count); + if (normalizedCount === 0) return; + const cacheSize = nonNegativeSafeInteger(this.runtimeState.cacheSize); + this.runtimeState.cacheSize = Math.max(0, cacheSize - normalizedCount); + safelyRecord(() => this.instruments.cacheInvalidateCounter?.add(normalizedCount, attributes)); } setCacheSize(size: number): void { - this.runtimeState.cacheSize = size; + this.runtimeState.cacheSize = nonNegativeSafeInteger(size); } recordRender(durationMs: number, attributes?: Record): void { - this.instruments.renderDuration?.record(durationMs, attributes); - this.instruments.renderCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.renderDuration?.record(nonNegativeFiniteMeasure(durationMs), attributes) + ); + safelyRecord(() => this.instruments.renderCounter?.add(1, attributes)); } recordRenderError(attributes?: Record): void { - this.instruments.renderErrorCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.renderErrorCounter?.add(1, attributes)); } recordRSCRender( durationMs: number, attributes?: Record, ): void { - this.instruments.rscRenderDuration?.record(durationMs, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.rscRenderDuration?.record( + nonNegativeFiniteMeasure(durationMs), + attributes, + ) + ); } recordRSCStream( durationMs: number, attributes?: Record, ): void { - this.instruments.rscStreamDuration?.record(durationMs, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.rscStreamDuration?.record( + nonNegativeFiniteMeasure(durationMs), + attributes, + ) + ); } recordRSCRequest( type: "manifest" | "page" | "stream" | "action", attributes?: Record, ): void { + attributes = sanitizeTelemetryAttributes(attributes); switch (type) { case "manifest": - this.instruments.rscManifestCounter?.add(1, attributes); + safelyRecord(() => this.instruments.rscManifestCounter?.add(1, attributes)); return; case "page": - this.instruments.rscPageCounter?.add(1, attributes); + safelyRecord(() => this.instruments.rscPageCounter?.add(1, attributes)); return; case "stream": - this.instruments.rscStreamCounter?.add(1, attributes); + safelyRecord(() => this.instruments.rscStreamCounter?.add(1, attributes)); return; case "action": - this.instruments.rscActionCounter?.add(1, attributes); + safelyRecord(() => this.instruments.rscActionCounter?.add(1, attributes)); return; } } recordRSCError(attributes?: Record): void { - this.instruments.rscErrorCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.rscErrorCounter?.add(1, attributes)); } recordBuild(durationMs: number, attributes?: Record): void { - this.instruments.buildDuration?.record(durationMs, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.buildDuration?.record(nonNegativeFiniteMeasure(durationMs), attributes) + ); } recordBundle(sizeKb: number, attributes?: Record): void { - this.instruments.bundleSizeHistogram?.record(sizeKb, attributes); - this.instruments.bundleCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.bundleSizeHistogram?.record( + nonNegativeFiniteMeasure(sizeKb), + attributes, + ) + ); + safelyRecord(() => this.instruments.bundleCounter?.add(1, attributes)); } recordDataFetch( durationMs: number, attributes?: Record, ): void { - this.instruments.dataFetchDuration?.record(durationMs, attributes); - this.instruments.dataFetchCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => + this.instruments.dataFetchDuration?.record( + nonNegativeFiniteMeasure(durationMs), + attributes, + ) + ); + safelyRecord(() => this.instruments.dataFetchCounter?.add(1, attributes)); } recordDataFetchError(attributes?: Record): void { - this.instruments.dataFetchErrorCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.dataFetchErrorCounter?.add(1, attributes)); } recordCorsRejection(attributes?: Record): void { - this.instruments.corsRejectionCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.corsRejectionCounter?.add(1, attributes)); } recordSecurityHeaders(attributes?: Record): void { - this.instruments.securityHeadersCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.securityHeadersCounter?.add(1, attributes)); } recordError(attributes?: Record): void { - this.instruments.errorCounter?.add(1, attributes); + attributes = sanitizeTelemetryAttributes(attributes); + safelyRecord(() => this.instruments.errorCounter?.add(1, attributes)); } recordStreamLifecycleOutcome(attributes: Record): void { diff --git a/src/observability/request-profiler.test.ts b/src/observability/request-profiler.test.ts index 4a6958cffe..42ac43f3a2 100644 --- a/src/observability/request-profiler.test.ts +++ b/src/observability/request-profiler.test.ts @@ -1,5 +1,10 @@ import { assert, assertEquals, assertExists } from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + MAX_OBSERVABILITY_NAME_LENGTH, + MAX_REQUEST_PROFILE_PHASES, +} from "./limits.ts"; import { buildServerTimingHeader, finalizeRequestProfiling, @@ -68,6 +73,109 @@ describe("request profiler", () => { assertEquals(result.phases["render.cache_hit"], 0); }); + it("returns detached records and normalizes explicit phase durations", async () => { + const returned = await runWithRequestProfiling( + { + category: "html", + method: "GET", + pathname: "/profiled", + }, + async () => { + markRequestProfilePhase("invalid", -5); + markRequestProfilePhase("invalid", Number.POSITIVE_INFINITY); + return finalizeRequestProfiling(200); + }, + ); + assertExists(returned); + returned.pathname = "/mutated"; + returned.phases.invalid = 99; + + const firstSnapshot = snapshotRequestProfiles(); + assertEquals(firstSnapshot.records[0]?.pathname, "/profiled"); + assertEquals(firstSnapshot.records[0]?.phases.invalid, 0); + + const firstRecord = firstSnapshot.records[0]; + assertExists(firstRecord); + firstRecord.pathname = "/snapshot-mutated"; + firstRecord.phases.invalid = 100; + + const secondSnapshot = snapshotRequestProfiles(); + assertEquals(secondSnapshot.records[0]?.pathname, "/profiled"); + assertEquals(secondSnapshot.records[0]?.phases.invalid, 0); + }); + + it("saturates accumulated phase durations at a finite safe bound", async () => { + const record = await runWithRequestProfiling( + { + category: "html", + method: "GET", + pathname: "/profiled", + }, + async () => { + markRequestProfilePhase("overflow", Number.MAX_SAFE_INTEGER); + markRequestProfilePhase("overflow", Number.MAX_SAFE_INTEGER); + return finalizeRequestProfiling(200); + }, + ); + + assertExists(record); + assertEquals(record.phases.overflow, Number.MAX_SAFE_INTEGER); + assert(Number.isFinite(record.totalMs)); + assert(record.totalMs <= Number.MAX_SAFE_INTEGER); + }); + + it("bounds phase cardinality and finalizes each request session once", async () => { + const results = await runWithRequestProfiling( + { + category: "html", + method: "GET", + pathname: "/profiled", + }, + async () => { + for (let index = 0; index < MAX_REQUEST_PROFILE_PHASES + 20; index++) { + markRequestProfilePhase(`phase-${index}`, 1); + } + return [ + finalizeRequestProfiling(200), + finalizeRequestProfiling(500), + ] as const; + }, + ); + + assertExists(results[0]); + assertEquals(Object.keys(results[0].phases).length, MAX_REQUEST_PROFILE_PHASES); + assertEquals(results[1], null); + assertEquals(snapshotRequestProfiles().records.length, 1); + }); + + it("bounds retained request identity and fails open for malformed options", async () => { + const record = await runWithRequestProfiling( + { + category: "c".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + method: "m".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + pathname: `/${"p".repeat(MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH + 100)}`, + projectSlug: "s".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + requestMode: "r".repeat(MAX_OBSERVABILITY_NAME_LENGTH + 100), + }, + async () => finalizeRequestProfiling(200), + ); + + assertExists(record); + assertEquals(record.category.length, MAX_OBSERVABILITY_NAME_LENGTH); + assertEquals(record.method.length, MAX_OBSERVABILITY_NAME_LENGTH); + assertEquals(record.pathname.length, MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH); + assertEquals(record.projectSlug?.length, MAX_OBSERVABILITY_NAME_LENGTH); + assertEquals(record.requestMode?.length, MAX_OBSERVABILITY_NAME_LENGTH); + + let calls = 0; + const result = await runWithRequestProfiling( + { category: "html", method: "GET", pathname: 1 } as never, + async () => ++calls, + ); + assertEquals(result, 1); + assertEquals(calls, 1); + }); + it("profiles page-data requests when Server-Timing diagnostics are enabled", () => { Deno.env.set("VERYFRONT_ENABLE_SERVER_TIMING", "1"); diff --git a/src/observability/request-profiler.ts b/src/observability/request-profiler.ts index e258f60075..88efc60bc5 100644 --- a/src/observability/request-profiler.ts +++ b/src/observability/request-profiler.ts @@ -1,5 +1,16 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { getEnv } from "#veryfront/platform/compat/process.ts"; +import { + nonNegativeFiniteMeasure, + saturatingAdd, + saturatingAddMeasure, +} from "./metrics/numeric.ts"; +import { + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + MAX_OBSERVABILITY_NAME_LENGTH, + MAX_REQUEST_PROFILE_PHASES, +} from "./limits.ts"; +import { sanitizeTelemetryText } from "./telemetry-error.ts"; export interface RequestProfileRecord { sequence: number; @@ -28,6 +39,7 @@ interface RequestProfileSession { requestMode?: string; startedAt: number; phases: Map; + finalized: boolean; } const storage = new AsyncLocalStorage(); @@ -37,9 +49,48 @@ let sequence = 0; /** Round to 2 decimal places (Server-Timing millisecond precision). */ export function roundMs(value: number): number { + if (value >= Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER; return Math.round(value * 100) / 100; } +function normalizeDuration(value: number): number { + return roundMs(nonNegativeFiniteMeasure(value)); +} + +function addPhaseDuration(session: RequestProfileSession, name: string, durationMs: number): void { + if (session.finalized || typeof name !== "string") return; + const normalizedName = sanitizeTelemetryText(name, MAX_OBSERVABILITY_NAME_LENGTH); + if ( + !session.phases.has(normalizedName) && + session.phases.size >= MAX_REQUEST_PROFILE_PHASES + ) { + return; + } + session.phases.set( + normalizedName, + normalizeDuration( + saturatingAddMeasure( + session.phases.get(normalizedName) ?? 0, + normalizeDuration(durationMs), + ), + ), + ); +} + +function snapshotRecord(record: RequestProfileRecord): RequestProfileRecord { + return { + ...record, + phases: { ...record.phases }, + }; +} + +function normalizeProfileText( + value: unknown, + maxLength: number, +): string | null { + return typeof value === "string" ? sanitizeTelemetryText(value, maxLength) : null; +} + function shouldEnableProfiling(): boolean { return getEnv("VERYFRONT_ENABLE_PERF_PROFILING") === "1"; } @@ -86,14 +137,52 @@ export async function runWithRequestProfiling( }, fn: () => Promise, ): Promise { - if (!isRequestProfilingEnabled(options.pathname)) { + let category: string | null; + let method: string | null; + let pathname: string | null; + let projectSlug: string | undefined; + let requestMode: string | undefined; + try { + category = normalizeProfileText( + options.category, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + method = normalizeProfileText( + options.method, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + pathname = normalizeProfileText( + options.pathname, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + projectSlug = options.projectSlug === undefined ? undefined : normalizeProfileText( + options.projectSlug, + MAX_OBSERVABILITY_NAME_LENGTH, + ) ?? undefined; + requestMode = options.requestMode === undefined ? undefined : normalizeProfileText( + options.requestMode, + MAX_OBSERVABILITY_NAME_LENGTH, + ) ?? undefined; + } catch { + return await fn(); + } + + if ( + category === null || method === null || pathname === null || + !isRequestProfilingEnabled(pathname) + ) { return await fn(); } const session: RequestProfileSession = { - ...options, + category, + method, + pathname, + projectSlug, + requestMode, startedAt: performance.now(), phases: new Map(), + finalized: false, }; return await storage.run(session, fn); @@ -108,7 +197,7 @@ export async function profilePhase(name: string, fn: () => Promise): Promi return await fn(); } finally { const duration = performance.now() - startedAt; - session.phases.set(name, roundMs((session.phases.get(name) ?? 0) + duration)); + addPhaseDuration(session, name, duration); } } @@ -116,7 +205,7 @@ export function markRequestProfilePhase(name: string, durationMs = 0): void { const session = storage.getStore(); if (!session) return; - session.phases.set(name, roundMs((session.phases.get(name) ?? 0) + durationMs)); + addPhaseDuration(session, name, durationMs); } export function profileSyncPhase(name: string, fn: () => T): T { @@ -128,24 +217,36 @@ export function profileSyncPhase(name: string, fn: () => T): T { return fn(); } finally { const duration = performance.now() - startedAt; - session.phases.set(name, roundMs((session.phases.get(name) ?? 0) + duration)); + addPhaseDuration(session, name, duration); } } export function updateRequestProfileContext(update: RequestProfileContextUpdate): void { const session = storage.getStore(); - if (!session) return; + if (!session || session.finalized) return; - if (update.projectSlug !== undefined) session.projectSlug = update.projectSlug; - if (update.requestMode !== undefined) session.requestMode = update.requestMode; + if (typeof update.projectSlug === "string") { + session.projectSlug = sanitizeTelemetryText( + update.projectSlug, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + } + if (typeof update.requestMode === "string") { + session.requestMode = sanitizeTelemetryText( + update.requestMode, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + } } export function finalizeRequestProfiling(status?: number): RequestProfileRecord | null { const session = storage.getStore(); - if (!session) return null; + if (!session || session.finalized) return null; + session.finalized = true; + sequence = saturatingAdd(sequence); const record: RequestProfileRecord = { - sequence: ++sequence, + sequence, category: session.category, method: session.method, pathname: session.pathname, @@ -154,14 +255,14 @@ export function finalizeRequestProfiling(status?: number): RequestProfileRecord status, startedAt: new Date(Date.now() - (performance.now() - session.startedAt)).toISOString(), completedAt: new Date().toISOString(), - totalMs: roundMs(performance.now() - session.startedAt), + totalMs: normalizeDuration(performance.now() - session.startedAt), phases: Object.fromEntries(session.phases.entries()), }; records.push(record); while (records.length > MAX_RECORDS) records.shift(); - return record; + return snapshotRecord(record); } function sanitizeMetricName(name: string): string { @@ -169,7 +270,7 @@ function sanitizeMetricName(name: string): string { } function formatDuration(value: number): string { - return Math.max(0, roundMs(value)).toFixed(2); + return normalizeDuration(value).toFixed(2); } /** Build a Server-Timing header value from a total plus named phase durations. */ @@ -224,7 +325,7 @@ export function snapshotRequestProfiles(): { enabled: shouldEnableProfiling() || shouldEnableServerTiming() || shouldEnableSlowRequestProfiling(), last_sequence: sequence, - records: [...records], + records: records.map(snapshotRecord), }; } diff --git a/src/observability/simple-metrics/metrics-recorder.test.ts b/src/observability/simple-metrics/metrics-recorder.test.ts index 9bbd942408..6f709b5e3c 100644 --- a/src/observability/simple-metrics/metrics-recorder.test.ts +++ b/src/observability/simple-metrics/metrics-recorder.test.ts @@ -8,6 +8,7 @@ import { recordCacheGet, recordCacheInvalidate, recordCacheSet, + recordContentNetworkFetch, recordCorsRejection, recordHttp, recordModuleServe, @@ -27,9 +28,41 @@ describe("observability/simple-metrics/metrics-recorder", () => { assertEquals(state.jitHttpBlocked, 2); assertEquals(state.jitHttpFetchMsTotal, 150); }); + + it("ignores negative and non-finite values", () => { + resetMetrics(); + recordHttp(-1, Number.NaN, Number.POSITIVE_INFINITY); + + assertEquals(state.jitHttpResolved, 0); + assertEquals(state.jitHttpBlocked, 0); + assertEquals(state.jitHttpFetchMsTotal, 0); + }); + + it("clamps inputs and accumulated totals to safe integers", () => { + resetMetrics(); + state.jitHttpResolved = Number.MAX_SAFE_INTEGER - 1; + + recordHttp(Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE); + + assertEquals(state.jitHttpResolved, Number.MAX_SAFE_INTEGER); + assertEquals(state.jitHttpBlocked, Number.MAX_SAFE_INTEGER); + assertEquals(state.jitHttpFetchMsTotal, Number.MAX_SAFE_INTEGER); + assertEquals(Number.isSafeInteger(state.jitHttpResolved), true); + }); }); describe("recordCacheGet", () => { + it("saturates counters instead of overflowing safe integer precision", () => { + resetMetrics(); + state.cacheGets = Number.MAX_SAFE_INTEGER; + state.cacheHits = Number.MAX_SAFE_INTEGER; + + recordCacheGet(true); + + assertEquals(state.cacheGets, Number.MAX_SAFE_INTEGER); + assertEquals(state.cacheHits, Number.MAX_SAFE_INTEGER); + }); + it("should increment gets and hits on hit", () => { resetMetrics(); recordCacheGet(true); @@ -77,6 +110,15 @@ describe("observability/simple-metrics/metrics-recorder", () => { assertEquals(state.moduleTransformDurationMsTotal, 12); }); + it("keeps module transform totals finite", () => { + resetMetrics(); + recordModuleTransform(Number.NaN); + recordModuleTransform(Number.POSITIVE_INFINITY); + + assertEquals(state.moduleTransformTotal, 2); + assertEquals(state.moduleTransformDurationMsTotal, 0); + }); + it("records route manifest LRU hits and misses", () => { resetMetrics(); recordRouteManifestLookup(true); @@ -96,15 +138,40 @@ describe("observability/simple-metrics/metrics-recorder", () => { recordCacheInvalidate(3); assertEquals(state.cacheInvalidations, 8); }); + + it("ignores negative and non-finite invalidation counts", () => { + resetMetrics(); + recordCacheInvalidate(-1); + recordCacheInvalidate(Number.NaN); + + assertEquals(state.cacheInvalidations, 0); + }); }); describe("recordSSR", () => { + it("saturates histogram bucket counts", () => { + resetMetrics(); + state._ssrCounts[0] = Number.MAX_SAFE_INTEGER; + + recordSSR(0); + + assertEquals(state._ssrCounts[0], Number.MAX_SAFE_INTEGER); + }); + it("should record duration in histogram bucket", () => { resetMetrics(); recordSSR(50); const bucket50 = state._ssrCounts.find((c) => c > 0); assertEquals(bucket50 !== undefined, true); }); + + it("records non-finite durations as zero instead of corrupting state", () => { + resetMetrics(); + recordSSR(Number.NaN); + + assertEquals(state._ssrCounts[0], 1); + assertEquals(state._ssrCounts.at(-1), 0); + }); }); describe("recordRSC", () => { @@ -173,4 +240,22 @@ describe("observability/simple-metrics/metrics-recorder", () => { assertEquals(state.apiRetries, 1); }); }); + + describe("recordContentNetworkFetch", () => { + it("keeps durations, request counters, and buckets within safe integer range", () => { + resetMetrics(); + state.contentNetworkFetches = Number.MAX_SAFE_INTEGER; + state.contentNetworkFetchMsTotal = Number.MAX_SAFE_INTEGER - 1; + state.contentPreviewRequests = Number.MAX_SAFE_INTEGER; + const lastBucket = state._contentNetworkCounts.length - 1; + state._contentNetworkCounts[lastBucket] = Number.MAX_SAFE_INTEGER; + + recordContentNetworkFetch(Number.MAX_VALUE, true); + + assertEquals(state.contentNetworkFetches, Number.MAX_SAFE_INTEGER); + assertEquals(state.contentNetworkFetchMsTotal, Number.MAX_SAFE_INTEGER); + assertEquals(state.contentPreviewRequests, Number.MAX_SAFE_INTEGER); + assertEquals(state._contentNetworkCounts[lastBucket], Number.MAX_SAFE_INTEGER); + }); + }); }); diff --git a/src/observability/simple-metrics/metrics-recorder.ts b/src/observability/simple-metrics/metrics-recorder.ts index 3a38d57a5f..5ff1fd035a 100644 --- a/src/observability/simple-metrics/metrics-recorder.ts +++ b/src/observability/simple-metrics/metrics-recorder.ts @@ -3,11 +3,15 @@ * @module */ -import { getSSRBoundaries, state } from "./metrics-state.ts"; +import { getContentNetworkBoundaries, getSSRBoundaries, state } from "./metrics-state.ts"; import { getObservabilityMetrics } from "./observability-loader.ts"; import { getOtelInstruments, safeOtelOperation } from "./otel-instruments.ts"; +import { nonNegativeSafeInteger, saturatingAdd } from "../metrics/numeric.ts"; import type { RSCRequestKind } from "./types.ts"; +const SSR_BOUNDARIES = getSSRBoundaries(); +const CONTENT_NETWORK_BOUNDARIES = getContentNetworkBoundaries(); + function recordObservability( fn: (obs: Awaited>) => void, ): void { @@ -30,10 +34,14 @@ function recordObservability( * ``` */ export async function incRequest(): Promise { - state.requests++; + state.requests = saturatingAdd(state.requests); - const obs = await getObservabilityMetrics(); - obs?.recordHttpRequest(); + try { + const obs = await getObservabilityMetrics(); + obs?.recordHttpRequest(); + } catch { + /* metrics recording failure - non-critical */ + } const otel = getOtelInstruments(); await safeOtelOperation(() => otel.requestCounter?.add(1), "incRequest counter add failed"); @@ -52,14 +60,17 @@ export async function incRequest(): Promise { * ``` */ export function recordHttp(resolved: number, blocked: number, fetchMsTotal: number): void { - state.jitHttpResolved += resolved; - state.jitHttpBlocked += blocked; - state.jitHttpFetchMsTotal += Math.floor(fetchMsTotal); + const resolvedCount = nonNegativeSafeInteger(resolved); + const blockedCount = nonNegativeSafeInteger(blocked); + const fetchDuration = nonNegativeSafeInteger(fetchMsTotal); + state.jitHttpResolved = saturatingAdd(state.jitHttpResolved, resolvedCount); + state.jitHttpBlocked = saturatingAdd(state.jitHttpBlocked, blockedCount); + state.jitHttpFetchMsTotal = saturatingAdd(state.jitHttpFetchMsTotal, fetchDuration); const otel = getOtelInstruments(); void safeOtelOperation(() => { - if (resolved) otel.jitResolvedCounter?.add(resolved); - if (blocked) otel.jitBlockedCounter?.add(blocked); + if (resolvedCount) otel.jitResolvedCounter?.add(resolvedCount); + if (blockedCount) otel.jitBlockedCounter?.add(blockedCount); }, "HTTP counters add failed"); } @@ -75,9 +86,9 @@ export function recordHttp(resolved: number, blocked: number, fetchMsTotal: numb * ``` */ export function recordCacheGet(hit: boolean): void { - state.cacheGets++; - if (hit) state.cacheHits++; - else state.cacheMisses++; + state.cacheGets = saturatingAdd(state.cacheGets); + if (hit) state.cacheHits = saturatingAdd(state.cacheHits); + else state.cacheMisses = saturatingAdd(state.cacheMisses); recordObservability((obs) => obs?.recordCacheGet(hit)); @@ -98,7 +109,7 @@ export function recordCacheGet(hit: boolean): void { * ``` */ export function recordCacheSet(): void { - state.cacheSets++; + state.cacheSets = saturatingAdd(state.cacheSets); recordObservability((obs) => obs?.recordCacheSet()); @@ -117,8 +128,9 @@ export function recordCacheSet(): void { * ``` */ export function recordCacheInvalidate(n: number): void { - const count = n | 0; - state.cacheInvalidations += count; + const count = nonNegativeSafeInteger(n); + if (count === 0) return; + state.cacheInvalidations = saturatingAdd(state.cacheInvalidations, count); recordObservability((obs) => obs?.recordCacheInvalidate(count)); @@ -132,17 +144,17 @@ export function recordCacheInvalidate(n: number): void { export type ModuleServeStatus = "ok" | "not_found" | "error"; export function recordModuleServe(status: ModuleServeStatus): void { - state.moduleServeTotal++; + state.moduleServeTotal = saturatingAdd(state.moduleServeTotal); switch (status) { case "ok": - state.moduleServeOk++; + state.moduleServeOk = saturatingAdd(state.moduleServeOk); break; case "not_found": - state.moduleServeNotFound++; + state.moduleServeNotFound = saturatingAdd(state.moduleServeNotFound); break; case "error": - state.moduleServeError++; + state.moduleServeError = saturatingAdd(state.moduleServeError); break; } @@ -154,9 +166,12 @@ export function recordModuleServe(status: ModuleServeStatus): void { } export function recordModuleTransform(durationMs: number): void { - const duration = Math.max(0, Math.floor(durationMs)); - state.moduleTransformTotal++; - state.moduleTransformDurationMsTotal += duration; + const duration = nonNegativeSafeInteger(durationMs); + state.moduleTransformTotal = saturatingAdd(state.moduleTransformTotal); + state.moduleTransformDurationMsTotal = saturatingAdd( + state.moduleTransformDurationMsTotal, + duration, + ); const otel = getOtelInstruments(); void safeOtelOperation(() => { @@ -166,8 +181,8 @@ export function recordModuleTransform(durationMs: number): void { } export function recordRouteManifestLookup(hit: boolean): void { - if (hit) state.routeManifestLruHits++; - else state.routeManifestLruMisses++; + if (hit) state.routeManifestLruHits = saturatingAdd(state.routeManifestLruHits); + else state.routeManifestLruMisses = saturatingAdd(state.routeManifestLruMisses); const otel = getOtelInstruments(); void safeOtelOperation( @@ -187,13 +202,12 @@ export function recordRouteManifestLookup(hit: boolean): void { * ``` */ export function recordSSR(durationMs: number): void { - const d = Math.max(0, Math.floor(durationMs)); - const boundaries = getSSRBoundaries(); + const d = nonNegativeSafeInteger(durationMs); - let idx = boundaries.findIndex((b) => d <= b); + let idx = SSR_BOUNDARIES.findIndex((b) => d <= b); if (idx === -1) idx = state._ssrCounts.length - 1; - state._ssrCounts[idx] = (state._ssrCounts[idx] ?? 0) + 1; + state._ssrCounts[idx] = saturatingAdd(state._ssrCounts[idx] ?? 0); recordObservability((obs) => obs?.recordRender(d)); @@ -212,23 +226,25 @@ export function recordSSR(durationMs: number): void { * ``` */ export function recordRSCStreamDuration(durationMs: number): void { - const boundaries = getSSRBoundaries(); - const d = Math.max(0, Math.floor(durationMs)); + const d = nonNegativeSafeInteger(durationMs); state.rscStreamHistogram ??= { - boundaries: [...boundaries], - counts: Array.from({ length: boundaries.length + 1 }, () => 0), + boundaries: [...SSR_BOUNDARIES], + counts: Array.from({ length: SSR_BOUNDARIES.length + 1 }, () => 0), }; - let idx = boundaries.findIndex((b) => d <= b); + let idx = SSR_BOUNDARIES.findIndex((b) => d <= b); if (idx === -1) idx = state.rscStreamHistogram.counts.length - 1; - state.rscStreamHistogram.counts[idx] = (state.rscStreamHistogram.counts[idx] ?? 0) + 1; + state.rscStreamHistogram.counts[idx] = saturatingAdd( + state.rscStreamHistogram.counts[idx] ?? 0, + ); recordObservability((obs) => obs?.recordRSCStream(d)); } type ObservabilityRSCKind = "manifest" | "page" | "stream" | "action"; +type RSCStateCounter = "rscManifest" | "rscPage" | "rscStream" | "rscAction" | "rscErrors"; function recordObservabilityRSC(obsKind: ObservabilityRSCKind): void { recordObservability((obs) => obs?.recordRSCRequest(obsKind)); @@ -237,7 +253,7 @@ function recordObservabilityRSC(obsKind: ObservabilityRSCKind): void { /** RSC kind to state property and observability kind mapping */ const RSC_KIND_MAP: Record< RSCRequestKind, - { prop: keyof typeof state; obs?: ObservabilityRSCKind } + { prop: RSCStateCounter; obs?: ObservabilityRSCKind } > = { manifest: { prop: "rscManifest", obs: "manifest" }, page: { prop: "rscPage", obs: "page" }, @@ -260,7 +276,7 @@ const RSC_KIND_MAP: Record< */ export function recordRSC(kind: RSCRequestKind): void { const { prop, obs } = RSC_KIND_MAP[kind]; - state[prop]++; + state[prop] = saturatingAdd(state[prop]); if (obs) recordObservabilityRSC(obs); } @@ -273,7 +289,7 @@ export function recordRSC(kind: RSCRequestKind): void { * ``` */ export function recordCorsRejection(): void { - state.corsRejections++; + state.corsRejections = saturatingAdd(state.corsRejections); } /** @@ -285,33 +301,32 @@ export function recordCorsRejection(): void { * ``` */ export function recordSecurityHeaders(): void { - state.securityHeadersApplied++; + state.securityHeadersApplied = saturatingAdd(state.securityHeadersApplied); } export function recordApiRequest(status: number): void { + if (!Number.isFinite(status)) return; if (status >= 200 && status < 300) { - state.apiRequests2xx++; + state.apiRequests2xx = saturatingAdd(state.apiRequests2xx); return; } if (status >= 400 && status < 500) { - state.apiRequests4xx++; + state.apiRequests4xx = saturatingAdd(state.apiRequests4xx); return; } - if (status >= 500) state.apiRequests5xx++; + if (status >= 500) state.apiRequests5xx = saturatingAdd(state.apiRequests5xx); } export function recordApiRetry(): void { - state.apiRetries++; + state.apiRetries = saturatingAdd(state.apiRetries); } // ============================================================================ // Content Cache Metrics - Track cache behavior for file reads // ============================================================================ -import { getContentNetworkBoundaries } from "./metrics-state.ts"; - export type ContentCacheLayer = "request" | "persistent" | "filelist"; /** @@ -329,13 +344,13 @@ export type ContentCacheLayer = "request" | "persistent" | "filelist"; export function recordContentCacheHit(layer: ContentCacheLayer): void { switch (layer) { case "request": - state.contentRequestScopedHits++; + state.contentRequestScopedHits = saturatingAdd(state.contentRequestScopedHits); break; case "persistent": - state.contentPersistentCacheHits++; + state.contentPersistentCacheHits = saturatingAdd(state.contentPersistentCacheHits); break; case "filelist": - state.contentFileListHits++; + state.contentFileListHits = saturatingAdd(state.contentFileListHits); break; } } @@ -353,22 +368,21 @@ export function recordContentCacheHit(layer: ContentCacheLayer): void { * ``` */ export function recordContentNetworkFetch(durationMs: number, isPreview: boolean): void { - const d = Math.max(0, Math.floor(durationMs)); - const boundaries = getContentNetworkBoundaries(); + const d = nonNegativeSafeInteger(durationMs); // Update counters - state.contentNetworkFetches++; - state.contentNetworkFetchMsTotal += d; + state.contentNetworkFetches = saturatingAdd(state.contentNetworkFetches); + state.contentNetworkFetchMsTotal = saturatingAdd(state.contentNetworkFetchMsTotal, d); // Track preview vs production if (isPreview) { - state.contentPreviewRequests++; + state.contentPreviewRequests = saturatingAdd(state.contentPreviewRequests); } else { - state.contentProductionRequests++; + state.contentProductionRequests = saturatingAdd(state.contentProductionRequests); } // Update histogram - let idx = boundaries.findIndex((b) => d <= b); + let idx = CONTENT_NETWORK_BOUNDARIES.findIndex((b) => d <= b); if (idx === -1) idx = state._contentNetworkCounts.length - 1; - state._contentNetworkCounts[idx] = (state._contentNetworkCounts[idx] ?? 0) + 1; + state._contentNetworkCounts[idx] = saturatingAdd(state._contentNetworkCounts[idx] ?? 0); } diff --git a/src/observability/simple-metrics/metrics-state.test.ts b/src/observability/simple-metrics/metrics-state.test.ts index 3eff15f1ee..e1be5eed3f 100644 --- a/src/observability/simple-metrics/metrics-state.test.ts +++ b/src/observability/simple-metrics/metrics-state.test.ts @@ -11,6 +11,13 @@ import { describe("observability/simple-metrics/metrics-state", () => { describe("getSSRBoundaries", () => { + it("returns a defensive copy", () => { + const boundaries = getSSRBoundaries(); + boundaries[0] = -1; + + assertEquals(getSSRBoundaries()[0], 5); + }); + it("should return array of boundary values", () => { const boundaries = getSSRBoundaries(); assertEquals(Array.isArray(boundaries), true); diff --git a/src/observability/simple-metrics/metrics-state.ts b/src/observability/simple-metrics/metrics-state.ts index 43a7edb7d3..0db0fdfbfd 100644 --- a/src/observability/simple-metrics/metrics-state.ts +++ b/src/observability/simple-metrics/metrics-state.ts @@ -49,11 +49,11 @@ export const state: MetricsState = { }; export function getSSRBoundaries(): number[] { - return SSR_BOUNDARIES_MS; + return [...SSR_BOUNDARIES_MS]; } export function getContentNetworkBoundaries(): number[] { - return CONTENT_NETWORK_BOUNDARIES_MS; + return [...CONTENT_NETWORK_BOUNDARIES_MS]; } export function createSnapshot(): VeryfrontMetrics { diff --git a/src/observability/simple-metrics/otel-instruments.test.ts b/src/observability/simple-metrics/otel-instruments.test.ts index 9483a1fca8..41b566f46d 100644 --- a/src/observability/simple-metrics/otel-instruments.test.ts +++ b/src/observability/simple-metrics/otel-instruments.test.ts @@ -1,6 +1,7 @@ import "#veryfront/schemas/_test-setup.ts"; import { assertEquals } from "#veryfront/testing/assert.ts"; -import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { afterEach, beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { _resetShimForTests, setGlobalMetricsAPI } from "../tracing/api-shim.ts"; import { getOtelInstruments, resetOtelInstruments, @@ -13,6 +14,10 @@ describe("observability/simple-metrics/otel-instruments", () => { resetOtelInstruments(); }); + afterEach(() => { + _resetShimForTests(); + }); + describe("getOtelInstruments", () => { it("should return an instruments object", () => { assertEquals(typeof getOtelInstruments(), "object"); @@ -47,6 +52,22 @@ describe("observability/simple-metrics/otel-instruments", () => { resetOtelInstruments(); await safeOtelOperation(() => {}, "test"); }); + + it("does not publish an in-flight candidate after reset", async () => { + const meter = { + createHistogram: () => ({ record() {} }), + createCounter: () => ({ add() {} }), + createUpDownCounter: () => ({ add() {} }), + createObservableGauge: () => ({ addCallback() {} }), + }; + setGlobalMetricsAPI({ getMeter: () => meter }); + + const pending = safeOtelOperation(() => {}, "pending init"); + resetOtelInstruments(); + await pending; + + assertEquals(getOtelInstruments().meter, undefined); + }); }); describe("safeLogWarn", () => { @@ -64,6 +85,59 @@ describe("observability/simple-metrics/otel-instruments", () => { }); describe("safeOtelOperation", () => { + it("initializes after a metrics provider is registered late", async () => { + _resetShimForTests(); + await safeOtelOperation(() => {}, "before provider"); + let getMeterCalls = 0; + const meter = { + createHistogram: () => ({ record() {} }), + createCounter: () => ({ add() {} }), + createUpDownCounter: () => ({ add() {} }), + createObservableGauge: () => ({ addCallback() {} }), + }; + setGlobalMetricsAPI({ + getMeter: () => { + getMeterCalls++; + return meter; + }, + }); + + await safeOtelOperation(() => {}, "after provider"); + + assertEquals(getMeterCalls, 1); + assertEquals(getOtelInstruments().meter, meter); + }); + + it("publishes instruments atomically and retries a failed revision", async () => { + let shouldFail = true; + let counterCreations = 0; + const meter = { + createHistogram: () => ({ record() {} }), + createCounter: () => { + counterCreations++; + if (shouldFail && counterCreations === 2) { + throw new Error("transient instrument failure"); + } + return { add() {} }; + }, + createUpDownCounter: () => ({ add() {} }), + createObservableGauge: () => ({ addCallback() {} }), + }; + setGlobalMetricsAPI({ getMeter: () => meter }); + + await safeOtelOperation(() => {}, "first attempt"); + + assertEquals(getOtelInstruments().meter, undefined); + assertEquals(getOtelInstruments().requestCounter, undefined); + + shouldFail = false; + await safeOtelOperation(() => {}, "retry"); + + assertEquals(getOtelInstruments().meter, meter); + assertEquals(typeof getOtelInstruments().requestCounter?.add, "function"); + assertEquals(typeof getOtelInstruments().routeManifestLookupCounter?.add, "function"); + }); + it("should execute the operation", async () => { let executed = false; diff --git a/src/observability/simple-metrics/otel-instruments.ts b/src/observability/simple-metrics/otel-instruments.ts index 0ef9b9f361..b361341837 100644 --- a/src/observability/simple-metrics/otel-instruments.ts +++ b/src/observability/simple-metrics/otel-instruments.ts @@ -3,18 +3,27 @@ * @module */ -import { isDeno } from "#veryfront/platform/compat/runtime.ts"; import { serverLogger as logger } from "#veryfront/utils"; import { VERSION } from "#veryfront/utils/version.ts"; -import { getGlobalMetricsAPI } from "#veryfront/observability/tracing/api-shim.ts"; +import { + getGlobalMetricsAPI, + getMetricsApiRevision, +} from "#veryfront/observability/tracing/api-shim.ts"; import type { OtelInstruments } from "./types.ts"; -// In-flight or completed init promise; null means init has not started. -// Using a promise (rather than a boolean flag) prevents a race where a second -// concurrent caller sees the flag set to true but instruments are not yet ready. +// A single in-flight attempt is shared by callers for a provider revision. let initPromise: Promise | null = null; +let initializingRevision = -1; +let initializedRevision = -1; +let lifecycleGeneration = 0; const otel: OtelInstruments = {}; +function clearOtelInstruments(): void { + for (const key of Object.keys(otel) as (keyof OtelInstruments)[]) { + delete otel[key]; + } +} + export function safeLogWarn(message: string, error?: unknown): void { try { logger.warn(message, error); @@ -23,72 +32,94 @@ export function safeLogWarn(message: string, error?: unknown): void { } } -async function doInitOtelInstruments(): Promise { - if (!isDeno) return; - +async function createOtelInstruments(): Promise { try { // The metrics API is injected by ext-observability-opentelemetry via setGlobalMetricsAPI(). // When the extension is not active, the meter is unavailable and we return. const metricsApi = getGlobalMetricsAPI(); - if (!metricsApi) return; + if (!metricsApi) return {}; const meter = metricsApi.getMeter("veryfront", VERSION); + const candidate: OtelInstruments = { meter }; - otel.meter = meter; - otel.ssrHistogram = meter.createHistogram("veryfront.ssr.duration", { + candidate.ssrHistogram = meter.createHistogram("veryfront.ssr.duration", { description: "SSR render duration (ms)", unit: "ms", }); - otel.requestCounter = meter.createCounter("veryfront.http.requests", { + candidate.requestCounter = meter.createCounter("veryfront.http.requests", { description: "Requests handled", }); - otel.jitResolvedCounter = meter.createCounter("veryfront.jit.http.resolved", { + candidate.jitResolvedCounter = meter.createCounter("veryfront.jit.http.resolved", { description: "JIT HTTP resolved", }); - otel.jitBlockedCounter = meter.createCounter("veryfront.jit.http.blocked", { + candidate.jitBlockedCounter = meter.createCounter("veryfront.jit.http.blocked", { description: "JIT HTTP blocked", }); - otel.cacheGetCounter = meter.createCounter("veryfront.cache.gets", { + candidate.cacheGetCounter = meter.createCounter("veryfront.cache.gets", { description: "Cache gets", }); - otel.cacheHitCounter = meter.createCounter("veryfront.cache.hits", { + candidate.cacheHitCounter = meter.createCounter("veryfront.cache.hits", { description: "Cache hits", }); - otel.cacheMissCounter = meter.createCounter("veryfront.cache.misses", { + candidate.cacheMissCounter = meter.createCounter("veryfront.cache.misses", { description: "Cache misses", }); - otel.cacheSetCounter = meter.createCounter("veryfront.cache.sets", { + candidate.cacheSetCounter = meter.createCounter("veryfront.cache.sets", { description: "Cache sets", }); - otel.cacheInvalidateCounter = meter.createCounter("veryfront.cache.invalidations", { + candidate.cacheInvalidateCounter = meter.createCounter("veryfront.cache.invalidations", { description: "Cache invalidations", }); - otel.moduleServeCounter = meter.createCounter("veryfront.module.serve.total", { + candidate.moduleServeCounter = meter.createCounter("veryfront.module.serve.total", { description: "Module server responses by status", }); - otel.moduleTransformCounter = meter.createCounter("veryfront.module.transform.total", { + candidate.moduleTransformCounter = meter.createCounter("veryfront.module.transform.total", { description: "Module transforms", }); - otel.moduleTransformDurationHistogram = meter.createHistogram( + candidate.moduleTransformDurationHistogram = meter.createHistogram( "veryfront.module.transform.duration", { description: "Module transform duration (ms)", unit: "ms", }, ); - otel.routeManifestLookupCounter = meter.createCounter("veryfront.route_manifest.lookup.total", { - description: "Route module manifest LRU lookups by hit status", - }); + candidate.routeManifestLookupCounter = meter.createCounter( + "veryfront.route_manifest.lookup.total", + { + description: "Route module manifest LRU lookups by hit status", + }, + ); + return candidate; } catch (e) { safeLogWarn("[metrics] OpenTelemetry init failed", e); + return null; } } export function ensureOtelInstruments(): Promise { - if (!initPromise) { - initPromise = doInitOtelInstruments(); - } - return initPromise; + const providerRevision = getMetricsApiRevision(); + if (initializedRevision === providerRevision) return Promise.resolve(); + if (initPromise && initializingRevision === providerRevision) return initPromise; + + clearOtelInstruments(); + initializingRevision = providerRevision; + const generation = lifecycleGeneration; + const attempt = createOtelInstruments() + .then((candidate) => { + if ( + !candidate || generation !== lifecycleGeneration || + getMetricsApiRevision() !== providerRevision + ) return; + Object.assign(otel, candidate); + initializedRevision = providerRevision; + }) + .finally(() => { + if (initPromise !== attempt) return; + initPromise = null; + initializingRevision = -1; + }); + initPromise = attempt; + return attempt; } export async function safeOtelOperation( @@ -108,9 +139,9 @@ export function getOtelInstruments(): OtelInstruments { } export function resetOtelInstruments(): void { + lifecycleGeneration++; initPromise = null; - - for (const key of Object.keys(otel) as (keyof OtelInstruments)[]) { - delete otel[key]; - } + initializingRevision = -1; + initializedRevision = -1; + clearOtelInstruments(); } diff --git a/src/observability/telemetry-error.test.ts b/src/observability/telemetry-error.test.ts new file mode 100644 index 0000000000..fc11766aa1 --- /dev/null +++ b/src/observability/telemetry-error.test.ts @@ -0,0 +1,626 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { API_CLIENT_ERROR } from "#veryfront/errors"; +import { assertEquals, assertExists } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { + MAX_STRING_DISPLAY_LENGTH, + MAX_TRACE_ATTRIBUTE_VALUE_SIZE, +} from "#veryfront/utils/constants/index.ts"; +import { + MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES, + MAX_TELEMETRY_ATTRIBUTE_COUNT, + MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH, +} from "./limits.ts"; +import { + sanitizeErrorForTelemetry, + sanitizeStructuredTelemetryData, + sanitizeTelemetryAttributes, + type TelemetryAttributeValue, +} from "./telemetry-error.ts"; +import { isNativeErrorWithoutHooks } from "#veryfront/platform/compat/error-introspection.ts"; + +describe("observability/telemetry-error", () => { + it("sanitizes hostile flattened attributes without throwing", () => { + const attributes: Record = { safe: "value" }; + Object.defineProperty(attributes, "detail", { + enumerable: true, + get() { + throw new Error("hostile attribute getter"); + }, + }); + Object.defineProperty(attributes, "apiKey", { + enumerable: true, + get() { + throw new Error("secret getter must not run"); + }, + }); + + assertEquals(sanitizeTelemetryAttributes(attributes), { + safe: "value", + detail: "[REDACTED]", + apiKey: "[REDACTED]", + }); + }); + + it("returns an empty safe attribute record when enumeration is hostile", () => { + const attributes = new Proxy({}, { + ownKeys() { + throw new Error("hostile ownKeys"); + }, + }); + + assertEquals(sanitizeTelemetryAttributes(attributes), {}); + }); + + it("preserves numeric semantic token counts while redacting token secrets", () => { + const attributes: Record = { + "gen_ai.usage.input_tokens": 2, + "gen_ai.usage.output_tokens": 3, + "gen_ai.usage.total_tokens": 5, + token: 12345, + "gen_ai.usage.prompt_tokens": "secret", + }; + + assertEquals( + sanitizeTelemetryAttributes(attributes), + { + "gen_ai.usage.input_tokens": 2, + "gen_ai.usage.output_tokens": 3, + "gen_ai.usage.total_tokens": 5, + token: "[REDACTED]", + "gen_ai.usage.prompt_tokens": "[REDACTED]", + }, + ); + }); + + it("sanitizes values with hostile prototype inspection without throwing", () => { + let proxyTrapCalls = 0; + const hostile = new Proxy({}, { + getPrototypeOf() { + proxyTrapCalls++; + throw new Error("prototype unavailable"); + }, + get() { + proxyTrapCalls++; + throw new Error("property unavailable"); + }, + getOwnPropertyDescriptor() { + proxyTrapCalls++; + throw new Error("descriptor unavailable"); + }, + ownKeys() { + proxyTrapCalls++; + throw new Error("keys unavailable"); + }, + }); + + const sanitized = sanitizeErrorForTelemetry(hostile); + + assertEquals(sanitized.name, "Unknown"); + assertEquals(sanitized.message, "Unknown error"); + assertEquals(proxyTrapCalls, 0); + }); + + it("snapshots native errors without invoking project-owned accessors", () => { + let accessorCalls = 0; + const hostile = new Error("must stay private"); + Reflect.deleteProperty(hostile, "stack"); + for (const key of ["stack", "message", "name"] as const) { + Object.defineProperty(hostile, key, { + configurable: true, + get(): never { + accessorCalls += 1; + throw new Error(`${key} accessor must not run`); + }, + }); + } + + const sanitized = sanitizeErrorForTelemetry(hostile); + + assertEquals(sanitized.name, "Error"); + assertEquals(sanitized.message, "Unknown error"); + assertEquals(accessorCalls, 0); + }); + + it("ignores inherited descriptor values without invoking accessors", () => { + const hostile = new Error("must stay private"); + let errorAccessorCalls = 0; + Object.defineProperty(hostile, "message", { + configurable: true, + get(): never { + errorAccessorCalls += 1; + throw new Error("error accessor must not run"); + }, + }); + + const previous = Object.getOwnPropertyDescriptor(Object.prototype, "value"); + let inheritedValueCalls = 0; + let sanitized: Error | undefined; + Object.defineProperty(Object.prototype, "value", { + configurable: true, + get(): never { + inheritedValueCalls += 1; + throw new Error("inherited descriptor value must not run"); + }, + }); + + try { + sanitized = sanitizeErrorForTelemetry(hostile); + } finally { + if (previous) { + Object.defineProperty(Object.prototype, "value", previous); + } else { + delete (Object.prototype as Record).value; + } + } + + assertEquals(sanitized?.name, "Error"); + assertEquals(sanitized?.message, "Unknown error"); + assertEquals(errorAccessorCalls, 0); + assertEquals(inheritedValueCalls, 0); + }); + + it("survives inherited property-descriptor poisoning without running the getter", () => { + const previous = Object.getOwnPropertyDescriptor(Object.prototype, "enumerable"); + let getterCalls = 0; + let sanitized: Error | undefined; + let failure: unknown; + Object.defineProperty(Object.prototype, "enumerable", { + configurable: true, + get(): never { + getterCalls += 1; + throw new Error("inherited descriptor getter must not run"); + }, + }); + + try { + sanitized = sanitizeErrorForTelemetry(new Error("application failure")); + } catch (error) { + failure = error; + } finally { + if (previous) { + Object.defineProperty(Object.prototype, "enumerable", previous); + } else { + delete (Object.prototype as Record).enumerable; + } + } + + assertEquals(failure, undefined); + assertEquals(getterCalls, 0); + assertEquals(sanitized?.name, "Error"); + assertEquals(sanitized?.message, "application failure"); + }); + + it("uses captured string slicing while redacting and bounding error messages", () => { + const previous = Object.getOwnPropertyDescriptor(String.prototype, "slice"); + let sliceCalls = 0; + let sanitized: Error | undefined; + let failure: unknown; + Object.defineProperty(String.prototype, "slice", { + configurable: true, + value: () => { + sliceCalls += 1; + throw new Error("poisoned String.prototype.slice"); + }, + writable: true, + }); + + try { + sanitized = sanitizeErrorForTelemetry( + new Error( + `https://user:password@example.test/${"x".repeat(MAX_STRING_DISPLAY_LENGTH + 1)}`, + ), + ); + } catch (error) { + failure = error; + } finally { + if (previous) Object.defineProperty(String.prototype, "slice", previous); + } + + assertEquals(failure, undefined); + assertEquals(sliceCalls, 0); + assertEquals(sanitized?.name, "Error"); + assertEquals(sanitized?.message.length, MAX_STRING_DISPLAY_LENGTH); + assertEquals(sanitized?.message.includes("password"), false); + }); + + it("uses captured slicing while bounding flattened attribute keys", () => { + const previousStringSlice = Object.getOwnPropertyDescriptor(String.prototype, "slice"); + const previousArraySlice = Object.getOwnPropertyDescriptor(Array.prototype, "slice"); + const key = "k".repeat(MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH + 1); + let sliceCalls = 0; + let snapshot: Record | undefined; + let failure: unknown; + const poisonedSlice = () => { + sliceCalls += 1; + throw new Error("poisoned slice must not run"); + }; + Object.defineProperty(String.prototype, "slice", { + configurable: true, + value: poisonedSlice, + writable: true, + }); + Object.defineProperty(Array.prototype, "slice", { + configurable: true, + value: poisonedSlice, + writable: true, + }); + + try { + snapshot = sanitizeTelemetryAttributes({ [key]: "value" }); + } catch (error) { + failure = error; + } finally { + if (previousStringSlice) { + Object.defineProperty(String.prototype, "slice", previousStringSlice); + } + if (previousArraySlice) { + Object.defineProperty(Array.prototype, "slice", previousArraySlice); + } + } + + const retainedKeys = Object.keys(snapshot ?? {}); + assertEquals(failure, undefined); + assertEquals(sliceCalls, 0); + assertEquals(retainedKeys.length, 1); + assertEquals(retainedKeys[0]?.length, MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH); + assertEquals(snapshot?.[retainedKeys[0] ?? ""], "value"); + }); + + it("never materializes a stack through Error.prepareStackTrace", () => { + const ErrorWithStackFormatter = Error as ErrorConstructor & { + prepareStackTrace?: (error: Error, callSites: unknown[]) => unknown; + }; + const previous = Object.getOwnPropertyDescriptor(ErrorWithStackFormatter, "prepareStackTrace"); + let formatterCalls = 0; + let sanitized: Error | undefined; + Object.defineProperty(ErrorWithStackFormatter, "prepareStackTrace", { + configurable: true, + value: () => { + formatterCalls += 1; + throw new Error("prepareStackTrace must not run"); + }, + writable: true, + }); + + try { + sanitized = sanitizeErrorForTelemetry(new Error("application failure")); + } finally { + if (previous) { + Object.defineProperty(ErrorWithStackFormatter, "prepareStackTrace", previous); + } else { + delete ErrorWithStackFormatter.prepareStackTrace; + } + } + + assertEquals(formatterCalls, 0); + assertEquals(sanitized?.message, "application failure"); + assertEquals(sanitized?.stack, undefined); + assertEquals(sanitized instanceof Error, true); + }); + + it("does not trust Error.isError when probing stack descriptor behavior", async () => { + const ErrorWithStackFormatter = Error as ErrorConstructor & { + isError?: unknown; + prepareStackTrace?: (error: Error, callSites: unknown[]) => unknown; + }; + const previousIsError = Object.getOwnPropertyDescriptor(ErrorWithStackFormatter, "isError"); + const previousFormatter = Object.getOwnPropertyDescriptor( + ErrorWithStackFormatter, + "prepareStackTrace", + ); + const previousValue = Object.getOwnPropertyDescriptor(Object.prototype, "value"); + let isErrorCalls = 0; + let formatterCalls = 0; + let inheritedValueCalls = 0; + const fakeIsError = () => { + isErrorCalls += 1; + return true; + }; + const hostileFormatter = () => { + formatterCalls += 1; + throw new Error("prepareStackTrace must not run"); + }; + Object.defineProperty(ErrorWithStackFormatter, "isError", { + configurable: true, + value: fakeIsError, + writable: true, + }); + Object.defineProperty(ErrorWithStackFormatter, "prepareStackTrace", { + configurable: true, + value: hostileFormatter, + writable: true, + }); + Object.defineProperty(Object.prototype, "value", { + configurable: true, + get(): never { + inheritedValueCalls += 1; + throw new Error("inherited descriptor value must not run"); + }, + }); + + let sanitized: Error | undefined; + let restoredFormatter: unknown; + try { + const isolated = await import("./telemetry-error.ts?hostile-stack-capability-flags"); + restoredFormatter = Object.getOwnPropertyDescriptor( + ErrorWithStackFormatter, + "prepareStackTrace", + )?.value; + sanitized = isolated.sanitizeErrorForTelemetry(new Error("application failure")); + } finally { + if (previousValue) { + Object.defineProperty(Object.prototype, "value", previousValue); + } else { + delete (Object.prototype as Record).value; + } + if (previousFormatter) { + Object.defineProperty(ErrorWithStackFormatter, "prepareStackTrace", previousFormatter); + } else { + delete ErrorWithStackFormatter.prepareStackTrace; + } + if (previousIsError) { + Object.defineProperty(ErrorWithStackFormatter, "isError", previousIsError); + } else { + delete (ErrorWithStackFormatter as unknown as { isError?: unknown }).isError; + } + } + + assertEquals(restoredFormatter, hostileFormatter); + assertEquals(isErrorCalls, 0); + assertEquals(formatterCalls, 0); + assertEquals(inheritedValueCalls, 0); + assertEquals(sanitized?.name, "Error"); + assertEquals(sanitized?.message, "application failure"); + }); + + it("groups safe built-in, DOM, custom, and framework errors", () => { + class CustomError extends Error {} + class HostileConstructorError extends Error {} + class NamedCustomError extends Error { + constructor(message: string) { + super(message); + this.name = "NamedCustomError"; + } + } + + let constructorReads = 0; + Object.defineProperty(HostileConstructorError.prototype, "constructor", { + configurable: true, + get(): never { + constructorReads += 1; + throw new Error("custom constructor getter must not run"); + }, + }); + + const aggregate = sanitizeErrorForTelemetry( + new AggregateError([new Error("nested secret")], "aggregate failure"), + ); + const custom = sanitizeErrorForTelemetry(new CustomError("custom failure")); + const hostileConstructor = sanitizeErrorForTelemetry( + new HostileConstructorError("hostile constructor failure"), + ); + const named = sanitizeErrorForTelemetry(new NamedCustomError("named failure")); + const dom = sanitizeErrorForTelemetry(new DOMException("request stopped", "AbortError")); + const framework = sanitizeErrorForTelemetry( + API_CLIENT_ERROR.create({ detail: "upstream unavailable" }), + ); + + assertEquals(aggregate.name, "AggregateError"); + assertEquals(aggregate.message, "aggregate failure"); + assertEquals(custom.name, "CustomError"); + assertEquals(custom.message, "custom failure"); + assertEquals(hostileConstructor.name, "Error"); + assertEquals(hostileConstructor.message, "hostile constructor failure"); + assertEquals(named.name, "NamedCustomError"); + assertEquals(named.message, "named failure"); + assertEquals( + dom.name, + isNativeErrorWithoutHooks(new DOMException()) ? "DOMException" : "Unknown", + ); + assertEquals( + dom.message, + isNativeErrorWithoutHooks(new DOMException()) ? "" : "Unknown error", + ); + assertEquals(framework.name, "VeryfrontError"); + assertEquals(framework.message, "upstream unavailable"); + assertEquals(constructorReads, 0); + }); + + it("preserves native errors when the standard Error.isError entry point is unavailable", async () => { + const previous = Object.getOwnPropertyDescriptor(Error, "isError"); + let proxyTrapCalls = 0; + Object.defineProperty(Error, "isError", { + configurable: true, + value: undefined, + writable: true, + }); + + try { + const isolated = await import("./telemetry-error.ts?without-hook-free-error-brand-check"); + const native = isolated.sanitizeErrorForTelemetry(new Error("must stay opaque")); + const proxy = isolated.sanitizeErrorForTelemetry( + new Proxy({}, { + get(): never { + proxyTrapCalls += 1; + throw new Error("get trap must not run"); + }, + getOwnPropertyDescriptor(): never { + proxyTrapCalls += 1; + throw new Error("descriptor trap must not run"); + }, + getPrototypeOf(): never { + proxyTrapCalls += 1; + throw new Error("prototype trap must not run"); + }, + ownKeys(): never { + proxyTrapCalls += 1; + throw new Error("keys trap must not run"); + }, + }), + ); + + assertEquals(native.name, "Error"); + assertEquals(native.message, "must stay opaque"); + assertEquals(proxy.name, "Unknown"); + assertEquals(proxy.message, "Unknown error"); + assertEquals(proxyTrapCalls, 0); + } finally { + if (previous) { + Object.defineProperty(Error, "isError", previous); + } else { + delete (Error as unknown as { isError?: unknown }).isError; + } + } + }); + + it("deeply detaches structured data and sanitizes every serialized string", () => { + const date = new Date("2025-01-02T03:04:05.000Z"); + const url = new URL("https://user:password@example.test/path?token=secret"); + const cycle: Record = { safe: "cycle" }; + cycle.self = cycle; + const hostile: Record = {}; + Object.defineProperty(hostile, "value", { + enumerable: true, + get() { + throw new Error("hostile getter"); + }, + }); + + const snapshot = sanitizeStructuredTelemetryData({ + message: "failed https://user:password@example.test/path?access_token=secret", + apiKey: "must-not-be-read", + date, + url, + scalarJson: { + toJSON: () => "https://example.test/path?token=secret", + }, + cycle, + hostile, + }) as Record; + + assertEquals(String(snapshot.message).includes("secret"), false); + assertEquals(snapshot.apiKey, "[REDACTED]"); + assertEquals(snapshot.date instanceof Date, true); + assertEquals(snapshot.date === date, false); + assertEquals(snapshot.url instanceof URL, true); + assertEquals(snapshot.url === url, false); + assertEquals((snapshot.url as URL).href.includes("secret"), false); + assertEquals(String(snapshot.scalarJson).includes("secret"), false); + assertEquals((snapshot.cycle as Record).self, "[REDACTED]"); + assertEquals((snapshot.hostile as Record).value, "[REDACTED]"); + + const snapshotDate = snapshot.date as Date; + snapshotDate.setUTCFullYear(2030); + assertEquals(date.getUTCFullYear(), 2025); + assertExists(snapshot.scalarJson); + }); + + it("redacts structured Error proxies without invoking traps", () => { + let trapCalls = 0; + const proxy = new Proxy(new Error("must stay private"), { + get(): never { + trapCalls += 1; + throw new Error("get trap must not run"); + }, + getOwnPropertyDescriptor(): never { + trapCalls += 1; + throw new Error("descriptor trap must not run"); + }, + getPrototypeOf(): never { + trapCalls += 1; + throw new Error("prototype trap must not run"); + }, + ownKeys(): never { + trapCalls += 1; + throw new Error("ownKeys trap must not run"); + }, + }); + + assertEquals(sanitizeStructuredTelemetryData(proxy) as unknown, "[REDACTED]"); + assertEquals(trapCalls, 0); + }); + + it("snapshots structured native errors without invoking field accessors", () => { + const hostile = new Error("must stay private"); + Reflect.deleteProperty(hostile, "stack"); + let accessorCalls = 0; + for (const key of ["message", "name", "stack"] as const) { + Object.defineProperty(hostile, key, { + configurable: true, + get(): never { + accessorCalls += 1; + throw new Error(`${key} accessor must not run`); + }, + }); + } + + assertEquals(sanitizeStructuredTelemetryData(hostile), { + message: "Unknown error", + name: "Error", + stack: undefined, + }); + assertEquals(accessorCalls, 0); + }); + + it("bounds flattened attribute count, keys, strings, and arrays", () => { + const attributes: Record = { + first: "x".repeat(MAX_TRACE_ATTRIBUTE_VALUE_SIZE + 100), + array: Array.from({ length: 200 }, () => "value"), + }; + for (let index = 0; index < MAX_TELEMETRY_ATTRIBUTE_COUNT + 20; index++) { + attributes[`attribute.${index}`] = index; + } + + const snapshot = sanitizeTelemetryAttributes(attributes); + assertEquals(Object.keys(snapshot).length, MAX_TELEMETRY_ATTRIBUTE_COUNT); + assertEquals( + (snapshot.first as string).length, + MAX_TRACE_ATTRIBUTE_VALUE_SIZE, + ); + assertEquals(snapshot.array, "[REDACTED]"); + }); + + it("bounds structured telemetry returned by custom serializers", () => { + let calls = 0; + const wide = { + toJSON() { + calls++; + return Array.from( + { length: MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES + 1 }, + (_, index) => index, + ); + }, + }; + + assertEquals(sanitizeStructuredTelemetryData(wide) as unknown, "[REDACTED]"); + assertEquals(calls, 1); + assertEquals( + sanitizeStructuredTelemetryData("x".repeat(MAX_STRING_DISPLAY_LENGTH + 100)).length, + MAX_STRING_DISPLAY_LENGTH, + ); + }); + + it("bounds own string-valued error messages and stacks", () => { + const source = new Error("x".repeat(MAX_STRING_DISPLAY_LENGTH + 100)); + Object.defineProperty(source, "stack", { + configurable: true, + value: "s".repeat(MAX_STRING_DISPLAY_LENGTH + 100), + writable: true, + }); + + const snapshot = sanitizeErrorForTelemetry(source); + const stackDescriptor = Object.getOwnPropertyDescriptor(new Error(), "stack"); + const canInspectStackWithoutFormatting = Boolean( + stackDescriptor && + Object.prototype.hasOwnProperty.call(stackDescriptor, "get") && + typeof stackDescriptor.get === "function", + ); + + assertEquals(snapshot.message.length, MAX_STRING_DISPLAY_LENGTH); + assertEquals( + snapshot.stack?.length, + canInspectStackWithoutFormatting ? MAX_STRING_DISPLAY_LENGTH : undefined, + ); + }); +}); diff --git a/src/observability/telemetry-error.ts b/src/observability/telemetry-error.ts new file mode 100644 index 0000000000..f4e9a6f697 --- /dev/null +++ b/src/observability/telemetry-error.ts @@ -0,0 +1,453 @@ +import { + isSensitiveKey, + REDACTED, + sanitizeUrlCredentials, +} from "#veryfront/utils/logger/redact.ts"; +import { + LOG_PREVIEW_MAX_LENGTH_CHARS, + MAX_STRING_DISPLAY_LENGTH, + MAX_TRACE_ATTRIBUTE_VALUE_SIZE, +} from "#veryfront/utils/constants/index.ts"; +import { + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES, + MAX_STRUCTURED_TELEMETRY_DEPTH, + MAX_STRUCTURED_TELEMETRY_NODES, + MAX_TELEMETRY_ATTRIBUTE_ARRAY_LENGTH, + MAX_TELEMETRY_ATTRIBUTE_COUNT, + MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH, +} from "./limits.ts"; +import { + canInspectErrorStackDescriptorWithoutHooks, + isNativeErrorWithoutHooks, + isProxyWithoutHooks, + readNativeErrorNameWithoutHooks, +} from "#veryfront/platform/compat/error-introspection.ts"; + +const apply = Reflect.apply; +const createObject = Object.create; +const defineProperty = Object.defineProperty; +const getOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; +const objectKeys = Object.keys; +const deleteProperty = Reflect.deleteProperty; +const mathMax = Math.max; +const NativeDate = Date; +const NativeError = Error; +const NativeString = String; +const NativeURL = URL; +const dateGetTime = Date.prototype.getTime; +const objectHasOwnProperty = Object.prototype.hasOwnProperty; +const stringSlice = String.prototype.slice; +const ERROR_PROTOTYPE = NativeError.prototype; +const URL_HREF_GETTER = readOwnDescriptorGetter(NativeURL.prototype, "href"); + +const INVALID_ERROR_FIELD = Symbol("invalid-error-field"); + +function hasOwn(object: object, key: PropertyKey): boolean { + return apply(objectHasOwnProperty, object, [key]) as boolean; +} + +function readOwnDescriptorGetter( + object: object, + key: PropertyKey, +): ((this: unknown) => unknown) | undefined { + try { + const descriptor = getOwnPropertyDescriptor(object, key); + if (!descriptor || !hasOwn(descriptor, "get")) return undefined; + const getter = descriptor.get; + return typeof getter === "function" ? getter : undefined; + } catch (_) { + return undefined; + } +} + +function readOwnErrorString( + error: Error, + key: PropertyKey, +): string | undefined | typeof INVALID_ERROR_FIELD { + try { + const descriptor = getOwnPropertyDescriptor(error, key); + if (!descriptor) return undefined; + if (!hasOwn(descriptor, "value")) return INVALID_ERROR_FIELD; + const value = descriptor.value; + return typeof value === "string" ? value : INVALID_ERROR_FIELD; + } catch (_) { + return INVALID_ERROR_FIELD; + } +} + +function readNativeErrorMessage(error: Error): string { + const ownMessage = readOwnErrorString(error, "message"); + if (typeof ownMessage === "string") return ownMessage; + if (ownMessage === INVALID_ERROR_FIELD) return "Unknown error"; + return ""; +} + +function readNativeErrorStack(error: Error): string | undefined { + // Some engines materialize lazy stacks while producing their descriptor. + // The module-init behavior probe disables source stack inspection there. + if (!canInspectErrorStackDescriptorWithoutHooks) return undefined; + const ownStack = readOwnErrorString(error, "stack"); + return typeof ownStack === "string" ? ownStack : undefined; +} + +function primitiveErrorMessage(error: unknown): string { + if ( + (typeof error === "object" && error !== null) || + typeof error === "function" + ) { + return "Unknown error"; + } + try { + return NativeString(error); + } catch (_) { + return "Unknown error"; + } +} + +function createDataDescriptor(value: unknown): PropertyDescriptor { + const descriptor = createObject(null) as PropertyDescriptor; + descriptor.configurable = true; + descriptor.enumerable = false; + descriptor.value = value; + descriptor.writable = true; + return descriptor; +} + +function createErrorShapedRecord( + message: string, + name: string, + stack?: string, +): Error { + const sanitized = createObject(ERROR_PROTOTYPE) as Error; + defineProperty(sanitized, "message", createDataDescriptor(message)); + defineProperty(sanitized, "name", createDataDescriptor(name)); + defineProperty(sanitized, "stack", createDataDescriptor(stack)); + return sanitized; +} + +function createDetachedTelemetryError( + message: string, + name: string, + stack?: string, +): Error { + const sanitized = new NativeError(); + // Deleting V8's configurable lazy stack does not materialize it. Redefining + // the property directly does on older V8 releases and can therefore execute + // Error.prepareStackTrace. + if (!deleteProperty(sanitized, "stack")) { + return createErrorShapedRecord(message, name, stack); + } + defineProperty(sanitized, "message", createDataDescriptor(message)); + defineProperty(sanitized, "name", createDataDescriptor(name)); + defineProperty(sanitized, "stack", createDataDescriptor(stack)); + return sanitized; +} + +export type TelemetryAttributeValue = + | string + | number + | boolean + | readonly (string | number | boolean)[] + | undefined; + +const SEMANTIC_TOKEN_COUNT_ATTRIBUTE = + /(?:^|[._-])(?:input|output|total|prompt|completion)[._-]?tokens?$/i; + +function isNumericSemanticTokenCount(key: string, value: TelemetryAttributeValue): boolean { + return typeof value === "number" && Number.isFinite(value) && + SEMANTIC_TOKEN_COUNT_ATTRIBUTE.test(key); +} + +/** Redact and bound text before retaining it or handing it to a provider. */ +export function sanitizeTelemetryText(value: string, maxLength: number): string { + const sanitized = sanitizeUrlCredentials(value); + if (sanitized.length <= maxLength) return sanitized; + const end = apply(mathMax, Math, [0, maxLength - 1]) as number; + return `${apply(stringSlice, sanitized, [0, end]) as string}…`; +} + +/** Redact a single flattened telemetry attribute. */ +export function sanitizeTelemetryAttributeValue( + key: string, + value: TelemetryAttributeValue, +): TelemetryAttributeValue { + if (isSensitiveKey(key) && !isNumericSemanticTokenCount(key, value)) return REDACTED; + if (typeof value === "string") { + return sanitizeTelemetryText(value, MAX_TRACE_ATTRIBUTE_VALUE_SIZE); + } + if (typeof value === "number" && !Number.isFinite(value)) return undefined; + if (Array.isArray(value)) { + try { + if (value.length > MAX_TELEMETRY_ATTRIBUTE_ARRAY_LENGTH) return REDACTED; + const sanitized: (string | number | boolean)[] = []; + for (let index = 0; index < value.length; index++) { + const item = value[index]; + if (typeof item === "number" && !Number.isFinite(item)) return REDACTED; + sanitized.push( + typeof item === "string" + ? sanitizeTelemetryText(item, MAX_TRACE_ATTRIBUTE_VALUE_SIZE) + : item, + ); + } + return sanitized; + } catch (_) { + return REDACTED; + } + } + return value; +} + +/** Return a redacted copy of a flattened telemetry attribute record. */ +export function sanitizeTelemetryAttributes< + T extends Record | undefined, +>(attributes: T): T { + if (!attributes) return attributes; + + let keys: string[]; + try { + keys = objectKeys(attributes); + } catch (_) { + return {} as T; + } + + const sanitized: Record = {}; + const retainedKeys = new Set(); + const keyCount = keys.length < MAX_TELEMETRY_ATTRIBUTE_COUNT + ? keys.length + : MAX_TELEMETRY_ATTRIBUTE_COUNT; + for (let index = 0; index < keyCount; index++) { + const key = keys[index]; + if (key === undefined) continue; + const boundedKey = key.length <= MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH + ? key + : apply(stringSlice, key, [0, MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH]) as string; + if (!boundedKey || retainedKeys.has(boundedKey)) continue; + + let value: TelemetryAttributeValue = REDACTED; + if (!isSensitiveKey(key) || SEMANTIC_TOKEN_COUNT_ATTRIBUTE.test(key)) { + try { + value = sanitizeTelemetryAttributeValue(key, attributes[key]); + } catch (_) { + value = REDACTED; + } + } + if (value === undefined) continue; + defineProperty(sanitized, boundedKey, { + configurable: true, + enumerable: true, + value, + writable: true, + }); + retainedKeys.add(boundedKey); + } + return sanitized as T; +} + +interface StructuredTelemetryBudget { + exhausted: boolean; + remainingNodes: number; +} + +function cloneNativeDate(value: object): Date | undefined { + try { + const timestamp = apply(dateGetTime, value, []) as number; + return new NativeDate(timestamp); + } catch (_) { + return undefined; + } +} + +const NOT_NATIVE_URL = Symbol("not-native-url"); + +function cloneNativeUrl(value: object): URL | string | typeof NOT_NATIVE_URL { + if (!URL_HREF_GETTER) return NOT_NATIVE_URL; + let href: unknown; + try { + href = apply(URL_HREF_GETTER, value, []); + } catch (_) { + return NOT_NATIVE_URL; + } + if (typeof href !== "string") return REDACTED; + try { + const sanitizedHref = sanitizeUrlCredentials(href); + if (sanitizedHref.length > MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH) return REDACTED; + return new NativeURL(sanitizedHref); + } catch (_) { + return REDACTED; + } +} + +function snapshotStructuredError(value: Error): Record { + const snapshot = sanitizeErrorForTelemetry(value); + return { + message: snapshot.message, + name: snapshot.name, + stack: snapshot.stack, + }; +} + +function sanitizeStructuredValue( + value: unknown, + depth: number, + seen: Set, + budget: StructuredTelemetryBudget, +): unknown { + if (budget.remainingNodes <= 0) { + budget.exhausted = true; + return REDACTED; + } + budget.remainingNodes--; + + if (typeof value === "string") { + return sanitizeTelemetryText(value, MAX_STRING_DISPLAY_LENGTH); + } + if ( + value === null || value === undefined || typeof value === "number" || + typeof value === "boolean" || typeof value === "bigint" + ) { + return value; + } + if (typeof value === "symbol" || typeof value === "function") return REDACTED; + if (depth >= MAX_STRUCTURED_TELEMETRY_DEPTH || seen.has(value)) return REDACTED; + if (isProxyWithoutHooks(value)) return REDACTED; + + if (isNativeErrorWithoutHooks(value)) return snapshotStructuredError(value); + const clonedDate = cloneNativeDate(value); + if (clonedDate) return clonedDate; + const clonedUrl = cloneNativeUrl(value); + if (clonedUrl !== NOT_NATIVE_URL) return clonedUrl; + + seen.add(value); + try { + let toJSON: unknown; + try { + toJSON = (value as { toJSON?: unknown }).toJSON; + } catch (_) { + return REDACTED; + } + if (typeof toJSON === "function") { + try { + return sanitizeStructuredValue( + toJSON.call(value), + depth + 1, + seen, + budget, + ); + } catch (_) { + return REDACTED; + } + } + + if (Array.isArray(value)) { + if (value.length > MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES) { + return REDACTED; + } + const copy: unknown[] = []; + for (let index = 0; index < value.length; index++) { + try { + copy.push(sanitizeStructuredValue(value[index], depth + 1, seen, budget)); + } catch (_) { + copy.push(REDACTED); + } + if (budget.exhausted) return REDACTED; + } + return copy; + } + + let keys: string[]; + try { + keys = objectKeys(value); + } catch (_) { + return REDACTED; + } + if (keys.length > MAX_STRUCTURED_TELEMETRY_CONTAINER_ENTRIES) { + return REDACTED; + } + + const copy: Record = {}; + const retainedKeys = new Set(); + for (const key of keys) { + const boundedKey = sanitizeTelemetryText(key, LOG_PREVIEW_MAX_LENGTH_CHARS); + if (retainedKeys.has(boundedKey)) return REDACTED; + let child: unknown = REDACTED; + if (!isSensitiveKey(key)) { + try { + child = sanitizeStructuredValue( + (value as Record)[key], + depth + 1, + seen, + budget, + ); + } catch (_) { + child = REDACTED; + } + } + if (budget.exhausted) return REDACTED; + defineProperty(copy, boundedKey, { + configurable: true, + enumerable: true, + value: child, + writable: true, + }); + retainedKeys.add(boundedKey); + } + return copy; + } finally { + seen.delete(value); + } +} + +/** + * Return a detached, fail-closed snapshot suitable for retained logs and + * errors. Credential-like keys and URL credentials are redacted recursively. + */ +export function sanitizeStructuredTelemetryData(value: T): T { + try { + return sanitizeStructuredValue( + value, + 0, + new Set(), + { + exhausted: false, + remainingNodes: MAX_STRUCTURED_TELEMETRY_NODES, + }, + ) as T; + } catch (_) { + return REDACTED as T; + } +} + +/** + * Create an error safe to send to telemetry backends without mutating or + * replacing the application error that will be returned to the caller. + * + * Native errors are classified through a hook-free runtime brand check. Older + * supported runtimes use the platform compatibility implementation instead of + * the unsafe `instanceof` fallback that executes Proxy traps. + */ +export function sanitizeErrorForTelemetry(error: unknown): Error { + try { + const isError = isNativeErrorWithoutHooks(error); + const source = isError ? error : undefined; + const message = sanitizeTelemetryText( + source ? readNativeErrorMessage(source) : primitiveErrorMessage(error), + MAX_STRING_DISPLAY_LENGTH, + ); + const name = source + ? sanitizeTelemetryText( + readNativeErrorNameWithoutHooks(source), + LOG_PREVIEW_MAX_LENGTH_CHARS, + ) + : "Unknown"; + const sourceStack = source ? readNativeErrorStack(source) : undefined; + const stack = sourceStack === undefined + ? undefined + : sanitizeTelemetryText(sourceStack, MAX_STRING_DISPLAY_LENGTH); + + return createDetachedTelemetryError(message, name, stack); + } catch (_) { + // Telemetry is best effort and must never replace the application outcome. + return createErrorShapedRecord("Unknown error", "Unknown"); + } +} diff --git a/src/observability/tracing/api-shim.test.ts b/src/observability/tracing/api-shim.test.ts index fb99793e18..43155a36cc 100644 --- a/src/observability/tracing/api-shim.test.ts +++ b/src/observability/tracing/api-shim.test.ts @@ -8,8 +8,12 @@ import { defaultTextMapGetter, defaultTextMapSetter, getGlobalMetricsAPI, + getGlobalTelemetryAPISnapshot, getGlobalTracerProvider, + getMetricsApiRevision, getTracer, + getTracerProviderRevision, + installGlobalTelemetryAPI, type MetricsAPI, propagation, setGlobalActiveSpanAccessor, @@ -105,6 +109,77 @@ describe("observability/tracing/api-shim", () => { }); }); + describe("atomic global telemetry API", () => { + it("installs and clears one complete provider generation atomically", () => { + const tracerProvider = { getTracer: () => getTracer("fallback") }; + const metricsApi = { getMeter: () => ({}) } as unknown as MetricsAPI; + const owner = installGlobalTelemetryAPI({ tracerProvider, metricsApi }); + + const installed = getGlobalTelemetryAPISnapshot(); + assertEquals(installed.generation, owner.generation); + assertEquals(installed.tracerProvider, tracerProvider); + assertEquals(installed.metricsApi, metricsApi); + + assertEquals(owner.dispose(), true); + assertEquals(owner.dispose(), false); + const cleared = getGlobalTelemetryAPISnapshot(); + assertEquals(cleared.generation > owner.generation, true); + assertEquals(cleared.metricsApi, null); + assertEquals( + cleared.tracerProvider.getTracer("cleared").startSpan("span").spanContext().traceId, + "00000000000000000000000000000000", + ); + }); + + it("increments both revisions for every explicit install of stable facades", () => { + const tracerProvider = { getTracer: () => getTracer("fallback") }; + const metricsApi = { getMeter: () => ({}) } as unknown as MetricsAPI; + const tracerRevision = getTracerProviderRevision(); + const metricsRevision = getMetricsApiRevision(); + + installGlobalTelemetryAPI({ tracerProvider, metricsApi }); + installGlobalTelemetryAPI({ tracerProvider, metricsApi }); + + assertEquals(getTracerProviderRevision(), tracerRevision + 2); + assertEquals(getMetricsApiRevision(), metricsRevision + 2); + }); + + it("does not let a stale owner clear a newer provider generation", () => { + const first = installGlobalTelemetryAPI({}); + const secondProvider = { getTracer: () => getTracer("fallback") }; + const second = installGlobalTelemetryAPI({ tracerProvider: secondProvider }); + + assertEquals(first.dispose(), false); + assertEquals(getGlobalTracerProvider(), secondProvider); + assertEquals(getGlobalTelemetryAPISnapshot().generation, second.generation); + }); + + it("pre-reads the complete install input before changing global state", () => { + const original = getGlobalTelemetryAPISnapshot(); + const input = { + get tracerProvider(): TracerProvider { + return { getTracer: () => getTracer("fallback") }; + }, + get metricsApi(): MetricsAPI { + throw new Error("hostile metrics getter"); + }, + }; + + let threw = false; + try { + installGlobalTelemetryAPI(input); + } catch { + threw = true; + } + + assertEquals(threw, true); + const after = getGlobalTelemetryAPISnapshot(); + assertEquals(after.generation, original.generation); + assertEquals(after.tracerProvider, original.tracerProvider); + assertEquals(after.metricsApi, original.metricsApi); + }); + }); + describe("active-span accessor", () => { it("returns real spans once an accessor is wired", () => { const real = { updateName() {} } as unknown as Span; @@ -149,6 +224,34 @@ describe("observability/tracing/api-shim", () => { return ctx; } + it("derives immutable fallback contexts", () => { + const key = Symbol("immutable-context"); + const base = context.active(); + const derived = base.setValue(key, "scoped"); + + assertEquals(derived === base, false); + assertEquals(base.getValue(key), undefined); + assertEquals(derived.getValue(key), "scoped"); + + const deleted = derived.deleteValue(key); + assertEquals(deleted === derived, false); + assertEquals(derived.getValue(key), "scoped"); + assertEquals(deleted.getValue(key), undefined); + }); + + it("reset installs a fresh fallback context without prior spans", () => { + const span = { updateName() {} } as unknown as Span; + const beforeReset = context.active(); + const scoped = trace.setSpan(beforeReset, span); + assertEquals(trace.getSpan(scoped), span); + + _resetShimForTests(); + + const afterReset = context.active(); + assertEquals(afterReset === beforeReset, false); + assertEquals(trace.getSpan(afterReset), undefined); + }); + it("context.with sets the active context for the duration of fn then restores it", () => { const base = context.active(); const scoped = makeScopedContext(); @@ -180,16 +283,95 @@ describe("observability/tracing/api-shim", () => { assertEquals(context.active() === base, true); }); - it("context.with keeps the active context until an async callback settles", async () => { + it("keeps the fallback context active across the awaits it wraps", async () => { const base = context.active(); const scoped = makeScopedContext(); + let synchronousContext: Context | undefined; + + const applicationPromise = context.with(scoped, async () => { + synchronousContext = context.active(); + await Promise.resolve(); + return context.active(); + }); + + assertEquals(synchronousContext === scoped, true); + // The activation stays scoped to its own async work, so ambient reads + // outside it are unaffected while it runs. + assertEquals(context.active() === base, true); + assertEquals(await applicationPromise === scoped, true); + }); + + it("isolates concurrent fallback activations from each other", async () => { + const first = makeScopedContext(); + const second = makeScopedContext(); + let releaseFirst!: () => void; + const firstGate = new Promise((resolve) => { + releaseFirst = resolve; + }); - const result = await context.with(scoped, async () => { + const firstScope = context.with(first, async () => { + await firstGate; + return context.active(); + }); + const secondScope = context.with(second, async () => { await Promise.resolve(); - return context.active() === scoped; + return context.active(); + }); + + assertEquals(await secondScope === second, true); + releaseFirst(); + assertEquals(await firstScope === first, true); + }); + + it("does not restore a stale span context when an async scope settles after reset", async () => { + const staleSpan = { updateName() {} } as unknown as Span; + const staleContext = trace.setSpan(context.active(), staleSpan); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); + const pending = context.with(staleContext, async () => { + await gate; + }); + + _resetShimForTests(); + const freshContext = context.active(); + release(); + await pending; + + assertEquals(context.active() === freshContext, true); + assertEquals(trace.getActiveSpan(), undefined); + }); + + it("does not cross-contaminate overlapping fallback async scopes", async () => { + const base = context.active(); + const first = makeScopedContext(); + const second = makeScopedContext(); + let releaseFirst!: () => void; + let releaseSecond!: () => void; + const firstGate = new Promise((resolve) => { + releaseFirst = resolve; + }); + const secondGate = new Promise((resolve) => { + releaseSecond = resolve; + }); + + const firstScope = context.with(first, () => { + assertEquals(context.active() === first, true); + return firstGate; + }); + assertEquals(context.active() === base, true); + const secondScope = context.with(second, () => { + assertEquals(context.active() === second, true); + return secondGate; + }); + assertEquals(context.active() === base, true); + releaseFirst(); + await firstScope; + assertEquals(context.active() === base, true); - assertEquals(result, true); + releaseSecond(); + await secondScope; assertEquals(context.active() === base, true); }); diff --git a/src/observability/tracing/api-shim.ts b/src/observability/tracing/api-shim.ts index dbe8e15cae..23c6bd04d4 100644 --- a/src/observability/tracing/api-shim.ts +++ b/src/observability/tracing/api-shim.ts @@ -19,6 +19,9 @@ * @module observability/tracing/api-shim */ +import { AsyncLocalStorage } from "node:async_hooks"; +import { runSyncWithContextFallback } from "./context-callback.ts"; + // --------------------------------------------------------------------------- // Tracing types // --------------------------------------------------------------------------- @@ -156,6 +159,7 @@ export interface Histogram { export interface ObservableGauge { addCallback(callback: (result: ObservableResult) => void): void; + removeCallback?(callback: (result: ObservableResult) => void): void; } export interface Meter { @@ -187,22 +191,25 @@ export interface MetricsAPI { // No-op provider (default when ext-observability-opentelemetry is not installed) // --------------------------------------------------------------------------- -function createNoopContext(): Context { - const store = new Map(); - return { - getValue: (key) => store.get(key), - setValue(key, value) { - store.set(key, value); - return this; +function createNoopContext( + entries: ReadonlyMap = new Map(), +): Context { + const store = new Map(entries); + return Object.freeze({ + getValue: (key: symbol) => store.get(key), + setValue(key: symbol, value: unknown) { + const next = new Map(store); + next.set(key, value); + return createNoopContext(next); }, - deleteValue(key) { - store.delete(key); - return this; + deleteValue(key: symbol) { + const next = new Map(store); + next.delete(key); + return createNoopContext(next); }, - }; + }); } -const NOOP_CONTEXT: Context = createNoopContext(); const EMPTY_TRACE_ID = "00000000000000000000000000000000"; const EMPTY_SPAN_ID = "0000000000000000"; @@ -265,19 +272,223 @@ function createNoopProvider(): TracerProvider { // Global provider state // --------------------------------------------------------------------------- -let _provider: TracerProvider = createNoopProvider(); -let _providerRevision = 0; -let _activeContext: Context = NOOP_CONTEXT; -let _propagator: TextMapPropagator | null = null; -let _contextAccessor: ContextAccessor | null = null; const ACTIVE_SPAN_CONTEXT_KEY = Symbol.for("veryfront.observability.active_span"); +/** Complete provider facade installed into the process-wide telemetry shim. */ +export interface GlobalTelemetryAPIConfig { + tracerProvider?: TracerProvider | null; + metricsApi?: MetricsAPI | null; + contextAccessor?: ContextAccessor | null; + activeSpanAccessor?: ActiveSpanAccessor | null; + propagator?: TextMapPropagator | null; +} + +/** Opaque ownership identity for one installed telemetry generation. */ +export interface GlobalTelemetryAPIOwner { + readonly generation: number; + readonly token: symbol; +} + +/** Handle returned by an atomic telemetry API installation. */ +export interface GlobalTelemetryAPIInstallation extends GlobalTelemetryAPIOwner { + /** Clear this generation if it is still current. Stale handles return false. */ + dispose(): boolean; +} + +/** Immutable point-in-time view of the currently installed telemetry facade. */ +export interface GlobalTelemetryAPISnapshot { + readonly generation: number; + readonly tracerProviderRevision: number; + readonly metricsApiRevision: number; + readonly tracerProviderInstalled: boolean; + readonly tracerProvider: TracerProvider; + readonly metricsApi: MetricsAPI | null; + readonly contextAccessor: ContextAccessor | null; + readonly activeSpanAccessor: ActiveSpanAccessor | null; + readonly propagator: TextMapPropagator | null; +} + +interface GlobalTelemetryAPIState extends GlobalTelemetryAPISnapshot { + readonly ownerToken: symbol; +} + +function createEmptyTelemetryState( + generation = 0, + tracerProviderRevision = 0, + metricsApiRevision = 0, +): GlobalTelemetryAPIState { + return Object.freeze({ + generation, + tracerProviderRevision, + metricsApiRevision, + tracerProviderInstalled: false, + tracerProvider: createNoopProvider(), + metricsApi: null, + contextAccessor: null, + activeSpanAccessor: null, + propagator: null, + ownerToken: Symbol("veryfront.telemetry.empty"), + }); +} + +let telemetryState = createEmptyTelemetryState(); + /** - * Optional accessor for the currently active span. Wired by - * ext-observability-opentelemetry (via `setGlobalActiveSpanAccessor`) so `trace.getActiveSpan()` - * and `trace.getSpan()` return the real SDK span once the extension is active. + * Async-scoped fallback context used until a real context accessor is installed. + * + * Async-local storage keeps an activation visible across the awaits of the work + * it wraps without leaking into concurrent work, so span attributes recorded + * after an await still reach the span that is active for that request. */ -let _activeSpanAccessor: ActiveSpanAccessor | null = null; +const fallbackContextStorage = new AsyncLocalStorage(); +let _rootContext: Context = createNoopContext(); + +function resetFallbackContext(): void { + _rootContext = createNoopContext(); +} + +function getFallbackContext(): Context { + try { + return fallbackContextStorage.getStore() ?? _rootContext; + } catch (_) { + return _rootContext; + } +} + +function assertOptionalMethod( + value: object | null | undefined, + method: string, + label: string, +): void { + if (value === null || value === undefined) return; + if (typeof (value as Record)[method] !== "function") { + throw new TypeError(`${label} must implement ${method}()`); + } +} + +function preReadTelemetryConfig(config: GlobalTelemetryAPIConfig): { + tracerProvider: TracerProvider; + tracerProviderInstalled: boolean; + metricsApi: MetricsAPI | null; + contextAccessor: ContextAccessor | null; + activeSpanAccessor: ActiveSpanAccessor | null; + propagator: TextMapPropagator | null; +} { + // Read every potentially accessor-backed property before mutating global + // state. A hostile or partially constructed facade therefore cannot leave + // the shim half-installed. + const suppliedTracerProvider = config.tracerProvider; + const metricsApi = config.metricsApi ?? null; + const contextAccessor = config.contextAccessor ?? null; + const activeSpanAccessor = config.activeSpanAccessor ?? null; + const propagator = config.propagator ?? null; + + assertOptionalMethod(suppliedTracerProvider, "getTracer", "tracerProvider"); + assertOptionalMethod(metricsApi, "getMeter", "metricsApi"); + assertOptionalMethod(contextAccessor, "active", "contextAccessor"); + assertOptionalMethod(contextAccessor, "with", "contextAccessor"); + assertOptionalMethod(activeSpanAccessor, "getActiveSpan", "activeSpanAccessor"); + assertOptionalMethod(activeSpanAccessor, "getSpan", "activeSpanAccessor"); + assertOptionalMethod(propagator, "inject", "propagator"); + assertOptionalMethod(propagator, "extract", "propagator"); + + return { + tracerProvider: suppliedTracerProvider ?? createNoopProvider(), + tracerProviderInstalled: suppliedTracerProvider !== null && + suppliedTracerProvider !== undefined, + metricsApi, + contextAccessor, + activeSpanAccessor, + propagator, + }; +} + +function installTelemetryState( + config: ReturnType, +): GlobalTelemetryAPIInstallation { + const previous = telemetryState; + const generation = previous.generation + 1; + const token = Symbol(`veryfront.telemetry.${generation}`); + telemetryState = Object.freeze({ + generation, + tracerProviderRevision: previous.tracerProviderRevision + 1, + metricsApiRevision: previous.metricsApiRevision + 1, + ...config, + ownerToken: token, + }); + resetFallbackContext(); + + return Object.freeze({ + generation, + token, + dispose: () => clearGlobalTelemetryAPI({ generation, token }), + }); +} + +/** Atomically install one complete telemetry facade and return its owner handle. */ +export function installGlobalTelemetryAPI( + config: GlobalTelemetryAPIConfig, +): GlobalTelemetryAPIInstallation { + return installTelemetryState(preReadTelemetryConfig(config)); +} + +/** Clear the current generation without allowing stale owners to clobber it. */ +export function clearGlobalTelemetryAPI(owner: GlobalTelemetryAPIOwner): boolean { + const current = telemetryState; + if (owner.generation !== current.generation || owner.token !== current.ownerToken) { + return false; + } + + telemetryState = createEmptyTelemetryState( + current.generation + 1, + current.tracerProviderRevision + 1, + current.metricsApiRevision + 1, + ); + resetFallbackContext(); + return true; +} + +/** Read one internally consistent telemetry facade snapshot. */ +export function getGlobalTelemetryAPISnapshot(): GlobalTelemetryAPISnapshot { + const current = telemetryState; + return Object.freeze({ + generation: current.generation, + tracerProviderRevision: current.tracerProviderRevision, + metricsApiRevision: current.metricsApiRevision, + tracerProviderInstalled: current.tracerProviderInstalled, + tracerProvider: current.tracerProvider, + metricsApi: current.metricsApi, + contextAccessor: current.contextAccessor, + activeSpanAccessor: current.activeSpanAccessor, + propagator: current.propagator, + }); +} + +function updateTelemetryState( + patch: Partial< + Pick< + GlobalTelemetryAPIState, + | "tracerProvider" + | "tracerProviderInstalled" + | "metricsApi" + | "contextAccessor" + | "activeSpanAccessor" + | "propagator" + > + >, + revisions: { tracer?: boolean; metrics?: boolean } = {}, +): void { + const current = telemetryState; + telemetryState = Object.freeze({ + ...current, + ...patch, + generation: current.generation + 1, + tracerProviderRevision: current.tracerProviderRevision + (revisions.tracer ? 1 : 0), + metricsApiRevision: current.metricsApiRevision + (revisions.metrics ? 1 : 0), + ownerToken: Symbol("veryfront.telemetry.legacy-install"), + }); + resetFallbackContext(); +} /** * Register the real OTel trace API's span accessors. Called by the @@ -287,7 +498,7 @@ let _activeSpanAccessor: ActiveSpanAccessor | null = null; export function setGlobalActiveSpanAccessor( accessor: ActiveSpanAccessor, ): void { - _activeSpanAccessor = accessor; + updateTelemetryState({ activeSpanAccessor: accessor }); } /** @@ -296,7 +507,7 @@ export function setGlobalActiveSpanAccessor( * AsyncLocalStorageContextManager. */ export function setGlobalContextAccessor(accessor: ContextAccessor): void { - _contextAccessor = accessor; + updateTelemetryState({ contextAccessor: accessor }); } /** @@ -304,16 +515,15 @@ export function setGlobalContextAccessor(accessor: ContextAccessor): void { * Called from `src/server/bootstrap.ts` after `orchestrateExtensions()` runs. */ export function setGlobalTracerProvider(p: TracerProvider): void { - _provider = p; - _providerRevision++; + updateTelemetryState({ tracerProvider: p, tracerProviderInstalled: true }, { tracer: true }); } export function getGlobalTracerProvider(): TracerProvider { - return _provider; + return telemetryState.tracerProvider; } export function getTracerProviderRevision(): number { - return _providerRevision; + return telemetryState.tracerProviderRevision; } /** @@ -321,7 +531,7 @@ export function getTracerProviderRevision(): number { * Returns the no-op tracer when ext-observability-opentelemetry is not installed. */ export function getTracer(name: string, version?: string): Tracer { - return _provider.getTracer(name, version); + return telemetryState.tracerProvider.getTracer(name, version); } // --------------------------------------------------------------------------- @@ -330,28 +540,24 @@ export function getTracer(name: string, version?: string): Tracer { export const context = { active(): Context { - return _contextAccessor?.active() ?? _activeContext; + try { + return telemetryState.contextAccessor?.active() ?? getFallbackContext(); + } catch (_) { + return getFallbackContext(); + } }, with(ctx: Context, fn: () => T): T { - if (_contextAccessor) { - return _contextAccessor.with(ctx, fn); + const accessor = telemetryState.contextAccessor; + if (accessor) { + return runSyncWithContextFallback( + (callback) => accessor.with(ctx, callback), + fn, + ); } - const prev = _activeContext; - _activeContext = ctx; - try { - const result = fn(); - if (result && typeof (result as { finally?: unknown }).finally === "function") { - return (result as unknown as Promise).finally(() => { - _activeContext = prev; - }) as T; - } - _activeContext = prev; - return result; - } catch (error) { - _activeContext = prev; - throw error; - } + // run() invokes fn exactly once and propagates its result or failure + // unchanged, so application outcomes never depend on activation. + return fallbackContextStorage.run(ctx, fn); }, setGlobalContextManager(_mgr: unknown): void { // no-op in shim; real SDK sets this via the real OTel API @@ -364,14 +570,13 @@ export const context = { export const trace = { getTracer(name: string, version?: string): Tracer { - return _provider.getTracer(name, version); + return telemetryState.tracerProvider.getTracer(name, version); }, setGlobalTracerProvider(p: TracerProvider): void { - _provider = p; - _providerRevision++; + setGlobalTracerProvider(p); }, getGlobalTracerProvider(): TracerProvider { - return _provider; + return telemetryState.tracerProvider; }, setSpan(ctx: Context, _span: Span): Context { try { @@ -382,17 +587,28 @@ export const trace = { } catch { // Keep structural test doubles usable even when they omit spanContext(). } - if (_activeSpanAccessor?.setSpan) { - return _activeSpanAccessor.setSpan(ctx, _span); + try { + const accessor = telemetryState.activeSpanAccessor; + if (accessor?.setSpan) return accessor.setSpan(ctx, _span); + return ctx.setValue(ACTIVE_SPAN_CONTEXT_KEY, _span); + } catch (_) { + return ctx; } - return ctx.setValue(ACTIVE_SPAN_CONTEXT_KEY, _span); }, getSpan(ctx: Context): Span | undefined { - return _activeSpanAccessor?.getSpan(ctx) ?? - (ctx.getValue(ACTIVE_SPAN_CONTEXT_KEY) as Span | undefined); + try { + return telemetryState.activeSpanAccessor?.getSpan(ctx) ?? + (ctx.getValue(ACTIVE_SPAN_CONTEXT_KEY) as Span | undefined); + } catch (_) { + return undefined; + } }, getActiveSpan(): Span | undefined { - return _activeSpanAccessor?.getActiveSpan() ?? trace.getSpan(context.active()); + try { + return telemetryState.activeSpanAccessor?.getActiveSpan() ?? trace.getSpan(context.active()); + } catch (_) { + return undefined; + } }, }; @@ -402,15 +618,17 @@ export const trace = { export const propagation = { setGlobalPropagator(p: TextMapPropagator): void { - _propagator = p; + updateTelemetryState({ propagator: p }); }, extract(ctx: Context, carrier: C, getter?: TextMapGetter): Context { - if (!_propagator) return ctx; - return _propagator.extract(ctx, carrier, getter as TextMapGetter | undefined); + const propagator = telemetryState.propagator; + if (!propagator) return ctx; + return propagator.extract(ctx, carrier, getter as TextMapGetter | undefined); }, inject(ctx: Context, carrier: C, setter?: TextMapSetter): void { - if (!_propagator) return; - _propagator.inject(ctx, carrier, setter as TextMapSetter | undefined); + const propagator = telemetryState.propagator; + if (!propagator) return; + propagator.inject(ctx, carrier, setter as TextMapSetter | undefined); }, }; @@ -437,19 +655,22 @@ export const defaultTextMapSetter: TextMapSetter> = { // Metrics API registry (injected by ext-observability-opentelemetry when active) // --------------------------------------------------------------------------- -let _metricsApi: MetricsAPI | null = null; - /** * Register the OTel Metrics API (from the SDK). * Called by ext-observability-opentelemetry in its setup hook so the metrics subsystem * can use `getMeter()` when available. */ export function setGlobalMetricsAPI(api: MetricsAPI): void { - _metricsApi = api; + updateTelemetryState({ metricsApi: api }, { metrics: true }); } export function getGlobalMetricsAPI(): MetricsAPI | null { - return _metricsApi; + return telemetryState.metricsApi; +} + +/** Monotonic revision used by lazy instruments to detect provider changes. */ +export function getMetricsApiRevision(): number { + return telemetryState.metricsApiRevision; } // --------------------------------------------------------------------------- @@ -457,11 +678,11 @@ export function getGlobalMetricsAPI(): MetricsAPI | null { // --------------------------------------------------------------------------- export function _resetShimForTests(): void { - _provider = createNoopProvider(); - _providerRevision++; - _activeContext = NOOP_CONTEXT; - _propagator = null; - _contextAccessor = null; - _metricsApi = null; - _activeSpanAccessor = null; + const current = telemetryState; + telemetryState = createEmptyTelemetryState( + current.generation + 1, + current.tracerProviderRevision + 1, + current.metricsApiRevision + 1, + ); + resetFallbackContext(); } diff --git a/src/observability/tracing/config.test.ts b/src/observability/tracing/config.test.ts index 241b47b885..2c25944015 100644 --- a/src/observability/tracing/config.test.ts +++ b/src/observability/tracing/config.test.ts @@ -1,5 +1,5 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertThrows } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { loadConfig } from "./config.ts"; @@ -44,6 +44,17 @@ describe("observability/tracing/config", () => { assertEquals(result.exporter, "otlp"); }); + it("gives the signal-specific endpoint precedence over the generic endpoint", () => { + const vars: Record = { + OTEL_EXPORTER_OTLP_ENDPOINT: "http://generic:4318", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://traces:4318/v1/traces", + }; + + const result = loadConfig({}, createAdapter((key) => vars[key])); + + assertEquals(result.endpoint, "http://traces:4318/v1/traces"); + }); + it("should enable via VERYFRONT_OTEL=1", () => { const result = loadConfig( {}, @@ -59,5 +70,40 @@ describe("observability/tracing/config", () => { ); assertEquals(result.exporter, "console"); }); + + it("rejects malformed caller configuration instead of enabling truthy values", () => { + assertThrows( + () => loadConfig({ enabled: "yes" } as never, emptyEnvAdapter), + TypeError, + "enabled", + ); + assertThrows( + () => loadConfig({ exporter: "invalid" } as never, emptyEnvAdapter), + TypeError, + "exporter", + ); + assertThrows( + () => loadConfig({ sampleRate: 2 }, emptyEnvAdapter), + RangeError, + "sampleRate", + ); + assertThrows( + () => loadConfig({ serviceName: " " }, emptyEnvAdapter), + TypeError, + "serviceName", + ); + }); + + it("contains adapter environment failures without consulting another environment", () => { + const result = loadConfig( + { enabled: false, serviceName: "configured" }, + createAdapter(() => { + throw new Error("environment unavailable"); + }), + ); + + assertEquals(result.enabled, false); + assertEquals(result.serviceName, "configured"); + }); }); }); diff --git a/src/observability/tracing/config.ts b/src/observability/tracing/config.ts index de947e1a51..a75aa56c87 100644 --- a/src/observability/tracing/config.ts +++ b/src/observability/tracing/config.ts @@ -1,20 +1,21 @@ import type { RuntimeAdapter } from "#veryfront/platform/adapters/base.ts"; import { getOtelTracingConfig } from "#veryfront/config/env.ts"; import type { TracingConfig } from "./types.ts"; +import { MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, MAX_OBSERVABILITY_NAME_LENGTH } from "../limits.ts"; -const DEFAULT_CONFIG: TracingConfig = { +export const DEFAULT_CONFIG: Readonly = Object.freeze({ enabled: false, exporter: "console", serviceName: "veryfront", sampleRate: 1.0, debug: false, -}; +}); export function loadConfig( config: Partial = {}, adapter?: RuntimeAdapter, ): TracingConfig { - const finalConfig: TracingConfig = { ...DEFAULT_CONFIG, ...config }; + const finalConfig = normalizeConfig(config); const envAdapter = adapter?.env; if (envAdapter) { @@ -30,43 +31,144 @@ function applyEnvFromAdapter( config: TracingConfig, envAdapter: RuntimeAdapter["env"], ): void { - config.enabled = envAdapter.get("OTEL_TRACES_ENABLED") === "true" || - envAdapter.get("VERYFRONT_OTEL") === "1" || - config.enabled; - - config.serviceName = envAdapter.get("OTEL_SERVICE_NAME") ?? config.serviceName; - - config.endpoint = envAdapter.get("OTEL_EXPORTER_OTLP_ENDPOINT") ?? - envAdapter.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") ?? - config.endpoint; - - const exporterType = envAdapter.get("OTEL_TRACES_EXPORTER"); - if (isValidExporter(exporterType)) config.exporter = exporterType; + try { + applyEnvValues(config, { + enabledFlag: envAdapter.get("OTEL_TRACES_ENABLED"), + veryfrontFlag: envAdapter.get("VERYFRONT_OTEL"), + serviceName: envAdapter.get("OTEL_SERVICE_NAME"), + endpoint: envAdapter.get("OTEL_EXPORTER_OTLP_ENDPOINT"), + signalEndpoint: envAdapter.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), + exporter: envAdapter.get("OTEL_TRACES_EXPORTER"), + }); + } catch { + // An explicit adapter owns this environment boundary. If it is + // unavailable, preserve caller configuration rather than consulting a + // different host environment. + } } function applyEnvFromDeno(config: TracingConfig): void { try { const tracingConfig = getOtelTracingConfig(); + applyEnvValues(config, { + enabledFlag: tracingConfig.enabledFlag, + veryfrontFlag: tracingConfig.veryfrontFlag, + serviceName: tracingConfig.serviceName, + endpoint: tracingConfig.endpoint, + signalEndpoint: tracingConfig.tracesEndpoint, + exporter: tracingConfig.exporter, + }); + } catch (_) { + /* expected: environment access may fail in some runtimes */ + } +} - config.enabled = tracingConfig.enabledFlag === "true" || - tracingConfig.veryfrontFlag === "1" || - config.enabled; +function normalizeConfig(config: Partial): TracingConfig { + if (config === null || typeof config !== "object" || Array.isArray(config)) { + throw new TypeError("Tracing config must be an object"); + } + const raw = config as Record; + const enabled = raw.enabled ?? DEFAULT_CONFIG.enabled; + if (typeof enabled !== "boolean") { + throw new TypeError("Tracing enabled must be a boolean"); + } + const exporter = raw.exporter ?? DEFAULT_CONFIG.exporter; + if (!isValidExporter(exporter)) { + throw new TypeError("Tracing exporter is invalid"); + } + const serviceName = requireOptionalText( + raw.serviceName ?? DEFAULT_CONFIG.serviceName, + "serviceName", + MAX_OBSERVABILITY_NAME_LENGTH, + ); + const endpoint = raw.endpoint === undefined ? undefined : requireOptionalText( + raw.endpoint, + "endpoint", + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + const sampleRate = raw.sampleRate ?? DEFAULT_CONFIG.sampleRate; + if ( + typeof sampleRate !== "number" || !Number.isFinite(sampleRate) || + sampleRate < 0 || sampleRate > 1 + ) { + throw new RangeError("Tracing sampleRate must be a finite number between 0 and 1"); + } + const debug = raw.debug ?? DEFAULT_CONFIG.debug; + if (typeof debug !== "boolean") { + throw new TypeError("Tracing debug must be a boolean"); + } - config.serviceName = tracingConfig.serviceName ?? config.serviceName; + return { + enabled, + exporter, + serviceName, + sampleRate, + debug, + ...(endpoint ? { endpoint } : {}), + }; +} - config.endpoint = tracingConfig.endpoint ?? - tracingConfig.tracesEndpoint ?? - config.endpoint; +function applyEnvValues( + config: TracingConfig, + values: { + enabledFlag?: unknown; + veryfrontFlag?: unknown; + serviceName?: unknown; + endpoint?: unknown; + signalEndpoint?: unknown; + exporter?: unknown; + }, +): void { + config.enabled = isTrueEnvironmentValue(values.enabledFlag) || + values.veryfrontFlag === "1" || config.enabled; - const exporterType = tracingConfig.exporter; - if (isValidExporter(exporterType)) config.exporter = exporterType; - } catch (_) { - /* expected: environment access may fail in some runtimes */ + const serviceName = normalizeEnvironmentText( + values.serviceName, + MAX_OBSERVABILITY_NAME_LENGTH, + ); + if (serviceName) config.serviceName = serviceName; + + const signalEndpoint = normalizeEnvironmentText( + values.signalEndpoint, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + const endpoint = normalizeEnvironmentText( + values.endpoint, + MAX_OBSERVABILITY_CONFIG_TEXT_LENGTH, + ); + config.endpoint = signalEndpoint || endpoint || config.endpoint; + + const exporter = typeof values.exporter === "string" + ? values.exporter.trim().toLowerCase() + : undefined; + if (isValidExporter(exporter)) config.exporter = exporter; +} + +function isTrueEnvironmentValue(value: unknown): boolean { + return typeof value === "string" && value.trim().toLowerCase() === "true"; +} + +function requireOptionalText(value: unknown, name: string, maxLength: number): string { + if (typeof value !== "string") { + throw new TypeError(`Tracing ${name} must be a string`); + } + const normalized = value.trim(); + if (!normalized || normalized.length > maxLength) { + throw new TypeError( + `Tracing ${name} must contain between 1 and ${maxLength} characters`, + ); } + return normalized; +} + +function normalizeEnvironmentText(value: unknown, maxLength: number): string | undefined { + if (typeof value !== "string") return undefined; + const normalized = value.trim(); + return normalized && normalized.length <= maxLength ? normalized : undefined; } function isValidExporter( - value: string | undefined, + value: unknown, ): value is TracingConfig["exporter"] { return ( value === "jaeger" || diff --git a/src/observability/tracing/context-callback.ts b/src/observability/tracing/context-callback.ts new file mode 100644 index 0000000000..821dca7cc7 --- /dev/null +++ b/src/observability/tracing/context-callback.ts @@ -0,0 +1,125 @@ +type ActivationFailureHandler = (error: unknown) => void; + +function reportActivationFailure( + handler: ActivationFailureHandler | undefined, + error: unknown, +): void { + try { + handler?.(error); + } catch (_) { + /* expected: diagnostics must not affect the protected callback */ + } +} + +function consumeIgnoredThenable(value: unknown): void { + if ((typeof value !== "object" || value === null) && typeof value !== "function") return; + + let then: unknown; + try { + then = (value as { then?: unknown }).then; + } catch (_) { + return; + } + if (typeof then !== "function") return; + + try { + Reflect.apply(then, value, [() => {}, () => {}]); + } catch (_) { + /* expected: provider-owned thenables cannot affect application outcomes */ + } +} + +/** + * Run an async callback at most once, falling back when context activation fails. + * + * Context providers are synchronous callback invokers by contract. Once the + * application callback has been invoked, its promise is authoritative: a + * provider-owned replacement (including a promise that never settles) must not + * delay or replace application work. + */ +export function runAsyncWithContextFallback( + activate: (callback: () => Promise) => Promise, + callback: () => Promise, + onActivationFailure?: ActivationFailureHandler, +): Promise { + let callbackInvoked = false; + let callbackResult: Promise | undefined; + const invoke = (): Promise => { + if (callbackInvoked && callbackResult) return callbackResult; + callbackInvoked = true; + try { + callbackResult = callback(); + } catch (error) { + callbackResult = Promise.reject(error); + } + return callbackResult; + }; + + try { + const providerResult = activate(invoke); + if (callbackInvoked && callbackResult) { + if (providerResult !== callbackResult) consumeIgnoredThenable(providerResult); + return callbackResult; + } + + reportActivationFailure( + onActivationFailure, + new Error("Context activation returned without invoking its callback"), + ); + consumeIgnoredThenable(providerResult); + return invoke(); + } catch (activationError) { + if (callbackInvoked && callbackResult) return callbackResult; + reportActivationFailure(onActivationFailure, activationError); + return invoke(); + } +} + +/** Run a sync callback at most once, falling back when context activation fails. */ +export function runSyncWithContextFallback( + activate: (callback: () => T) => T, + callback: () => T, + onActivationFailure?: ActivationFailureHandler, +): T { + let callbackInvoked = false; + let callbackSucceeded = false; + let callbackResult: T | undefined; + let callbackError: unknown; + + const invoke = (): T => { + if (callbackInvoked) { + if (!callbackSucceeded) throw callbackError; + return callbackResult as T; + } + callbackInvoked = true; + try { + callbackResult = callback(); + callbackSucceeded = true; + return callbackResult; + } catch (error) { + callbackError = error; + throw error; + } + }; + + try { + activate(invoke); + if (callbackInvoked) { + if (!callbackSucceeded) throw callbackError; + return callbackResult as T; + } + + reportActivationFailure( + onActivationFailure, + new Error("Context activation returned without invoking its callback"), + ); + return invoke(); + } catch (activationError) { + if (callbackInvoked) { + if (!callbackSucceeded) throw callbackError; + return callbackResult as T; + } + reportActivationFailure(onActivationFailure, activationError); + return invoke(); + } +} diff --git a/src/observability/tracing/context-propagation.test.ts b/src/observability/tracing/context-propagation.test.ts index e2502f02bf..6fe91b3538 100644 --- a/src/observability/tracing/context-propagation.test.ts +++ b/src/observability/tracing/context-propagation.test.ts @@ -1,5 +1,11 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assert, assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { + assert, + assertEquals, + assertRejects, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; import { ContextPropagation } from "./context-propagation.ts"; import type { Context, OpenTelemetryAPI, Span, TextMapPropagator } from "./types.ts"; @@ -157,6 +163,113 @@ describe("observability/tracing/context-propagation", () => { }); describe("withActiveSpan", () => { + it("returns the application promise without waiting for a provider-owned promise", async () => { + const never = new Promise(() => {}); + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context: Context, fn: () => T): T => { + fn(); + return never as T; + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + const applicationPromise = Promise.resolve("application result"); + + const result = badCtx.withActiveSpan(createMockSpan(), () => applicationPromise); + + assertStrictEquals(result, applicationPromise); + assertEquals(await result, "application result"); + }); + + it("preserves the callback result when the context provider replaces it", async () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context: Context, fn: () => T): T => { + fn(); + return Promise.resolve("provider result") as T; + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + + const result = await badCtx.withActiveSpan( + createMockSpan(), + () => Promise.resolve("application result"), + ); + + assertEquals(result, "application result"); + }); + + it("runs the callback when the context provider returns without invoking it", async () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context: Context, _fn: () => T): T => "provider result" as T, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + const result = await badCtx.withActiveSpan(createMockSpan(), async () => { + calls++; + return "application result"; + }); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + + it("invokes the callback at most once when the context provider invokes it repeatedly", async () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context, fn) => { + const result = fn(); + fn(); + return result; + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + await badCtx.withActiveSpan(createMockSpan(), async () => { + calls++; + }); + + assertEquals(calls, 1); + }); + + it("preserves the callback result when the context provider fails after invocation", async () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context, fn) => { + fn(); + throw new Error("context provider failed"); + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + const result = await badCtx.withActiveSpan(createMockSpan(), async () => { + calls++; + return "application result"; + }); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + it("should execute function with span context", async () => { const span = createMockSpan(); let executed = false; @@ -200,6 +313,140 @@ describe("observability/tracing/context-propagation", () => { }); describe("withSpan", () => { + it("passes the exact unknown thrown value through span finalization", () => { + const thrown = { reason: "non-error" }; + let finalizedError: unknown; + + try { + ctx.withSpan( + "test", + () => { + throw thrown; + }, + () => createMockSpan(), + (_span, error) => { + finalizedError = error; + }, + ); + } catch (error) { + assertStrictEquals(error, thrown); + } + + assertStrictEquals(finalizedError, thrown); + }); + + it("distinguishes falsey thrown values from successful span completion", () => { + for (const thrown of [undefined, null, 0, ""] as const) { + let caught = false; + let finalized: unknown[] = []; + try { + ctx.withSpan( + "test", + () => { + throw thrown; + }, + () => createMockSpan(), + (_span, ...failure) => { + finalized = failure; + }, + ); + } catch (error) { + caught = true; + assertStrictEquals(error, thrown); + } + assertEquals(caught, true); + assertEquals(finalized.length, 1); + assertStrictEquals(finalized[0], thrown); + } + }); + + it("preserves a callback error when the context provider swallows it", () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context: Context, fn: () => T): T => { + try { + fn(); + } catch (_) { + return "provider result" as T; + } + return "provider result" as T; + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + + assertThrows( + () => + badCtx.withSpan( + "test", + () => { + throw new Error("application failure"); + }, + () => createMockSpan(), + () => {}, + ), + Error, + "application failure", + ); + }); + + it("runs the callback when the context provider returns without invoking it", () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context: Context, _fn: () => T): T => "provider result" as T, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + const result = badCtx.withSpan( + "test", + () => { + calls++; + return "application result"; + }, + () => createMockSpan(), + () => {}, + ); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + + it("preserves a completed callback result when context and finalization fail", () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context, fn) => { + fn(); + throw new Error("context provider failed"); + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + const result = badCtx.withSpan( + "test", + () => { + calls++; + return "application result"; + }, + () => createMockSpan(), + () => { + throw new Error("telemetry end failed"); + }, + ); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + it("should create span, execute fn, and end span", () => { let startCalled = false; let endCalled = false; @@ -272,7 +519,7 @@ describe("observability/tracing/context-propagation", () => { it("should end span with error when function throws", () => { const mockSpan = createMockSpan(); - let endError: Error | undefined; + let endError: unknown; assertThrows( () => @@ -296,6 +543,58 @@ describe("observability/tracing/context-propagation", () => { }); describe("withSpanAsync", () => { + it("passes the exact unknown rejection through span finalization", async () => { + const thrown = { reason: "async non-error" }; + let finalizedError: unknown; + let caught: unknown; + + try { + await ctx.withSpanAsync( + "test", + () => Promise.reject(thrown), + () => createMockSpan(), + (_span, error) => { + finalizedError = error; + }, + ); + } catch (error) { + caught = error; + } + + assertStrictEquals(caught, thrown); + assertStrictEquals(finalizedError, thrown); + }); + + it("preserves a completed callback result when context and finalization fail", async () => { + const badApi: OpenTelemetryAPI = { + ...api, + context: { + ...api.context, + with: (_context, fn) => { + fn(); + throw new Error("context provider failed"); + }, + }, + }; + const badCtx = new ContextPropagation(badApi, propagator); + let calls = 0; + + const result = await badCtx.withSpanAsync( + "test", + async () => { + calls++; + return "application result"; + }, + () => createMockSpan(), + () => { + throw new Error("telemetry end failed"); + }, + ); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + it("should create span, execute async fn, and end span", async () => { let startCalled = false; let endCalled = false; @@ -351,7 +650,7 @@ describe("observability/tracing/context-propagation", () => { it("should end span with error when async function rejects", async () => { const mockSpan = createMockSpan(); - let endError: Error | undefined; + let endError: unknown; await assertRejects( () => diff --git a/src/observability/tracing/context-propagation.ts b/src/observability/tracing/context-propagation.ts index c60ad94fa6..2c0e073d5a 100644 --- a/src/observability/tracing/context-propagation.ts +++ b/src/observability/tracing/context-propagation.ts @@ -1,7 +1,12 @@ import { serverLogger } from "#veryfront/utils"; +import { runAsyncWithContextFallback, runSyncWithContextFallback } from "./context-callback.ts"; import type { Context, OpenTelemetryAPI, Span, TextMapPropagator } from "./types.ts"; const logger = serverLogger.component("tracing"); +type SpanFailure = [] | [error: unknown]; +type SpanFinalizer = { + bivarianceHack(span: Span | null, error?: unknown): void; +}["bivarianceHack"]; export class ContextPropagation { constructor( @@ -14,7 +19,7 @@ export class ContextPropagation { const carrier: Record = Object.fromEntries(headers); return this.api.propagation.extract(this.api.context.active(), carrier); } catch (error) { - logger.debug("Failed to extract context from headers", error); + this.debug("Failed to extract context from headers", error); return undefined; } } @@ -28,7 +33,7 @@ export class ContextPropagation { headers.set(key, value); } } catch (error) { - logger.debug("Failed to inject context into headers", error); + this.debug("Failed to inject context into headers", error); } } @@ -36,7 +41,7 @@ export class ContextPropagation { try { return this.api.context.active(); } catch (error) { - logger.debug("Failed to get active context", error); + this.debug("Failed to get active context", error); return undefined; } } @@ -44,9 +49,13 @@ export class ContextPropagation { withActiveSpan(span: Span | null, fn: () => Promise): Promise { if (!span) return fn(); - return this.api.context.with( - this.api.trace.setSpan(this.api.context.active(), span), + const spanContext = this.resolveSpanContext(span); + if (!spanContext) return fn(); + + return runAsyncWithContextFallback( + (callback) => this.api.context.with(spanContext, callback), fn, + (error) => this.debug("Failed to activate span context", error), ); } @@ -54,19 +63,23 @@ export class ContextPropagation { name: string, fn: (span: Span | null) => T, startSpan: (name: string) => Span | null, - endSpan: (span: Span | null, error?: Error) => void, + endSpan: SpanFinalizer, ): T { - const span = startSpan(name); - const spanContext = span - ? this.api.trace.setSpan(this.api.context.active(), span) - : this.api.context.active(); + const span = this.startSpanSafely(name, startSpan); + const spanContext = this.resolveSpanContext(span); try { - const result = this.api.context.with(spanContext, () => fn(span)); - endSpan(span); + const result = spanContext + ? runSyncWithContextFallback( + (callback) => this.api.context.with(spanContext, callback), + () => fn(span), + (error) => this.debug("Failed to activate span context", error), + ) + : fn(span); + this.endSpanSafely(span, endSpan); return result; } catch (error) { - endSpan(span, error instanceof Error ? error : undefined); + this.endSpanSafely(span, endSpan, [error]); throw error; } } @@ -75,20 +88,69 @@ export class ContextPropagation { name: string, fn: (span: Span | null) => Promise, startSpan: (name: string) => Span | null, - endSpan: (span: Span | null, error?: Error) => void, + endSpan: SpanFinalizer, ): Promise { - const span = startSpan(name); - const spanContext = span - ? this.api.trace.setSpan(this.api.context.active(), span) - : this.api.context.active(); + const span = this.startSpanSafely(name, startSpan); + const spanContext = this.resolveSpanContext(span); try { - const result = await this.api.context.with(spanContext, () => fn(span)); - endSpan(span); + const result = spanContext + ? await runAsyncWithContextFallback( + (callback) => this.api.context.with(spanContext, callback), + () => fn(span), + (error) => this.debug("Failed to activate span context", error), + ) + : await fn(span); + this.endSpanSafely(span, endSpan); return result; } catch (error) { - endSpan(span, error instanceof Error ? error : undefined); + this.endSpanSafely(span, endSpan, [error]); throw error; } } + + private resolveSpanContext(span: Span | null): Context | undefined { + const activeContext = this.getActiveContext(); + if (!activeContext || !span) return activeContext; + + try { + return this.api.trace.setSpan(activeContext, span); + } catch (error) { + this.debug("Failed to associate span with context", error); + return activeContext; + } + } + + private startSpanSafely( + name: string, + startSpan: (name: string) => Span | null, + ): Span | null { + try { + return startSpan(name); + } catch (error) { + this.debug("Failed to start span", error); + return null; + } + } + + private endSpanSafely( + span: Span | null, + endSpan: SpanFinalizer, + failure: SpanFailure = [], + ): void { + try { + if (failure.length > 0) endSpan(span, failure[0]); + else endSpan(span); + } catch (endError) { + this.debug("Failed to end span", endError); + } + } + + private debug(message: string, error: unknown): void { + try { + logger.debug(message, error); + } catch (_) { + /* expected: logging failures must not affect application work */ + } + } } diff --git a/src/observability/tracing/index.ts b/src/observability/tracing/index.ts index 0e5deb5120..afccae2f4c 100644 --- a/src/observability/tracing/index.ts +++ b/src/observability/tracing/index.ts @@ -54,8 +54,8 @@ export function startSpan(name: string, options: SpanOptions = {}): Span | null } /** End an active tracing span. */ -export function endSpan(span: Span | null, error?: Error): void { - getSpanOps()?.endSpan(span, error); +export function endSpan(span: Span | null, ...failure: [] | [error: unknown]): void { + getSpanOps()?.endSpan(span, ...failure); } /** Sets span attributes. */ @@ -121,7 +121,10 @@ export async function withSpan( name, fn, (n) => spanOps.startSpan(n, options), - (s, e) => spanOps.endSpan(s, e), + (s: Span | null, ...failure: [] | [error: unknown]) => { + if (failure.length > 0) spanOps.endSpanWithFailure(s, failure[0]); + else spanOps.endSpan(s); + }, ); } @@ -140,7 +143,10 @@ export function withSpanSync( name, fn, (n) => spanOps.startSpan(n, options), - (s, e) => spanOps.endSpan(s, e), + (s: Span | null, ...failure: [] | [error: unknown]) => { + if (failure.length > 0) spanOps.endSpanWithFailure(s, failure[0]); + else spanOps.endSpan(s); + }, ); } diff --git a/src/observability/tracing/manager.test.ts b/src/observability/tracing/manager.test.ts index 6cd65db176..bc084d80a0 100644 --- a/src/observability/tracing/manager.test.ts +++ b/src/observability/tracing/manager.test.ts @@ -1,8 +1,43 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; -import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { assertEquals, assertNotEquals, assertStrictEquals } from "#veryfront/testing/assert.ts"; +import { afterEach, beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { + _resetShimForTests, + installGlobalTelemetryAPI, + type Span, + type Tracer, + type TracerProvider, +} from "./api-shim.ts"; import { TracingManager } from "./manager.ts"; +function createProvider(label: string, calls: string[]): TracerProvider { + const span: Span = { + setAttribute: () => span, + setAttributes: () => span, + setStatus: () => span, + recordException: () => {}, + addEvent: () => span, + end: () => {}, + spanContext: () => ({ + traceId: label.padEnd(32, "0"), + spanId: label.padEnd(16, "0"), + traceFlags: 1, + }), + updateName: () => {}, + }; + const tracer: Tracer = { + startSpan(name) { + calls.push(`${label}:${name}`); + return span; + }, + startActiveSpan: ((_name: string, ...args: unknown[]) => { + const callback = args.find((arg) => typeof arg === "function") as (span: Span) => unknown; + return callback(span); + }) as Tracer["startActiveSpan"], + }; + return { getTracer: () => tracer }; +} + describe("observability/tracing/manager", () => { let manager: TracingManager; @@ -10,6 +45,11 @@ describe("observability/tracing/manager", () => { manager = new TracingManager(); }); + afterEach(() => { + manager.shutdown(); + _resetShimForTests(); + }); + describe("initial state", () => { it("should not be enabled before initialization", () => { assertEquals(manager.isEnabled(), false); @@ -39,6 +79,26 @@ describe("observability/tracing/manager", () => { }); describe("initialize", () => { + it("follows provider A to B to none without retaining stale span operations", async () => { + const calls: string[] = []; + const providerA = installGlobalTelemetryAPI({ + tracerProvider: createProvider("A", calls), + }); + await manager.initialize({ enabled: true, serviceName: "test" }); + manager.getSpanOperations()?.startSpan("first"); + + const providerB = installGlobalTelemetryAPI({ + tracerProvider: createProvider("B", calls), + }); + manager.getSpanOperations()?.startSpan("second"); + assertEquals(providerA.dispose(), false); + assertEquals(providerB.dispose(), true); + + assertEquals(manager.isEnabled(), false); + assertEquals(manager.getSpanOperations(), null); + assertEquals(calls, ["A:first", "B:second"]); + }); + it("should mark as initialized with disabled config", async () => { await manager.initialize({ enabled: false }); assertEquals(manager.getState().initialized, true); @@ -51,6 +111,17 @@ describe("observability/tracing/manager", () => { assertEquals(manager.isEnabled(), false); }); + it("shares one readiness promise across concurrent initialization", async () => { + installGlobalTelemetryAPI({ tracerProvider: createProvider("A", []) }); + + const first = manager.initialize({ enabled: true, serviceName: "test" }); + const second = manager.initialize({ enabled: true, serviceName: "ignored" }); + + assertStrictEquals(second, first); + await first; + assertEquals(manager.getState().initialized, true); + }); + it("should accept empty config", async () => { await manager.initialize({}); assertEquals(manager.getState().initialized, true); @@ -103,6 +174,40 @@ describe("observability/tracing/manager", () => { }); describe("shutdown", () => { + it("prevents an in-flight initialization from restoring stale state", async () => { + installGlobalTelemetryAPI({ tracerProvider: createProvider("A", []) }); + + const initializing = manager.initialize({ enabled: true, serviceName: "test" }); + manager.shutdown(); + await initializing; + + assertEquals(manager.getState(), { + initialized: false, + degraded: false, + tracer: null, + api: null, + propagator: null, + }); + assertEquals(manager.getSpanOperations(), null); + }); + + it("releases cached state and permits a fresh initialization", async () => { + const calls: string[] = []; + installGlobalTelemetryAPI({ tracerProvider: createProvider("A", calls) }); + await manager.initialize({ enabled: true, serviceName: "test" }); + const firstOperations = manager.getSpanOperations(); + + manager.shutdown(); + + assertEquals(manager.getState().initialized, false); + assertEquals(manager.getSpanOperations(), null); + installGlobalTelemetryAPI({ tracerProvider: createProvider("B", calls) }); + await manager.initialize({ enabled: true, serviceName: "test" }); + assertNotEquals(manager.getSpanOperations(), firstOperations); + manager.getSpanOperations()?.startSpan("fresh"); + assertEquals(calls, ["B:fresh"]); + }); + it("should not throw when not initialized", () => { manager.shutdown(); }); @@ -132,6 +237,8 @@ describe("observability/tracing/manager", () => { assertEquals(state1.initialized, state2.initialized); assertEquals(state1.degraded, state2.degraded); + state1.initialized = true; + assertEquals(manager.getState().initialized, false); }); }); }); diff --git a/src/observability/tracing/manager.ts b/src/observability/tracing/manager.ts index e1d0491f61..06876ec0b0 100644 --- a/src/observability/tracing/manager.ts +++ b/src/observability/tracing/manager.ts @@ -1,10 +1,11 @@ import { serverLogger } from "#veryfront/utils"; import type { RuntimeAdapter } from "#veryfront/platform/adapters/base.ts"; import { VERSION } from "#veryfront/utils/version.ts"; +import { getGlobalTelemetryAPISnapshot } from "./api-shim.ts"; import { loadConfig } from "./config.ts"; import { ContextPropagation } from "./context-propagation.ts"; import { SpanOperations } from "./span-operations.ts"; -import type { OpenTelemetryAPI, TracingConfig, TracingState } from "./types.ts"; +import type { OpenTelemetryAPI, TextMapPropagator, TracingConfig, TracingState } from "./types.ts"; const logger = serverLogger.component("tracing"); @@ -23,39 +24,67 @@ export class TracingManager { private spanOps: SpanOperations | null = null; private contextProp: ContextPropagation | null = null; - - async initialize(config: Partial = {}, adapter?: RuntimeAdapter): Promise { + private configuredEnabled = false; + private providerRevision = -1; + private serviceName = "veryfront"; + private initializationPromise: Promise | null = null; + private lifecycleGeneration = 0; + + initialize(config: Partial = {}, adapter?: RuntimeAdapter): Promise { + if (this.initializationPromise) return this.initializationPromise; if (this.state.initialized) { logger.debug("Already initialized"); - return; + return Promise.resolve(); } const finalConfig = loadConfig(config, adapter); + const generation = this.lifecycleGeneration; this.state.initialized = true; + this.configuredEnabled = finalConfig.enabled; + this.serviceName = finalConfig.serviceName ?? "veryfront"; if (!finalConfig.enabled) { logger.debug("Tracing disabled"); - return; + return Promise.resolve(); } - try { - await this.initializeTracer(finalConfig); - - logger.info("OpenTelemetry tracing initialized", { - exporter: finalConfig.exporter, - serviceName: finalConfig.serviceName, - endpoint: finalConfig.endpoint, - }); - } catch (error) { - logger.error( - "[tracing] Failed to initialize OpenTelemetry tracing - running in degraded mode", - error, - ); - this.state.degraded = true; - } + const attempt = (async (): Promise => { + try { + const runtime = await this.createTracerRuntime(); + if (generation !== this.lifecycleGeneration) return; + + this.state.api = runtime.api; + this.state.propagator = runtime.propagator; + this.refreshProvider(true); + + logger.info("OpenTelemetry tracing initialized", { + exporter: finalConfig.exporter, + serviceName: finalConfig.serviceName, + endpoint: finalConfig.endpoint, + }); + } catch (error) { + if (generation !== this.lifecycleGeneration) return; + logger.error( + "[tracing] Failed to initialize OpenTelemetry tracing - running in degraded mode", + error, + ); + this.state.degraded = true; + } + })(); + + const tracked = attempt.finally(() => { + if (this.initializationPromise === tracked) { + this.initializationPromise = null; + } + }); + this.initializationPromise = tracked; + return tracked; } - private async initializeTracer(config: TracingConfig): Promise { + private async createTracerRuntime(): Promise<{ + api: OpenTelemetryAPI; + propagator: TextMapPropagator; + }> { // Use the shim API — delegates to the real SDK when ext-observability-opentelemetry is wired. const shimApi = await import("./api-shim.ts"); const api: OpenTelemetryAPI = { @@ -75,27 +104,57 @@ export class TracingManager { SpanKind: shimApi.SpanKind, SpanStatusCode: { OK: shimApi.SpanStatusCode.OK, ERROR: shimApi.SpanStatusCode.ERROR }, }; - this.state.api = api; - - this.state.tracer = api.trace.getTracer(config.serviceName ?? "veryfront", VERSION); // No-op propagator used only when ext-observability-opentelemetry is NOT installed. // When the extension is active, it registers W3CTraceContextPropagator // on the shim directly; we intentionally do NOT wrap shimApi.propagation // here (doing so would cause infinite recursion when the global // propagator is the wrapper itself). - const propagator = { + const propagator: TextMapPropagator = { inject: (_ctx: import("./api-shim.ts").Context, _carrier: unknown) => {}, extract: (ctx: import("./api-shim.ts").Context, _carrier: unknown) => ctx, fields: () => [] as string[], }; - this.state.propagator = propagator; + return { api, propagator }; + } - this.spanOps = this.state.tracer ? new SpanOperations(api, this.state.tracer) : null; - this.contextProp = new ContextPropagation(api, propagator); + private refreshProvider(force = false): void { + if (!this.state.initialized || !this.configuredEnabled || !this.state.api) return; + + const snapshot = getGlobalTelemetryAPISnapshot(); + if (!force && snapshot.tracerProviderRevision === this.providerRevision) return; + this.providerRevision = snapshot.tracerProviderRevision; + + if (!snapshot.tracerProviderInstalled) { + this.state.tracer = null; + this.spanOps = null; + this.contextProp = null; + return; + } + + try { + const tracer = this.state.api.trace.getTracer(this.serviceName, VERSION); + this.state.tracer = tracer; + this.spanOps = new SpanOperations(this.state.api, tracer); + this.contextProp = this.state.propagator + ? new ContextPropagation(this.state.api, this.state.propagator) + : null; + this.state.degraded = false; + } catch (error) { + this.state.tracer = null; + this.spanOps = null; + this.contextProp = null; + this.state.degraded = true; + try { + logger.warn("Failed to refresh OpenTelemetry tracer provider", error); + } catch (_) { + /* expected: telemetry lifecycle remains fail-open */ + } + } } isEnabled(): boolean { + this.refreshProvider(); return this.state.initialized && this.state.tracer !== null; } @@ -104,25 +163,42 @@ export class TracingManager { } getSpanOperations(): SpanOperations | null { + this.refreshProvider(); return this.spanOps; } getContextPropagation(): ContextPropagation | null { + this.refreshProvider(); return this.contextProp; } getState(): TracingState { - return this.state; + this.refreshProvider(); + return { ...this.state }; } shutdown(): void { - if (!this.state.initialized) return; + if (!this.state.initialized && !this.initializationPromise) return; try { logger.info("Tracing shutdown initiated"); } catch (error) { logger.warn("Error during tracing shutdown", error); } + this.lifecycleGeneration++; + this.initializationPromise = null; + this.state = { + initialized: false, + degraded: false, + tracer: null, + api: null, + propagator: null, + }; + this.spanOps = null; + this.contextProp = null; + this.configuredEnabled = false; + this.providerRevision = -1; + this.serviceName = "veryfront"; } } diff --git a/src/observability/tracing/otlp-setup.test.ts b/src/observability/tracing/otlp-setup.test.ts index 9347edb5b3..7a4cd2d052 100644 --- a/src/observability/tracing/otlp-setup.test.ts +++ b/src/observability/tracing/otlp-setup.test.ts @@ -1,15 +1,22 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals, assertExists } from "#veryfront/testing/assert.ts"; +import { + assertEquals, + assertExists, + assertRejects, + assertStrictEquals, +} from "#veryfront/testing/assert.ts"; import { afterEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor, -} from "npm:@opentelemetry/sdk-trace-base@2.8.0"; +} from "npm:@opentelemetry/sdk-trace-base@2.9.0"; import { _resetShimForTests, type AttributeValue, type Context, + propagation, setGlobalActiveSpanAccessor, setGlobalContextAccessor, setGlobalTracerProvider, @@ -93,6 +100,39 @@ describe("observability/tracing/otlp-setup", () => { assertEquals(capturedKind, SpanKind.CLIENT); }); + it("bounds direct helper span and event names before provider calls", async () => { + const { addSpanEvent, withSpan } = await import("./otlp-setup.ts"); + const received: { span?: string; event?: string } = {}; + const span = createTestSpan({ + addEvent(name) { + received.event = name; + return span; + }, + }); + setGlobalTracerProvider({ + getTracer: () => ({ + startSpan(name) { + received.span = name; + return span; + }, + startActiveSpan: (() => span) as never, + }), + }); + + await withSpan( + "s".repeat(MAX_SPAN_NAME_LENGTH + 100), + async (activeSpan) => { + addSpanEvent( + activeSpan, + "e".repeat(MAX_SPAN_NAME_LENGTH + 100), + ); + }, + ); + + assertEquals(received.span?.length, MAX_SPAN_NAME_LENGTH); + assertEquals(received.event?.length, MAX_SPAN_NAME_LENGTH); + }); + it("withSpan preserves callback-owned ERROR status on real OpenTelemetry spans", async () => { const exporter = new InMemorySpanExporter(); const provider = new BasicTracerProvider({ @@ -133,6 +173,195 @@ describe("observability/tracing/otlp-setup", () => { } }); + it("withSpan preserves callback outcomes when span completion fails", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + const applicationError = new Error("application failed"); + const span = createTestSpan({ + setStatus: () => { + throw new Error("telemetry status failed"); + }, + end: () => { + throw new Error("telemetry end failed"); + }, + }); + setGlobalTracerProvider({ + getTracer: () => ({ + startSpan: () => span, + startActiveSpan: (() => span) as never, + }), + }); + + assertEquals(await withSpan("success", async () => "application result"), "application result"); + await assertRejects( + () => + withSpan("failure", async () => { + throw applicationError; + }), + Error, + "application failed", + ); + }); + + it("withSpan preserves exact accessor-backed errors during descriptor-value poisoning", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + const applicationError = new Error("application failure"); + let applicationAccessorCalls = 0; + Object.defineProperty(applicationError, "message", { + configurable: true, + get(): never { + applicationAccessorCalls += 1; + throw new Error("application error accessor must not run"); + }, + }); + const previous = Object.getOwnPropertyDescriptor(Object.prototype, "value"); + let descriptorValueCalls = 0; + let caught: unknown; + Object.defineProperty(Object.prototype, "value", { + configurable: true, + get(): never { + descriptorValueCalls += 1; + throw new Error("inherited descriptor value must not run"); + }, + }); + + try { + await withSpan("descriptor-poisoning", async () => { + throw applicationError; + }); + } catch (error) { + caught = error; + } finally { + if (previous) { + Object.defineProperty(Object.prototype, "value", previous); + } else { + delete (Object.prototype as Record).value; + } + } + + assertStrictEquals(caught, applicationError); + assertEquals(applicationAccessorCalls, 0); + assertEquals(descriptorValueCalls, 0); + }); + + it("withSpan rethrows hostile proxy failures without inspecting them", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + let trapCalls = 0; + const applicationError = new Proxy(Object.create(null), { + get(): never { + trapCalls += 1; + throw new Error("get trap must not run"); + }, + getOwnPropertyDescriptor(): never { + trapCalls += 1; + throw new Error("descriptor trap must not run"); + }, + getPrototypeOf(): never { + trapCalls += 1; + throw new Error("prototype trap must not run"); + }, + }); + let caught: unknown; + + try { + await withSpan("hostile-proxy", async () => { + throw applicationError; + }); + } catch (error) { + caught = error; + } + + assertStrictEquals(caught, applicationError); + assertEquals(trapCalls, 0); + }); + + it("withSpan rethrows revoked proxy failures without replacing them", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + const revocable = Proxy.revocable(Object.create(null), {}); + const applicationError = revocable.proxy; + revocable.revoke(); + let caught: unknown; + + try { + await withSpan("revoked-proxy", async () => { + throw applicationError; + }); + } catch (error) { + caught = error; + } + + assertStrictEquals(caught, applicationError); + }); + + it("withSpan invokes its callback once with an inert span when tracer setup fails", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + + for (const failure of ["getTracer", "startSpan"] as const) { + _resetShimForTests(); + setGlobalTracerProvider({ + getTracer() { + if (failure === "getTracer") throw new Error("getTracer failed"); + return { + startSpan() { + throw new Error("startSpan failed"); + }, + startActiveSpan: (() => undefined) as never, + }; + }, + }); + let calls = 0; + + const result = await withSpan("fallback", async (span) => { + calls++; + span.setAttribute("safe", true).addEvent("still-safe").end(); + return `application-${failure}`; + }); + + assertEquals(result, `application-${failure}`); + assertEquals(calls, 1); + } + }); + + it("withSpanSync invokes its callback once when tracer setup fails", async () => { + const { withSpanSync } = await import("./otlp-setup.ts"); + setGlobalTracerProvider({ + getTracer() { + throw new Error("getTracer failed"); + }, + }); + let calls = 0; + + const result = withSpanSync("fallback", () => { + calls++; + return "application result"; + }); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + + it("withSpan preserves callback outcomes when context access and activation fail", async () => { + const { withSpan } = await import("./otlp-setup.ts"); + setGlobalContextAccessor({ + active() { + throw new Error("active context failed"); + }, + with(_context, callback) { + callback(); + callback(); + throw new Error("context activation failed"); + }, + }); + let calls = 0; + + const result = await withSpan("context-fallback", async () => { + calls++; + return "application result"; + }); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + it("withSpanSync should execute the callback when OTLP is unavailable", async () => { const { withSpanSync } = await import("./otlp-setup.ts"); @@ -158,6 +387,25 @@ describe("observability/tracing/otlp-setup", () => { assertEquals(Array.from(headers.entries()), [["x-test", "1"]]); }); + it("context propagation helpers fail open when a propagator throws", async () => { + const { extractContext, injectContext } = await import("./otlp-setup.ts"); + propagation.setGlobalPropagator({ + extract() { + throw new Error("extract failed"); + }, + inject() { + throw new Error("inject failed"); + }, + fields: () => [], + }); + const headers = new Headers([["x-test", "1"]]); + + assertEquals(extractContext(headers), undefined); + injectContext(headers); + + assertEquals(Array.from(headers.entries()), [["x-test", "1"]]); + }); + it("withContext should execute the callback when APIs are unavailable", async () => { const { withContext } = await import("./otlp-setup.ts"); @@ -166,6 +414,28 @@ describe("observability/tracing/otlp-setup", () => { assertEquals(result, "ok"); }); + it("withContext invokes application code once when context activation misbehaves", async () => { + const { withContext } = await import("./otlp-setup.ts"); + const context = createTestContext(); + setGlobalContextAccessor({ + active: () => context, + with: (_context, fn) => { + fn(); + fn(); + throw new Error("context provider failed"); + }, + }); + let calls = 0; + + const result = await withContext(context, async () => { + calls++; + return "application result"; + }); + + assertEquals(result, "application result"); + assertEquals(calls, 1); + }); + it("getTraceContext should return an empty object when no span is active", async () => { const { getTraceContext } = await import("./otlp-setup.ts"); @@ -248,6 +518,43 @@ describe("observability/tracing/otlp-setup", () => { assertEquals(getTracerCalls, 2); }); + it("startServerSpan removes query data from its name and target", async () => { + const { startServerSpan } = await import("./otlp-setup.ts"); + let startedName = ""; + const attributes: Record = {}; + const span = createTestSpan({ + setAttribute(key, value) { + attributes[key] = value; + return span; + }, + }); + setGlobalTracerProvider({ + getTracer: () => ({ + startSpan(name) { + startedName = name; + return span; + }, + startActiveSpan: (() => span) as never, + }), + }); + + startServerSpan("GET", "/items?access_token=secret"); + + assertEquals(startedName, "GET /items"); + assertEquals(attributes["http.target"], "/items"); + }); + + it("startServerSpan returns null when tracer setup fails", async () => { + const { startServerSpan } = await import("./otlp-setup.ts"); + setGlobalTracerProvider({ + getTracer() { + throw new Error("getTracer failed"); + }, + }); + + assertEquals(startServerSpan("GET", "/items"), null); + }); + it("withSpan starts nested spans with the active parent context", async () => { const { withSpan } = await import("./otlp-setup.ts"); @@ -374,4 +681,94 @@ describe("observability/tracing/otlp-setup", () => { assertEquals(childStart.name, "child"); assertEquals(childStart.parentSpan, spansByName.get("parent")); }); + + it("withSpanSync starts nested spans with the active parent context", async () => { + const { withSpanSync } = await import("./otlp-setup.ts"); + const rootContext = createTestContext(); + let activeContext = rootContext; + const contextSpans = new WeakMap(); + const starts: Array<{ name: string; parentSpan: Span | undefined; span: Span }> = []; + + setGlobalContextAccessor({ + active: () => activeContext, + with: (context, fn) => { + const previous = activeContext; + activeContext = context; + try { + return fn(); + } finally { + activeContext = previous; + } + }, + }); + setGlobalActiveSpanAccessor({ + getActiveSpan: () => contextSpans.get(activeContext), + getSpan: (context) => contextSpans.get(context), + setSpan: (_context, span) => { + const next = createTestContext(); + contextSpans.set(next, span); + return next; + }, + }); + setGlobalTracerProvider({ + getTracer: () => ({ + startSpan(name, _options, parentContext) { + const span = createTestSpan({ + spanContext: () => ({ + traceId: "00000000000000000000000000000001", + spanId: `000000000000000${starts.length + 1}`, + traceFlags: 1, + }), + }); + starts.push({ + name, + parentSpan: parentContext ? contextSpans.get(parentContext) : undefined, + span, + }); + return span; + }, + startActiveSpan: (() => createTestSpan()) as never, + }), + }); + + withSpanSync("parent", () => withSpanSync("child", () => "ok")); + + assertEquals(starts.length, 2); + assertEquals(starts[1]?.parentSpan, starts[0]?.span); + }); }); + +function createTestSpan(overrides: Partial = {}): Span { + const spanContext: SpanContext = { + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + traceFlags: 0, + }; + const span: Span = { + setAttribute: () => span, + setAttributes: () => span, + setStatus: () => span, + recordException: () => {}, + addEvent: () => span, + end: () => {}, + spanContext: () => spanContext, + updateName: () => {}, + ...overrides, + }; + return span; +} + +function createTestContext(): Context { + const values = new Map(); + return { + getValue: (key) => values.get(key), + setValue(key, value) { + values.set(key, value); + return this; + }, + deleteValue(key) { + values.delete(key); + return this; + }, + }; +} diff --git a/src/observability/tracing/otlp-setup.ts b/src/observability/tracing/otlp-setup.ts index bdcddcaabd..c5865f69d6 100644 --- a/src/observability/tracing/otlp-setup.ts +++ b/src/observability/tracing/otlp-setup.ts @@ -13,7 +13,9 @@ **************************/ import { isTruthyEnvValue } from "#veryfront/utils/constants/env.ts"; -import { serverLogger } from "#veryfront/utils/logger/logger.ts"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { __registerTraceContextGetter, serverLogger } from "#veryfront/utils/logger/logger.ts"; +import { sanitizeUrlForSpan } from "#veryfront/utils/logger/redact.ts"; import { type AttributeValue, type Context, @@ -30,6 +32,13 @@ import { type Tracer, } from "./api-shim.ts"; import { getHostTelemetryEnv } from "./telemetry-env.ts"; +import { + sanitizeErrorForTelemetry, + sanitizeTelemetryAttributes, + sanitizeTelemetryAttributeValue, + sanitizeTelemetryText, +} from "../telemetry-error.ts"; +import { runAsyncWithContextFallback, runSyncWithContextFallback } from "./context-callback.ts"; const logger = serverLogger.component("otel"); @@ -132,12 +141,135 @@ export async function initializeOTLPWithApis(): Promise { // Span helpers — delegate to shim (which delegates to SDK if wired) // --------------------------------------------------------------------------- -function setSpanErrorStatus(span: Span, error: unknown): void { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error instanceof Error ? error.message : String(error), +function reportTelemetryFailure(failureMessage: string, error: unknown): void { + try { + logger.debug(failureMessage, error); + } catch (_) { + /* expected: telemetry and logging failures must not affect application work */ + } +} + +function runTelemetryOperation(operation: () => void, failureMessage: string): void { + try { + operation(); + } catch (error) { + reportTelemetryFailure(failureMessage, error); + } +} + +function createInertContext( + entries: ReadonlyMap = new Map(), +): Context { + const values = new Map(entries); + return Object.freeze({ + getValue: (key: symbol) => values.get(key), + setValue(key: symbol, value: unknown) { + const next = new Map(values); + next.set(key, value); + return createInertContext(next); + }, + deleteValue(key: symbol) { + const next = new Map(values); + next.delete(key); + return createInertContext(next); + }, + }); +} + +function createInertSpan(): Span { + const spanContext = Object.freeze({ + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + traceFlags: 0, }); - if (error instanceof Error) span.recordException(error); + const span: Span = Object.freeze({ + setAttribute: () => span, + setAttributes: () => span, + setStatus: () => span, + recordException: () => {}, + addEvent: () => span, + end: () => {}, + spanContext: () => spanContext, + updateName: () => {}, + }); + return span; +} + +function isUsableSpan(value: unknown): value is Span { + if ((typeof value !== "object" && typeof value !== "function") || value === null) return false; + try { + const candidate = value as Span; + return typeof candidate.setAttribute === "function" && + typeof candidate.setAttributes === "function" && + typeof candidate.setStatus === "function" && + typeof candidate.recordException === "function" && + typeof candidate.addEvent === "function" && + typeof candidate.end === "function" && + typeof candidate.spanContext === "function" && + typeof candidate.updateName === "function"; + } catch (_) { + return false; + } +} + +function getActiveContextSafely(): Context { + try { + const activeContext = shimContext.active(); + if (activeContext) return activeContext; + } catch (error) { + reportTelemetryFailure("Failed to read active tracing context", error); + } + return createInertContext(); +} + +function startSpanWithFallback( + name: string, + attributes: Record | undefined, + options: WithSpanOptions | undefined, +): { span: Span; context: Context } { + const parentContext = getActiveContextSafely(); + let span = createInertSpan(); + + try { + const candidate = getTracingRuntime().tracer.startSpan( + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), + { + kind: options?.kind ?? SpanKind.INTERNAL, + attributes: sanitizeTelemetryAttributes(attributes), + }, + parentContext, + ); + if (!isUsableSpan(candidate)) { + throw new TypeError("Tracer returned an invalid span"); + } + span = candidate; + } catch (error) { + reportTelemetryFailure("Failed to start span; using inert span", error); + } + + let spanContext = parentContext; + try { + spanContext = shimTrace.setSpan(parentContext, span); + } catch (error) { + reportTelemetryFailure("Failed to associate span with context", error); + } + return { span, context: spanContext }; +} + +function setSpanErrorStatus(span: Span, error: unknown): void { + const telemetryError = sanitizeErrorForTelemetry(error); + runTelemetryOperation( + () => + span.setStatus({ + code: SpanStatusCode.ERROR, + message: telemetryError.message, + }), + "Failed to set span error status", + ); + runTelemetryOperation( + () => span.recordException(telemetryError), + "Failed to record span exception", + ); } export type WithSpanOptions = { @@ -151,25 +283,20 @@ export async function withSpan( attributes?: Record, options?: WithSpanOptions, ): Promise { - const { tracer } = getTracingRuntime(); - const parentContext = shimContext.active(); - - const span = tracer.startSpan( - name, - { kind: options?.kind ?? SpanKind.INTERNAL, attributes }, - parentContext, - ); - - const spanContext = shimTrace.setSpan(parentContext, span); + const { span, context: spanContext } = startSpanWithFallback(name, attributes, options); try { - const result = await shimContext.with(spanContext, () => fn(span)); + const result = await runAsyncWithContextFallback( + (callback) => shimContext.with(spanContext, callback), + () => fn(span), + (error) => reportTelemetryFailure("Failed to activate span context", error), + ); return result; } catch (error) { setSpanErrorStatus(span, error); throw error; } finally { - span.end(); + runTelemetryOperation(() => span.end(), "Failed to end span"); } } @@ -180,40 +307,48 @@ export function withSpanSync( attributes?: Record, options?: WithSpanOptions, ): T { - const { tracer } = getTracingRuntime(); - const parentContext = shimContext.active(); - - const span = tracer.startSpan( - name, - { kind: options?.kind ?? SpanKind.INTERNAL, attributes }, - parentContext, - ); + const { span, context: spanContext } = startSpanWithFallback(name, attributes, options); try { - const result = fn(); - span.setStatus({ code: SpanStatusCode.OK }); + const result = runSyncWithContextFallback( + (callback) => shimContext.with(spanContext, callback), + fn, + (error) => reportTelemetryFailure("Failed to activate span context", error), + ); + runTelemetryOperation( + () => span.setStatus({ code: SpanStatusCode.OK }), + "Failed to set span success status", + ); return result; } catch (error) { setSpanErrorStatus(span, error); throw error; } finally { - span.end(); + runTelemetryOperation(() => span.end(), "Failed to end span"); } } /** Context for extract. */ export function extractContext(headers: Headers): Context | undefined { - const carrier: Record = {}; - for (const [k, v] of headers) carrier[k.toLowerCase()] = v; - - return shimPropagation.extract(shimContext.active(), carrier, defaultTextMapGetter); + try { + const carrier: Record = {}; + for (const [k, v] of headers) carrier[k.toLowerCase()] = v; + return shimPropagation.extract(getActiveContextSafely(), carrier, defaultTextMapGetter); + } catch (error) { + reportTelemetryFailure("Failed to extract tracing context", error); + return undefined; + } } /** Context for inject. */ export function injectContext(headers: Headers): void { - const carrier: Record = {}; - shimPropagation.inject(shimContext.active(), carrier, defaultTextMapSetter); - for (const [k, v] of Object.entries(carrier)) headers.set(k, v); + try { + const carrier: Record = {}; + shimPropagation.inject(getActiveContextSafely(), carrier, defaultTextMapSetter); + for (const [k, v] of Object.entries(carrier)) headers.set(k, v); + } catch (error) { + reportTelemetryFailure("Failed to inject tracing context", error); + } } /** Starts server span. */ @@ -222,14 +357,44 @@ export function startServerSpan( path: string, parentContext?: unknown, ): { span: Span; context: Context } | null { - const { tracer } = getTracingRuntime(); - const ctx = (parentContext || shimContext.active()) as Context; + let ctx: Context; + let spanPath: string; + let span: Span; + try { + ctx = (parentContext ?? getActiveContextSafely()) as Context; + spanPath = sanitizeUrlForSpan(path); + const candidate = getTracingRuntime().tracer.startSpan( + sanitizeTelemetryText(`${method} ${spanPath}`, MAX_SPAN_NAME_LENGTH), + { kind: SpanKind.SERVER }, + ctx, + ); + if (!isUsableSpan(candidate)) throw new TypeError("Tracer returned an invalid server span"); + span = candidate; + } catch (error) { + reportTelemetryFailure("Failed to start server span", error); + return null; + } - const span = tracer.startSpan(`${method} ${path}`, { kind: SpanKind.SERVER }, ctx); - span.setAttribute("http.method", method); - span.setAttribute("http.target", path); + runTelemetryOperation( + () => span.setAttribute("http.method", sanitizeTelemetryAttributeValue("http.method", method)), + "Failed to set server span method", + ); + runTelemetryOperation( + () => + span.setAttribute( + "http.target", + sanitizeTelemetryAttributeValue("http.target", spanPath), + ), + "Failed to set server span target", + ); - return { span, context: shimTrace.setSpan(ctx, span) }; + let spanContext = ctx; + try { + spanContext = shimTrace.setSpan(ctx, span); + } catch (error) { + reportTelemetryFailure("Failed to associate server span with context", error); + } + return { span, context: spanContext }; } /** End an active server tracing span. */ @@ -237,22 +402,31 @@ export function endServerSpan(span: unknown, statusCode: number, error?: Error): if (!span) return; const otelSpan = span as Span; - otelSpan.setAttribute("http.status_code", statusCode); + runTelemetryOperation( + () => otelSpan.setAttribute("http.status_code", statusCode), + "Failed to set server span status code", + ); if (error) { setSpanErrorStatus(otelSpan, error); - otelSpan.end(); + runTelemetryOperation(() => otelSpan.end(), "Failed to end server span"); return; } if (statusCode >= 400) { - otelSpan.setStatus({ code: SpanStatusCode.ERROR }); - otelSpan.end(); + runTelemetryOperation( + () => otelSpan.setStatus({ code: SpanStatusCode.ERROR }), + "Failed to set server span error status", + ); + runTelemetryOperation(() => otelSpan.end(), "Failed to end server span"); return; } - otelSpan.setStatus({ code: SpanStatusCode.OK }); - otelSpan.end(); + runTelemetryOperation( + () => otelSpan.setStatus({ code: SpanStatusCode.OK }), + "Failed to set server span success status", + ); + runTelemetryOperation(() => otelSpan.end(), "Failed to end server span"); } /** Sets span attributes. */ @@ -263,7 +437,12 @@ export function setSpanAttributes( if (!span) return; const otelSpan = span as Span; - for (const [key, value] of Object.entries(attributes)) otelSpan.setAttribute(key, value); + for (const [key, value] of Object.entries(sanitizeTelemetryAttributes(attributes))) { + runTelemetryOperation( + () => otelSpan.setAttribute(key, value), + "Failed to set span attribute", + ); + } } /** Adds an event to a span. */ @@ -275,7 +454,14 @@ export function addSpanEvent( if (!span) return; const otelSpan = span as Span; - otelSpan.addEvent(name, attributes); + runTelemetryOperation( + () => + otelSpan.addEvent( + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), + sanitizeTelemetryAttributes(attributes), + ), + "Failed to add span event", + ); } /** Sets active span attributes. */ @@ -285,7 +471,12 @@ export function setActiveSpanAttributes( const span = shimTrace.getActiveSpan?.(); if (!span) return; - for (const [key, value] of Object.entries(attributes)) span.setAttribute(key, value); + for (const [key, value] of Object.entries(sanitizeTelemetryAttributes(attributes))) { + runTelemetryOperation( + () => span.setAttribute(key, value), + "Failed to set active span attribute", + ); + } } /** Marks the active span as failed. */ @@ -298,7 +489,11 @@ export function setActiveSpanErrorStatus(error: unknown): void { /** Context for with. */ export async function withContext(spanContext: unknown, fn: () => Promise): Promise { - return shimContext.with(spanContext as Context, fn); + return await runAsyncWithContextFallback( + (callback) => shimContext.with(spanContext as Context, callback), + fn, + (error) => logger.debug("Failed to activate explicit span context", error), + ); } /** Context for get trace. */ @@ -306,6 +501,15 @@ export function getTraceContext(): { traceId?: string; spanId?: string } { const span = shimTrace.getActiveSpan?.(); if (!span) return {}; - const ctx = span.spanContext(); - return { traceId: ctx.traceId, spanId: ctx.spanId }; + try { + const ctx = span.spanContext(); + return { traceId: ctx.traceId, spanId: ctx.spanId }; + } catch (_) { + return {}; + } } + +// The higher observability layer owns this adapter registration. Keeping the +// dependency direction here prevents generic logging utilities from importing +// OpenTelemetry implementation code. +__registerTraceContextGetter(getTraceContext); diff --git a/src/observability/tracing/service-tracer.test.ts b/src/observability/tracing/service-tracer.test.ts index 7bb2e804f6..8b0d01f318 100644 --- a/src/observability/tracing/service-tracer.test.ts +++ b/src/observability/tracing/service-tracer.test.ts @@ -1,5 +1,12 @@ -import { assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { + assertEquals, + assertRejects, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { MAX_TELEMETRY_ATTRIBUTE_COUNT, MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH } from "../limits.ts"; import { createOpenTelemetryServiceTracer } from "./service-tracer.ts"; type FakeContext = { @@ -17,6 +24,8 @@ class FakeSpan { status: { code: number } | null = null; exceptions: unknown[] = []; ended = false; + throwOnSetAttribute = false; + throwOnEnd = false; constructor(readonly name: string) { this.context = { @@ -26,6 +35,7 @@ class FakeSpan { } setAttribute(key: string, value: unknown): FakeSpan { + if (this.throwOnSetAttribute) throw new Error("telemetry attribute failure"); this.attributes[key] = value; return this; } @@ -47,6 +57,7 @@ class FakeSpan { } end(): void { + if (this.throwOnEnd) throw new Error("telemetry end failure"); this.ended = true; } @@ -99,6 +110,336 @@ function createHarness() { } describe("observability/tracing/service-tracer", () => { + it("resolves the current tracer for every operation after provider transitions", () => { + const harness = createHarness(); + const providerNames: string[] = []; + let provider = "A"; + harness.traceApi.getTracer = () => ({ + startSpan: (name: string) => { + providerNames.push(`${provider}:${name}`); + return new FakeSpan(`${provider}:${name}`); + }, + startActiveSpan: (name: string, fn: (span: FakeSpan) => T): T => { + providerNames.push(`${provider}:${name}`); + return fn(new FakeSpan(`${provider}:${name}`)); + }, + }); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + + serviceTracer.tracer.wrap("wrapped-a", () => undefined)(); + provider = "B"; + serviceTracer.tracer.wrap("wrapped-b", () => undefined)(); + serviceTracer.tracer.trace("traced-b", () => undefined); + + assertEquals(providerNames, ["A:wrapped-a", "B:wrapped-b", "B:traced-b"]); + }); + + it("preserves the exact promise returned by wrapped application code", async () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationPromise = Promise.resolve("application result"); + const wrapped = serviceTracer.tracer.wrap("async-operation", () => applicationPromise); + + const result = wrapped(); + + assertStrictEquals(result, applicationPromise); + assertEquals(await result, "application result"); + assertEquals(harness.startedSpans[0]?.ended, true); + }); + + it("preserves custom thenable identity while observing its settlement", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + let settle!: (value: string) => void; + const thenable = { + then(onFulfilled: (value: string) => void) { + settle = onFulfilled; + }, + }; + const wrapped = serviceTracer.tracer.wrap("thenable-operation", () => thenable); + + const result = wrapped(); + + assertStrictEquals(result, thenable); + assertEquals(harness.startedSpans[0]?.ended, false); + settle("done"); + assertEquals(harness.startedSpans[0]?.ended, true); + }); + + it("returns objects with hostile then getters unchanged and closes their spans", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationResult = Object.defineProperty({}, "then", { + get() { + throw new Error("hostile then getter"); + }, + }); + const wrapped = serviceTracer.tracer.wrap("hostile-then", () => applicationResult); + + const result = wrapped(); + + assertStrictEquals(result, applicationResult); + assertEquals(harness.startedSpans[0]?.ended, true); + }); + + it("runs wrapped code once when tracer and context setup fail", () => { + for (const failure of ["getTracer", "active", "startSpan", "setSpan"] as const) { + const harness = createHarness(); + const baseTracer = harness.traceApi.getTracer("test-service"); + if (failure === "getTracer") { + harness.traceApi.getTracer = () => { + throw new Error("getTracer failed"); + }; + } else if (failure === "active") { + harness.contextApi.active = () => { + throw new Error("active failed"); + }; + } else if (failure === "startSpan") { + harness.traceApi.getTracer = () => ({ + ...baseTracer, + startSpan() { + throw new Error("startSpan failed"); + }, + }); + } else { + harness.traceApi.setSpan = () => { + throw new Error("setSpan failed"); + }; + } + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + let calls = 0; + const expected = { failure }; + const wrapped = serviceTracer.tracer.wrap("operation", () => { + calls++; + return expected; + }); + + assertStrictEquals(wrapped(), expected); + assertEquals(calls, 1); + } + }); + + it("runs traced code once despite adversarial active-span providers", () => { + for (const behavior of ["duplicate", "omit", "replace", "throw-after"] as const) { + const harness = createHarness(); + const baseTracer = harness.traceApi.getTracer("test-service"); + harness.traceApi.getTracer = () => ({ + ...baseTracer, + startActiveSpan: (_name: string, callback: (span: FakeSpan) => T): T => { + const span = new FakeSpan(`active-${behavior}`); + harness.startedSpans.push(span); + if (behavior === "omit") return "provider replacement" as T; + const applicationResult = callback(span); + if (behavior === "duplicate") callback(span); + if (behavior === "throw-after") throw new Error("provider failed after callback"); + if (behavior === "replace") return "provider replacement" as T; + return applicationResult; + }, + }); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + let calls = 0; + const expected = { behavior }; + + const result = serviceTracer.tracer.trace("operation", () => { + calls++; + return expected; + }); + + assertStrictEquals(result, expected); + assertEquals(calls, 1); + } + }); + + it("preserves exact traced failures when the provider replaces callback results", () => { + const harness = createHarness(); + const baseTracer = harness.traceApi.getTracer("test-service"); + harness.traceApi.getTracer = () => ({ + ...baseTracer, + startActiveSpan: (_name: string, callback: (span: FakeSpan) => T): T => { + callback(new FakeSpan("replacement")); + return "provider replacement" as T; + }, + }); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationError = new Error("application failure"); + let caught: unknown; + + try { + serviceTracer.tracer.trace("operation", () => { + throw applicationError; + }); + } catch (error) { + caught = error; + } + + assertStrictEquals(caught, applicationError); + }); + + it("keeps wrapped async spans open until the operation settles", async () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const wrapped = serviceTracer.tracer.wrap("async-operation", async () => { + await gate; + return "done"; + }); + + const resultPromise = wrapped(); + assertEquals(harness.startedSpans[0]?.ended, false); + + release(); + assertEquals(await resultPromise, "done"); + assertEquals(harness.startedSpans[0]?.ended, true); + }); + + it("invokes wrapped application code at most once when context activation misbehaves", () => { + const harness = createHarness(); + harness.contextApi.with = (_context: FakeContext, fn: () => T): T => { + const result = fn(); + fn(); + return result; + }; + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + let calls = 0; + const wrapped = serviceTracer.tracer.wrap("operation", () => ++calls); + + assertEquals(wrapped(), 1); + assertEquals(calls, 1); + }); + + it("records wrapped async failures before ending and rethrowing", async () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationError = new Error("async wrapped failure"); + const wrapped = serviceTracer.tracer.wrap( + "async-operation", + () => Promise.reject(applicationError), + ); + + await assertRejects(() => wrapped(), Error, "async wrapped failure"); + + assertEquals(harness.startedSpans[0]?.status, { code: 2 }); + assertEquals( + (harness.startedSpans[0]?.exceptions[0] as Error | undefined)?.message, + applicationError.message, + ); + assertEquals(harness.startedSpans[0]?.ended, true); + }); + + it("preserves hostile thrown values and still closes the span", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationError = new Proxy({}, { + getPrototypeOf() { + throw new Error("prototype inspection failed"); + }, + get() { + throw new Error("property inspection failed"); + }, + }); + const wrapped = serviceTracer.tracer.wrap("hostile-error", () => { + throw applicationError; + }); + let caught: unknown; + + try { + wrapped(); + } catch (error) { + caught = error; + } + + assertEquals(caught === applicationError, true); + assertEquals(harness.startedSpans[0]?.status, { code: 2 }); + assertEquals(harness.startedSpans[0]?.ended, true); + assertEquals((harness.startedSpans[0]?.exceptions[0] as Error).name, "Unknown"); + }); + + it("redacts URL credentials from recorded exceptions without changing the rejection", async () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const applicationError = new Error( + "failed https://user:password@example.test/path?access_token=secret", + ); + const wrapped = serviceTracer.tracer.wrap( + "async-operation", + () => Promise.reject(applicationError), + ); + + try { + await wrapped(); + throw new Error("expected wrapped operation to reject"); + } catch (error) { + assertEquals(error, applicationError); + } + + const recorded = harness.startedSpans[0]?.exceptions[0] as Error; + assertEquals(recorded.message.includes("secret"), false); + assertEquals(recorded.message.includes("[REDACTED]"), true); + }); + it("creates active spans and exposes trace context", () => { const harness = createHarness(); const serviceTracer = createOpenTelemetryServiceTracer({ @@ -144,7 +485,10 @@ describe("observability/tracing/service-tracer", () => { "sync failed", ); assertEquals(syncHarness.startedSpans[0]?.status, { code: 2 }); - assertEquals(syncHarness.startedSpans[0]?.exceptions, [syncError]); + assertEquals( + (syncHarness.startedSpans[0]?.exceptions[0] as Error | undefined)?.message, + syncError.message, + ); assertEquals(syncHarness.startedSpans[0]?.ended, true); const asyncHarness = createHarness(); @@ -165,7 +509,10 @@ describe("observability/tracing/service-tracer", () => { "async failed", ); assertEquals(asyncHarness.startedSpans[0]?.status, { code: 2 }); - assertEquals(asyncHarness.startedSpans[0]?.exceptions, [asyncError]); + assertEquals( + (asyncHarness.startedSpans[0]?.exceptions[0] as Error | undefined)?.message, + asyncError.message, + ); assertEquals(asyncHarness.startedSpans[0]?.ended, true); }); @@ -195,4 +542,87 @@ describe("observability/tracing/service-tracer", () => { undefinedValue: "", }); }); + + it("bounds manual span names, attribute names, and attribute cardinality", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const attributes = Object.fromEntries( + Array.from( + { length: MAX_TELEMETRY_ATTRIBUTE_COUNT + 20 }, + (_, index) => [`attribute-${index}`, index], + ), + ); + + const span = serviceTracer.tracer.startSpan( + "s".repeat(MAX_SPAN_NAME_LENGTH + 100), + ); + span.setTag( + "k".repeat(MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH + 100), + "bounded", + ); + span.setAttributes(attributes); + + const providerSpan = harness.startedSpans[0]; + assertEquals(providerSpan?.name.length, MAX_SPAN_NAME_LENGTH); + assertEquals( + Object.keys(providerSpan?.attributes ?? {}).length, + MAX_TELEMETRY_ATTRIBUTE_COUNT + 1, + ); + assertEquals( + Object.keys(providerSpan?.attributes ?? {})[0]?.length, + MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH, + ); + }); + + it("redacts sensitive object attributes and safely serializes cycles", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const cyclic: { safe: string; self?: unknown } = { safe: "value" }; + cyclic.self = cyclic; + + const span = serviceTracer.tracer.startSpan("manual-operation"); + span.setTag("apiKey", "secret"); + span.setTag("endpoint", "https://example.test/path?access_token=secret"); + span.setTag("metadata", { + apiKey: "secret", + nested: { password: "also-secret" }, + }); + span.setTag("cyclic", cyclic); + + assertEquals(harness.startedSpans[0]?.attributes, { + apiKey: "[REDACTED]", + endpoint: "https://example.test/path?access_token=[REDACTED]", + metadata: '{"apiKey":"[REDACTED]","nested":{"password":"[REDACTED]"}}', + cyclic: '{"safe":"value","self":"[REDACTED]"}', + }); + }); + + it("isolates manual span attribute and finish failures", () => { + const harness = createHarness(); + const serviceTracer = createOpenTelemetryServiceTracer({ + serviceName: "test-service", + context: harness.contextApi, + trace: harness.traceApi, + errorStatusCode: 2, + }); + const span = serviceTracer.tracer.startSpan("manual-operation"); + const otelSpan = harness.startedSpans[0]; + if (!otelSpan) throw new Error("expected span"); + otelSpan.throwOnSetAttribute = true; + otelSpan.throwOnEnd = true; + + span.setTag("safe", "value"); + span.setAttributes({ another: "value" }); + span.finish(); + }); }); diff --git a/src/observability/tracing/service-tracer.ts b/src/observability/tracing/service-tracer.ts index cbaaa9e3b9..6acd233a54 100644 --- a/src/observability/tracing/service-tracer.ts +++ b/src/observability/tracing/service-tracer.ts @@ -1,3 +1,17 @@ +import { REDACTED, redactForSerialization } from "#veryfront/utils/logger/redact.ts"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { + MAX_OBSERVABILITY_NAME_LENGTH, + MAX_TELEMETRY_ATTRIBUTE_COUNT, + MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH, +} from "../limits.ts"; +import { + sanitizeErrorForTelemetry, + sanitizeTelemetryAttributeValue, + sanitizeTelemetryText, +} from "../telemetry-error.ts"; +import { runSyncWithContextFallback } from "./context-callback.ts"; + /** Context for open telemetry span. */ export type OpenTelemetrySpanContext = { traceId: string; @@ -144,38 +158,75 @@ function toAttributeValue( } if (typeof value === "object") { - return JSON.stringify(value); + try { + const redacted = redactForSerialization(value); + if (typeof redacted === "string") return redacted; + return JSON.stringify(redacted) ?? REDACTED; + } catch (_) { + return REDACTED; + } } return value; } +function setSpanAttribute( + span: TSpan, + key: string, + value: ServiceTracerAttributeInput, +): void { + try { + if (typeof key !== "string" || !key) return; + const boundedKey = key.slice(0, MAX_TELEMETRY_ATTRIBUTE_KEY_LENGTH); + span.setAttribute( + boundedKey, + sanitizeTelemetryAttributeValue(key, toAttributeValue(value)) ?? "", + ); + } catch (_) { + /* expected: telemetry failures must not replace application results */ + } +} + function createTracerSpan( contextApi: OpenTelemetryContextApi, span: TSpan, context: TContext, ): ServiceTracerSpan { - const spanContext = span.spanContext(); - return { setTag: (key, value) => { - span.setAttribute(key, toAttributeValue(value)); + setSpanAttribute(span, key, value); return span; }, setAttributes: (attributes) => { - for (const [key, value] of Object.entries(attributes)) { - span.setAttribute(key, toAttributeValue(value)); + try { + const keys = Object.keys(attributes).slice(0, MAX_TELEMETRY_ATTRIBUTE_COUNT); + for (const key of keys) { + setSpanAttribute(span, key, attributes[key]); + } + } catch (_) { + /* expected: hostile attribute containers fail closed */ } return span; }, finish: () => { - span.end(); + endSpan(span); + }, + withContext: (fn: () => T): T => + runSyncWithContextFallback( + (callback) => contextApi.with(context, callback), + fn, + ), + context: () => { + try { + const spanContext = span.spanContext(); + return { + toTraceId: () => spanContext.traceId, + toSpanId: () => spanContext.spanId, + }; + } catch (_) { + return undefined; + } }, - withContext: (fn: () => T): T => contextApi.with(context, fn), - context: () => ({ - toTraceId: () => spanContext.traceId, - toSpanId: () => spanContext.spanId, - }), otelSpan: span, otelContext: context, }; @@ -186,14 +237,133 @@ function setSpanErrorStatus( errorStatusCode: number, error: unknown, ): void { - span.setStatus({ code: errorStatusCode }); - if (error instanceof Error) { - span.recordException(error); + try { + span.setStatus({ code: errorStatusCode }); + } catch (_) { + /* expected: telemetry failures must not replace application failures */ + } + try { + span.recordException(sanitizeErrorForTelemetry(error)); + } catch (_) { + /* expected: telemetry failures must not replace application failures */ + } +} + +function endSpan(span: TSpan): void { + try { + span.end(); + } catch (_) { + /* expected: telemetry failures must not replace application results */ + } +} + +function createInertSpan(): TSpan { + const spanContext = Object.freeze({ + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + }); + const span: OpenTelemetrySpan = Object.freeze({ + setAttribute: () => span, + setAttributes: () => span, + setStatus: () => span, + recordException: () => {}, + end: () => {}, + spanContext: () => spanContext, + }); + return span as TSpan; +} + +function isUsableSpan(value: unknown): value is TSpan { + if ((typeof value !== "object" && typeof value !== "function") || value === null) return false; + try { + const span = value as OpenTelemetrySpan; + return typeof span.setAttribute === "function" && + typeof span.setAttributes === "function" && + typeof span.setStatus === "function" && + typeof span.recordException === "function" && + typeof span.end === "function" && + typeof span.spanContext === "function"; + } catch (_) { + return false; + } +} + +type SpanFinisher = (failed: boolean, error?: unknown) => void; + +function createSpanFinisher( + span: TSpan, + errorStatusCode: number, +): SpanFinisher { + let finished = false; + return (failed, error) => { + if (finished) return; + finished = true; + if (failed) setSpanErrorStatus(span, errorStatusCode, error); + endSpan(span); + }; +} + +function getThenMethod(value: unknown): ((...args: unknown[]) => unknown) | null { + if ( + !((typeof value === "object" && value !== null) || typeof value === "function") + ) { + return null; + } + const then = (value as { then?: unknown }).then; + return typeof then === "function" ? then as (...args: unknown[]) => unknown : null; +} + +function observeSettlement( + value: unknown, + finish: SpanFinisher, +): void { + let then: ((...args: unknown[]) => unknown) | null; + try { + then = getThenMethod(value); + } catch (error) { + finish(true, error); + return; + } + + if (!then) { + finish(false); + return; + } + + let settled = false; + try { + Reflect.apply(then, value, [ + () => { + if (settled) return; + settled = true; + finish(false); + }, + (error: unknown) => { + if (settled) return; + settled = true; + finish(true, error); + }, + ]); + } catch (error) { + if (settled) return; + settled = true; + finish(true, error); } } -function isPromise(value: T | Promise): value is Promise { - return value instanceof Promise; +function consumeIgnoredThenable(value: unknown): void { + let then: ((...args: unknown[]) => unknown) | null; + try { + then = getThenMethod(value); + } catch (_) { + return; + } + if (!then) return; + try { + Reflect.apply(then, value, [() => {}, () => {}]); + } catch (_) { + /* expected: provider-owned thenables cannot affect application outcomes */ + } } /** Create open telemetry service tracer. */ @@ -204,74 +374,184 @@ export function createOpenTelemetryServiceTracer< >( options: CreateOpenTelemetryServiceTracerOptions, ): OpenTelemetryServiceTracer { - const otelTracer = options.trace.getTracer(options.serviceName); + function getOtelTracer(): OpenTelemetryTracer | undefined { + try { + const candidate = options.trace.getTracer( + sanitizeTelemetryText( + options.serviceName, + MAX_OBSERVABILITY_NAME_LENGTH, + ), + ); + if ( + !candidate || typeof candidate.startSpan !== "function" || + typeof candidate.startActiveSpan !== "function" + ) { + return undefined; + } + return candidate; + } catch (_) { + return undefined; + } + } + + function getActiveContext(): TContext { + try { + return options.context.active(); + } catch (_) { + return undefined as TContext; + } + } + + function setSpanOnContext(context: TContext, span: TSpan): TContext { + try { + return options.trace.setSpan(context, span); + } catch (_) { + return context; + } + } + + function startSpan( + name: string, + startOptions: TSpanOptions | undefined, + context: TContext, + ): TSpan { + const tracer = getOtelTracer(); + if (tracer) { + try { + const candidate = tracer.startSpan( + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), + startOptions, + context, + ); + if (isUsableSpan(candidate)) return candidate; + } catch (_) { + /* expected: invalid providers fall back to an inert span */ + } + } + return createInertSpan(); + } + + function getSpanFromContext(context: TContext): TSpan | undefined { + try { + const span = options.trace.getSpan(context); + return isUsableSpan(span) ? span : undefined; + } catch (_) { + return undefined; + } + } function wrap( name: string, fn: (...args: TArgs) => TResult, ): (...args: TArgs) => TResult { return (...args: TArgs): TResult => { - const span = otelTracer.startSpan(name, undefined, options.context.active()); - const contextWithSpan = options.trace.setSpan(options.context.active(), span); + const parentContext = getActiveContext(); + const span = startSpan(name, undefined, parentContext); + const contextWithSpan = setSpanOnContext(parentContext, span); + const finish = createSpanFinisher(span, options.errorStatusCode); + let result: TResult; try { - return options.context.with(contextWithSpan, () => fn(...args)); + result = runSyncWithContextFallback( + (callback) => options.context.with(contextWithSpan, callback), + () => fn(...args), + ); } catch (error) { - setSpanErrorStatus(span, options.errorStatusCode, error); + finish(true, error); throw error; - } finally { - span.end(); } + + observeSettlement(result, finish); + return result; }; } function trace(name: string, fn: () => Promise): Promise; function trace(name: string, fn: () => T): T; function trace(name: string, fn: () => T | Promise): T | Promise { - return otelTracer.startActiveSpan(name, (span) => { + let callbackInvoked = false; + let callbackSucceeded = false; + let callbackResult!: T | Promise; + let callbackError: unknown; + let selectedSpan: TSpan | undefined; + + const selectSpan = (candidate?: unknown): TSpan => { + if (!selectedSpan) { + selectedSpan = isUsableSpan(candidate) ? candidate : createInertSpan(); + } else if ( + isUsableSpan(candidate) && candidate !== selectedSpan + ) { + endSpan(candidate); + } + return selectedSpan; + }; + + const invoke = (): T | Promise => { + if (callbackInvoked) { + if (!callbackSucceeded) throw callbackError; + return callbackResult; + } + callbackInvoked = true; + const finish = createSpanFinisher(selectSpan(), options.errorStatusCode); try { - const result = fn(); - if (isPromise(result)) { - return result - .then((value) => { - span.end(); - return value; - }) - .catch((error) => { - setSpanErrorStatus(span, options.errorStatusCode, error); - span.end(); - throw error; - }); - } - span.end(); - return result; + callbackResult = fn(); + callbackSucceeded = true; + observeSettlement(callbackResult, finish); + return callbackResult; } catch (error) { - setSpanErrorStatus(span, options.errorStatusCode, error); - span.end(); + callbackError = error; + finish(true, error); throw error; } - }); + }; + + const activeTracer = getOtelTracer(); + if (!activeTracer) return invoke(); + + let providerResult: unknown; + try { + providerResult = activeTracer.startActiveSpan( + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), + (span) => { + selectSpan(span); + return invoke(); + }, + ); + } catch (_) { + if (!callbackInvoked) return invoke(); + if (!callbackSucceeded) throw callbackError; + return callbackResult; + } + + if (!callbackInvoked) { + consumeIgnoredThenable(providerResult); + return invoke(); + } + if (providerResult !== callbackResult) consumeIgnoredThenable(providerResult); + if (!callbackSucceeded) throw callbackError; + return callbackResult; } const tracer: ServiceTracer = { init: () => {}, startSpan: (name, startOptions) => { - let parentContext = options.context.active(); - - if (startOptions?.childOf?.otelSpan) { - parentContext = options.trace.setSpan( - options.context.active(), - startOptions.childOf.otelSpan, - ); + let parentContext = getActiveContext(); + try { + const parentSpan = startOptions?.childOf?.otelSpan; + if (parentSpan) { + parentContext = setSpanOnContext(parentContext, parentSpan); + } + } catch (_) { + /* expected: hostile option accessors do not prevent span creation */ } - const span = otelTracer.startSpan(name, startOptions, parentContext); - const spanContext = options.trace.setSpan(parentContext, span); + const span = startSpan(name, startOptions, parentContext); + const spanContext = setSpanOnContext(parentContext, span); return createTracerSpan(options.context, span, spanContext); }, scope: () => ({ active: () => { - const activeContext = options.context.active(); - const activeSpan = options.trace.getSpan(activeContext); + const activeContext = getActiveContext(); + const activeSpan = getSpanFromContext(activeContext); if (!activeSpan) return null; return createTracerSpan(options.context, activeSpan, activeContext); @@ -284,22 +564,26 @@ export function createOpenTelemetryServiceTracer< return { tracer, setActiveSpanAttributes(attributes) { - const activeSpan = tracer.scope().active(); - if (!activeSpan) return; - - activeSpan.setAttributes(attributes); + try { + const activeSpan = tracer.scope().active(); + if (!activeSpan) return; + activeSpan.setAttributes(attributes); + } catch (_) { + /* expected: telemetry failures do not escape */ + } }, getTraceContext() { - const activeSpan = tracer.scope().active(); - if (!activeSpan) { + try { + const activeSpan = tracer.scope().active(); + const spanContext = activeSpan?.context(); + if (!spanContext) return {}; + return { + traceId: spanContext.toTraceId(), + spanId: spanContext.toSpanId(), + }; + } catch (_) { return {}; } - - const spanContext = activeSpan.otelSpan.spanContext(); - return { - traceId: spanContext.traceId, - spanId: spanContext.spanId, - }; }, }; } diff --git a/src/observability/tracing/span-names.test.ts b/src/observability/tracing/span-names.test.ts index 1fed32b1b7..3b284795fb 100644 --- a/src/observability/tracing/span-names.test.ts +++ b/src/observability/tracing/span-names.test.ts @@ -74,6 +74,7 @@ describe("observability/tracing/span-names", () => { HTML_GENERATE_SHELL_PARTS: "html.generate_shell_parts", HTML_WRAP_IN_SHELL: "html.wrap_in_shell", HTML_GENERATE_TAILWIND_CSS: "html.generate_tailwind_css", + HTML_GENERATE_CSS: "html.generate_css", }; for (const [key, value] of Object.entries(expected)) { diff --git a/src/observability/tracing/span-names.ts b/src/observability/tracing/span-names.ts index 2825358f60..a47d2dfe50 100644 --- a/src/observability/tracing/span-names.ts +++ b/src/observability/tracing/span-names.ts @@ -97,6 +97,9 @@ export const SpanNames = { CACHE_REGISTRY_SCAN_REDIS: "cache.registry.scan_redis", CACHE_REGISTRY_GET_REDIS_KEYS: "cache.registry.get_redis_keys", CACHE_REGISTRY_DELETE_REDIS_KEYS: "cache.registry.delete_redis_keys", + CACHE_REGISTRY_LIST_DISTRIBUTED_KEYS: "cache.registry.list_distributed_keys", + CACHE_REGISTRY_GET_DISTRIBUTED_KEYS: "cache.registry.get_distributed_keys", + CACHE_REGISTRY_DELETE_DISTRIBUTED_KEYS: "cache.registry.delete_distributed_keys", CACHE_KEYS_GET_ALL_ASYNC: "cache.keys.get_all_async", CACHE_KEYS_DELETE_ALL_ASYNC: "cache.keys.delete_all_async", CACHE_MULTI_TIER_GET: "cache.multi_tier.get", @@ -105,6 +108,7 @@ export const SpanNames = { HTML_GENERATE_SHELL_PARTS: "html.generate_shell_parts", HTML_WRAP_IN_SHELL: "html.wrap_in_shell", HTML_GENERATE_TAILWIND_CSS: "html.generate_tailwind_css", + HTML_GENERATE_CSS: "html.generate_css", HTML_GET_CSS_BY_HASH: "html.get_css_by_hash", HTML_REGENERATE_CSS_BY_HASH: "html.regenerate_css_by_hash", diff --git a/src/observability/tracing/span-operations.test.ts b/src/observability/tracing/span-operations.test.ts index a977dfbe48..4745f20a01 100644 --- a/src/observability/tracing/span-operations.test.ts +++ b/src/observability/tracing/span-operations.test.ts @@ -1,8 +1,9 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assertEquals } from "#veryfront/testing/assert.ts"; +import { assertEquals, assertStrictEquals } from "#veryfront/testing/assert.ts"; import { beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; import { SpanOperations } from "./span-operations.ts"; -import type { OpenTelemetryAPI, Span, Tracer } from "./types.ts"; +import type { Context, OpenTelemetryAPI, Span, Tracer } from "./types.ts"; type MockSpan = Span & { _ended: boolean; @@ -24,15 +25,19 @@ function createMockSpan(): MockSpan { }, setStatus(status: { code: number; message?: string }) { span._status = status; + return span; }, setAttributes(attrs: Record) { Object.assign(span._attributes, attrs); + return span; }, setAttribute(key: string, value: unknown) { span._attributes[key] = value; + return span; }, addEvent(name: string, attributes?: Record) { span._events.push({ name, attributes }); + return span; }, recordException(error: Error) { span._exception = error; @@ -48,7 +53,7 @@ function createMockSpan(): MockSpan { }, }; - return span as MockSpan; + return span as unknown as MockSpan; } function createMockTracer(): Tracer { @@ -99,11 +104,115 @@ describe("observability/tracing/span-operations", () => { }); describe("startSpan", () => { + it("redacts sensitive and URL credential attribute values", () => { + let receivedAttributes: Record | undefined; + tracer = { + startSpan: (_name, options) => { + receivedAttributes = options?.attributes; + return createMockSpan(); + }, + startActiveSpan: (() => {}) as never, + }; + ops = new SpanOperations(api, tracer); + + ops.startSpan("test", { + attributes: { + apiKey: "secret", + endpoint: "https://example.test/path?token=secret", + }, + }); + + assertEquals(receivedAttributes, { + apiKey: "[REDACTED]", + endpoint: "https://example.test/path?token=[REDACTED]", + }); + }); + + it("converts a Span parent into an OpenTelemetry Context", () => { + const parent = createMockSpan(); + const expectedContext = { _type: "parent-context" } as never; + let receivedContext: unknown; + api.trace.setSpan = (_context, span) => { + assertEquals(span, parent); + return expectedContext; + }; + tracer = { + startSpan: (_name, _options, context) => { + receivedContext = context; + return createMockSpan(); + }, + startActiveSpan: (() => {}) as never, + }; + ops = new SpanOperations(api, tracer); + + ops.startSpan("child", { parent }); + + assertEquals(receivedContext, expectedContext); + }); + + it("passes through a Context whose spanContext getter throws", () => { + const parent = Object.defineProperty({}, "spanContext", { + get(): never { + throw new Error("span context unavailable"); + }, + }); + let receivedContext: unknown; + tracer = { + startSpan: (_name, _options, context) => { + receivedContext = context; + return createMockSpan(); + }, + startActiveSpan: (() => {}) as never, + }; + ops = new SpanOperations(api, tracer); + + const span = ops.startSpan("child", { parent: parent as Context }); + + assertEquals(span !== null, true); + assertStrictEquals(receivedContext, parent); + }); + + it("passes through a revoked Proxy Context without probing it", () => { + const revocable = Proxy.revocable({}, {}); + const parent = revocable.proxy; + revocable.revoke(); + let receivedContext: unknown; + tracer = { + startSpan: (_name, _options, context) => { + receivedContext = context; + return createMockSpan(); + }, + startActiveSpan: (() => {}) as never, + }; + ops = new SpanOperations(api, tracer); + + const span = ops.startSpan("child", { parent: parent as Context }); + + assertEquals(span !== null, true); + assertStrictEquals(receivedContext, parent); + }); + it("should create a span with given name", () => { const span = ops.startSpan("test.operation"); assertEquals(span !== null, true); }); + it("bounds span names before invoking the provider", () => { + let receivedName = ""; + tracer = { + startSpan: (name) => { + receivedName = name; + return createMockSpan(); + }, + startActiveSpan: (() => {}) as never, + }; + ops = new SpanOperations(api, tracer); + + ops.startSpan("x".repeat(MAX_SPAN_NAME_LENGTH + 100)); + + assertEquals(receivedName.length, MAX_SPAN_NAME_LENGTH); + }); + it("should create a span with default options", () => { const span = ops.startSpan("test.operation"); assertEquals(span !== null, true); @@ -136,6 +245,23 @@ describe("observability/tracing/span-operations", () => { }); describe("endSpan", () => { + it("still attempts to end a span when status recording fails", () => { + let ended = false; + const badSpan = { + ...createMockSpan(), + setStatus() { + throw new Error("setStatus failed"); + }, + end() { + ended = true; + }, + } as unknown as Span; + + ops.endSpan(badSpan); + + assertEquals(ended, true); + }); + it("should end a span with OK status", () => { const mockSpan = createMockSpan(); ops.endSpan(mockSpan); @@ -143,6 +269,26 @@ describe("observability/tracing/span-operations", () => { assertEquals(mockSpan._status?.code, 1); }); + it("treats an explicitly forwarded undefined error as success", () => { + const mockSpan = createMockSpan(); + + ops.endSpan(mockSpan, undefined); + + assertEquals(mockSpan._ended, true); + assertEquals(mockSpan._status?.code, 1); + assertEquals(mockSpan._exception, null); + }); + + it("records an observed thrown undefined value as a failure", () => { + const mockSpan = createMockSpan(); + + ops.endSpanWithFailure(mockSpan, undefined); + + assertEquals(mockSpan._ended, true); + assertEquals(mockSpan._status?.code, 2); + assertEquals(mockSpan._exception?.message, "undefined"); + }); + it("should end a span with error status", () => { const mockSpan = createMockSpan(); const error = new Error("test error"); @@ -150,7 +296,20 @@ describe("observability/tracing/span-operations", () => { assertEquals(mockSpan._ended, true); assertEquals(mockSpan._status?.code, 2); assertEquals(mockSpan._status?.message, "test error"); - assertEquals(mockSpan._exception, error); + assertEquals(mockSpan._exception?.message, error.message); + }); + + it("redacts URL credentials from error telemetry", () => { + const mockSpan = createMockSpan(); + const error = new Error( + "failed https://user:password@example.test/path?access_token=secret", + ); + + ops.endSpan(mockSpan, error); + + assertEquals(mockSpan._status?.message?.includes("secret"), false); + assertEquals(mockSpan._exception?.message.includes("secret"), false); + assertEquals(error.message.includes("secret"), true); }); it("should handle null span gracefully", () => { @@ -193,6 +352,14 @@ describe("observability/tracing/span-operations", () => { }); describe("addEvent", () => { + it("bounds event names before invoking the provider", () => { + const mockSpan = createMockSpan(); + + ops.addEvent(mockSpan, "x".repeat(MAX_SPAN_NAME_LENGTH + 100)); + + assertEquals(mockSpan._events[0]?.name.length, MAX_SPAN_NAME_LENGTH); + }); + it("should add an event to a span", () => { const mockSpan = createMockSpan(); ops.addEvent(mockSpan, "user.action", { "user.id": "123" }); diff --git a/src/observability/tracing/span-operations.ts b/src/observability/tracing/span-operations.ts index f24ec57888..c1f93f8a67 100644 --- a/src/observability/tracing/span-operations.ts +++ b/src/observability/tracing/span-operations.ts @@ -1,8 +1,22 @@ import { serverLogger } from "#veryfront/utils"; +import { MAX_SPAN_NAME_LENGTH } from "#veryfront/utils/constants/index.ts"; +import { + sanitizeErrorForTelemetry, + sanitizeTelemetryAttributes, + sanitizeTelemetryText, +} from "../telemetry-error.ts"; import type { Context, OpenTelemetryAPI, Span, SpanKind, SpanOptions, Tracer } from "./types.ts"; const logger = serverLogger.component("tracing"); +function reportTelemetryFailure(message: string, error: unknown): void { + try { + logger.debug(message, error); + } catch (_) { + /* expected: telemetry and logging failures must not affect application work */ + } +} + export class SpanOperations { constructor( private api: OpenTelemetryAPI, @@ -11,37 +25,61 @@ export class SpanOperations { startSpan(name: string, options: SpanOptions = {}): Span | null { try { + const parent = this.resolveParent(options.parent); return this.tracer.startSpan( - name, + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), { kind: this.mapSpanKind(options.kind), - attributes: options.attributes ?? {}, + attributes: sanitizeTelemetryAttributes(options.attributes) ?? {}, }, - options.parent as Context | undefined, + parent, ); } catch (error) { - logger.debug("Failed to start span", { name, error }); + reportTelemetryFailure("Failed to start span", error); return null; } } - endSpan(span: Span | null, error?: Error): void { + endSpan(span: Span | null, error?: unknown): void { + this.finishSpan(span, error === undefined ? [] : [error]); + } + + /** @internal Preserve an observed failure even when JavaScript throws undefined. */ + endSpanWithFailure(span: Span | null, error: unknown): void { + this.finishSpan(span, [error]); + } + + private finishSpan(span: Span | null, failure: [] | [error: unknown]): void { if (!span) return; - try { - if (error) { - span.recordException(error); + if (failure.length > 0) { + const error = failure[0]; + const telemetryError = sanitizeErrorForTelemetry(error); + try { + span.recordException(telemetryError); + } catch (recordError) { + reportTelemetryFailure("Failed to record span exception", recordError); + } + try { span.setStatus({ code: this.api.SpanStatusCode.ERROR, - message: error.message, + message: telemetryError.message, }); - } else { + } catch (statusError) { + reportTelemetryFailure("Failed to set span error status", statusError); + } + } else { + try { span.setStatus({ code: this.api.SpanStatusCode.OK }); + } catch (statusError) { + reportTelemetryFailure("Failed to set span status", statusError); } + } + try { span.end(); - } catch (error) { - logger.debug("Failed to end span", error); + } catch (endError) { + reportTelemetryFailure("Failed to end span", endError); } } @@ -49,9 +87,9 @@ export class SpanOperations { if (!span) return; try { - span.setAttributes(attributes); + span.setAttributes(sanitizeTelemetryAttributes(attributes)); } catch (error) { - logger.debug("Failed to set span attributes", error); + reportTelemetryFailure("Failed to set span attributes", error); } } @@ -63,9 +101,12 @@ export class SpanOperations { if (!span) return; try { - span.addEvent(name, attributes); + span.addEvent( + sanitizeTelemetryText(name, MAX_SPAN_NAME_LENGTH), + sanitizeTelemetryAttributes(attributes), + ); } catch (error) { - logger.debug("Failed to add span event", error); + reportTelemetryFailure("Failed to add span event", error); } } @@ -76,7 +117,7 @@ export class SpanOperations { const parentContext = this.api.trace.setSpan(this.api.context.active(), parentSpan); return this.startSpan(name, { ...options, parent: parentContext }); } catch (error) { - logger.debug("Failed to create child span", error); + reportTelemetryFailure("Failed to create child span", error); return null; } } @@ -99,4 +140,14 @@ export class SpanOperations { return this.api.SpanKind.INTERNAL; } } + + private resolveParent(parent: SpanOptions["parent"]): Context | undefined { + if (!parent) return undefined; + try { + if (typeof (parent as Span).spanContext !== "function") return parent as Context; + return this.api.trace.setSpan(this.api.context.active(), parent as Span); + } catch (_) { + return parent as Context; + } + } }