diff --git a/js/.gitignore b/js/.gitignore index 7cdc90e3b..3cd1787c5 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -103,6 +103,14 @@ Chinook_Sqlite.sql /utils/jestlike.js /utils/jestlike.d.ts /utils/jestlike.d.cts +/experimental/otel/setup.cjs +/experimental/otel/setup.js +/experimental/otel/setup.d.ts +/experimental/otel/setup.d.cts +/experimental/otel/exporter.cjs +/experimental/otel/exporter.js +/experimental/otel/exporter.d.ts +/experimental/otel/exporter.d.cts /index.cjs /index.js /index.d.ts diff --git a/js/package.json b/js/package.json index cd666c52a..7064f0bcf 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.3.34", + "version": "0.3.35", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ @@ -77,6 +77,14 @@ "utils/jestlike.js", "utils/jestlike.d.ts", "utils/jestlike.d.cts", + "experimental/otel/setup.cjs", + "experimental/otel/setup.js", + "experimental/otel/setup.d.ts", + "experimental/otel/setup.d.cts", + "experimental/otel/exporter.cjs", + "experimental/otel/exporter.js", + "experimental/otel/exporter.d.ts", + "experimental/otel/exporter.d.cts", "index.cjs", "index.js", "index.d.ts", @@ -172,11 +180,23 @@ "zod": "^3.23.8" }, "peerDependencies": { - "openai": "*" + "openai": "*", + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*" }, "peerDependenciesMeta": { "openai": { "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true } }, "lint-staged": { @@ -357,6 +377,24 @@ "import": "./utils/jestlike.js", "require": "./utils/jestlike.cjs" }, + "./experimental/otel/setup": { + "types": { + "import": "./experimental/otel/setup.d.ts", + "require": "./experimental/otel/setup.d.cts", + "default": "./experimental/otel/setup.d.ts" + }, + "import": "./experimental/otel/setup.js", + "require": "./experimental/otel/setup.cjs" + }, + "./experimental/otel/exporter": { + "types": { + "import": "./experimental/otel/exporter.d.ts", + "require": "./experimental/otel/exporter.d.cts", + "default": "./experimental/otel/exporter.d.ts" + }, + "import": "./experimental/otel/exporter.js", + "require": "./experimental/otel/exporter.cjs" + }, "./package.json": "./package.json" } } diff --git a/js/scripts/create-entrypoints.js b/js/scripts/create-entrypoints.js index 417b569ed..de3e7fada 100644 --- a/js/scripts/create-entrypoints.js +++ b/js/scripts/create-entrypoints.js @@ -25,6 +25,8 @@ const entrypoints = { "wrappers/vercel": "wrappers/vercel", "singletons/traceable": "singletons/traceable", "utils/jestlike": "utils/jestlike/index", + "experimental/otel/setup": "experimental/otel/setup", + "experimental/otel/exporter": "experimental/otel/exporter", }; const defaultEntrypoints = [ diff --git a/js/src/client.ts b/js/src/client.ts index d8f9ab4a1..45bb690af 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -1,5 +1,14 @@ import * as uuid from "uuid"; - +import type { OTELContext } from "./experimental/otel/types.js"; +import { + LangSmithToOTELTranslator, + SerializedRunOperation, +} from "./experimental/otel/translator.js"; +import { + getDefaultOTLPTracerComponents, + getOTELTrace, + getOTELContext, +} from "./singletons/otel.js"; import { AsyncCaller, AsyncCallerParams } from "./utils/async_caller.js"; import { ComparativeExperiment, @@ -373,6 +382,7 @@ export type CreateProjectParams = { type AutoBatchQueueItem = { action: "create" | "update"; item: RunCreate | RunUpdate; + otelContext?: OTELContext; }; type MultipartPart = { @@ -487,6 +497,7 @@ export class AutoBatchQueue { items: { action: "create" | "update"; payload: RunCreate | RunUpdate; + otelContext?: OTELContext; itemPromiseResolve: () => void; itemPromise: Promise; size: number; @@ -512,6 +523,7 @@ export class AutoBatchQueue { this.items.push({ action: item.action, payload: item.item, + otelContext: item.otelContext, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion itemPromiseResolve: itemPromiseResolve!, itemPromise, @@ -548,7 +560,11 @@ export class AutoBatchQueue { this.sizeBytes -= item.size; } return [ - popped.map((it) => ({ action: it.action, item: it.payload })), + popped.map((it) => ({ + action: it.action, + item: it.payload, + otelContext: it.otelContext, + })), () => popped.forEach((it) => it.itemPromiseResolve()), ]; } @@ -608,6 +624,8 @@ export class Client implements LangSmithTracingClientInterface { private manualFlushMode = false; + private langSmithToOTELTranslator?: LangSmithToOTELTranslator; + debug = getEnvironmentVariable("LANGSMITH_DEBUG") === "true"; constructor(config: ClientConfig = {}) { @@ -653,6 +671,9 @@ export class Client implements LangSmithTracingClientInterface { this.batchSizeBytesLimit = config.batchSizeBytesLimit; this.fetchOptions = config.fetchOptions || {}; this.manualFlushMode = config.manualFlushMode ?? this.manualFlushMode; + if (getEnvironmentVariable("OTEL_ENABLED") === "true") { + this.langSmithToOTELTranslator = new LangSmithToOTELTranslator(); + } } public static getDefaultClientConfig(): { @@ -958,25 +979,59 @@ export class Client implements LangSmithTracingClientInterface { return; } try { - const ingestParams = { - runCreates: batch - .filter((item) => item.action === "create") - .map((item) => item.item) as RunCreate[], - runUpdates: batch - .filter((item) => item.action === "update") - .map((item) => item.item) as RunUpdate[], - }; - const serverInfo = await this._ensureServerInfo(); - if (serverInfo?.batch_ingest_config?.use_multipart_endpoint) { - await this.multipartIngestRuns(ingestParams); + if (this.langSmithToOTELTranslator !== undefined) { + this._sendBatchToOTELTranslator(batch); } else { - await this.batchIngestRuns(ingestParams); + const ingestParams = { + runCreates: batch + .filter((item) => item.action === "create") + .map((item) => item.item) as RunCreate[], + runUpdates: batch + .filter((item) => item.action === "update") + .map((item) => item.item) as RunUpdate[], + }; + const serverInfo = await this._ensureServerInfo(); + if (serverInfo?.batch_ingest_config?.use_multipart_endpoint) { + await this.multipartIngestRuns(ingestParams); + } else { + await this.batchIngestRuns(ingestParams); + } } + } catch (e) { + console.error("Error exporting batch:", e); } finally { done(); } } + private _sendBatchToOTELTranslator(batch: AutoBatchQueueItem[]) { + if (this.langSmithToOTELTranslator !== undefined) { + const otelContextMap = new Map(); + const operations: SerializedRunOperation[] = []; + for (const item of batch) { + if (item.item.id && item.otelContext) { + otelContextMap.set(item.item.id, item.otelContext); + if (item.action === "create") { + operations.push({ + operation: "post", + id: item.item.id, + trace_id: item.item.trace_id ?? item.item.id, + run: item.item as RunCreate, + }); + } else { + operations.push({ + operation: "patch", + id: item.item.id, + trace_id: item.item.trace_id ?? item.item.id, + run: item.item as RunUpdate, + }); + } + } + } + this.langSmithToOTELTranslator.exportBatch(operations, otelContextMap); + } + } + private async processRunOperation(item: AutoBatchQueueItem) { clearTimeout(this.autoBatchTimeout); this.autoBatchTimeout = undefined; @@ -1063,6 +1118,18 @@ export class Client implements LangSmithTracingClientInterface { await this.drainAutoBatchQueue(sizeLimitBytes); } + private _cloneCurrentOTELContext() { + const otel_trace = getOTELTrace(); + const otel_context = getOTELContext(); + if (this.langSmithToOTELTranslator !== undefined) { + const currentSpan = otel_trace.getActiveSpan(); + if (currentSpan) { + return otel_trace.setSpan(otel_context.active(), currentSpan); + } + } + return undefined; + } + public async createRun(run: CreateRunParams): Promise { if (!this._filterForSampling([run]).length) { return; @@ -1081,9 +1148,11 @@ export class Client implements LangSmithTracingClientInterface { runCreate.trace_id !== undefined && runCreate.dotted_order !== undefined ) { + const otelContext = this._cloneCurrentOTELContext(); void this.processRunOperation({ action: "create", item: runCreate, + otelContext, }).catch(console.error); return; } @@ -1511,6 +1580,7 @@ export class Client implements LangSmithTracingClientInterface { data.trace_id !== undefined && data.dotted_order !== undefined ) { + const otelContext = this._cloneCurrentOTELContext(); if ( run.end_time !== undefined && data.parent_run_id === undefined && @@ -1519,14 +1589,18 @@ export class Client implements LangSmithTracingClientInterface { ) { // Trigger batches as soon as a root trace ends and wait to ensure trace finishes // in serverless environments. - await this.processRunOperation({ action: "update", item: data }).catch( - console.error - ); + await this.processRunOperation({ + action: "update", + item: data, + otelContext, + }).catch(console.error); return; } else { - void this.processRunOperation({ action: "update", item: data }).catch( - console.error - ); + void this.processRunOperation({ + action: "update", + item: data, + otelContext, + }).catch(console.error); } return; } @@ -5055,17 +5129,20 @@ export class Client implements LangSmithTracingClientInterface { * * @returns A promise that resolves once all currently pending traces have sent. */ - public awaitPendingTraceBatches() { + public async awaitPendingTraceBatches() { if (this.manualFlushMode) { console.warn( "[WARNING]: When tracing in manual flush mode, you must call `await client.flush()` manually to submit trace batches." ); return Promise.resolve(); } - return Promise.all([ + await Promise.all([ ...this.autoBatchQueue.items.map(({ itemPromise }) => itemPromise), this.batchIngestCaller.queue.onIdle(), ]); + if (this.langSmithToOTELTranslator !== undefined) { + await getDefaultOTLPTracerComponents()?.DEFAULT_LANGSMITH_SPAN_PROCESSOR?.forceFlush(); + } } } diff --git a/js/src/experimental/otel/constants.ts b/js/src/experimental/otel/constants.ts new file mode 100644 index 000000000..af3a1e0e7 --- /dev/null +++ b/js/src/experimental/otel/constants.ts @@ -0,0 +1,62 @@ +// OpenTelemetry GenAI semantic convention attribute names +export const GEN_AI_OPERATION_NAME = "gen_ai.operation.name"; +export const GEN_AI_SYSTEM = "gen_ai.system"; +export const GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; +export const GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; +export const GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; +export const GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; +export const GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"; +export const GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; +export const GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; +export const GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"; +export const GEN_AI_REQUEST_FREQUENCY_PENALTY = + "gen_ai.request.frequency_penalty"; +export const GEN_AI_REQUEST_PRESENCE_PENALTY = + "gen_ai.request.presence_penalty"; +export const GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"; +export const GENAI_PROMPT = "gen_ai.prompt"; +export const GENAI_COMPLETION = "gen_ai.completion"; + +export const GEN_AI_REQUEST_EXTRA_QUERY = "gen_ai.request.extra_query"; +export const GEN_AI_REQUEST_EXTRA_BODY = "gen_ai.request.extra_body"; +export const GEN_AI_SERIALIZED_NAME = "gen_ai.serialized.name"; +export const GEN_AI_SERIALIZED_SIGNATURE = "gen_ai.serialized.signature"; +export const GEN_AI_SERIALIZED_DOC = "gen_ai.serialized.doc"; +export const GEN_AI_RESPONSE_ID = "gen_ai.response.id"; +export const GEN_AI_RESPONSE_SERVICE_TIER = "gen_ai.response.service_tier"; +export const GEN_AI_RESPONSE_SYSTEM_FINGERPRINT = + "gen_ai.response.system_fingerprint"; +export const GEN_AI_USAGE_INPUT_TOKEN_DETAILS = + "gen_ai.usage.input_token_details"; +export const GEN_AI_USAGE_OUTPUT_TOKEN_DETAILS = + "gen_ai.usage.output_token_details"; + +// LangSmith custom attributes +export const LANGSMITH_SESSION_ID = "langsmith.trace.session_id"; +export const LANGSMITH_SESSION_NAME = "langsmith.trace.session_name"; +export const LANGSMITH_RUN_TYPE = "langsmith.span.kind"; +export const LANGSMITH_NAME = "langsmith.trace.name"; +export const LANGSMITH_METADATA = "langsmith.metadata"; +export const LANGSMITH_TAGS = "langsmith.span.tags"; +export const LANGSMITH_RUNTIME = "langsmith.span.runtime"; +export const LANGSMITH_REQUEST_STREAMING = "langsmith.request.streaming"; +export const LANGSMITH_REQUEST_HEADERS = "langsmith.request.headers"; +export const LANGSMITH_RUN_ID = "langsmith.span.id"; +export const LANGSMITH_TRACE_ID = "langsmith.trace.id"; +export const LANGSMITH_DOTTED_ORDER = "langsmith.span.dotted_order"; +export const LANGSMITH_PARENT_RUN_ID = "langsmith.span.parent_id"; + +// GenAI event names +export const GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message"; +export const GEN_AI_USER_MESSAGE = "gen_ai.user.message"; +export const GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message"; +export const GEN_AI_CHOICE = "gen_ai.choice"; + +export const AI_SDK_LLM_OPERATIONS = [ + "ai.generateText.doGenerate", + "ai.streamText.doStream", + "ai.generateObject.doGenerate", + "ai.streamObject.doStream", +]; + +export const AI_SDK_TOOL_OPERATIONS = ["ai.toolCall"]; diff --git a/js/src/experimental/otel/exporter.ts b/js/src/experimental/otel/exporter.ts new file mode 100644 index 000000000..a0e69c28a --- /dev/null +++ b/js/src/experimental/otel/exporter.ts @@ -0,0 +1,150 @@ +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto"; +import { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import * as constants from "./constants.js"; +import { isTracingEnabled } from "../../env.js"; +import { + getEnvironmentVariable, + getLangSmithEnvironmentVariable, +} from "../../utils/env.js"; + +/** + * Convert headers string in format "name=value,name2=value2" to object + */ +function parseHeadersString(headersStr: string): Record { + const headers: Record = {}; + if (!headersStr) return headers; + + headersStr.split(",").forEach((pair) => { + const [name, ...valueParts] = pair.split("="); + if (name && valueParts.length > 0) { + headers[name.trim()] = valueParts.join("=").trim(); + } + }); + + return headers; +} + +/** + * LangSmith OpenTelemetry trace exporter that extends the standard OTLP trace exporter + * with LangSmith-specific configuration and span attribute transformations. + * + * This exporter automatically configures itself with LangSmith endpoints and API keys, + * based on your LANGSMITH_API_KEY and LANGSMITH_PROJECT environment variables. + * Will also respect OTEL_EXPORTER_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_HEADERS environment + * variables if set. + * + * @param config - Optional configuration object that accepts all OTLPTraceExporter parameters. + * If not provided, uses default LangSmith configuration: + * - `url`: Defaults to LangSmith OTEL endpoint (`${LANGSMITH_ENDPOINT}/otel/v1/traces`) + * - `headers`: Auto-configured with LangSmith API key and project headers + * Any provided config will override these defaults. + */ +export class LangSmithOTLPTraceExporter extends OTLPTraceExporter { + constructor(config?: ConstructorParameters[0]) { + const lsEndpoint = + getEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") || + getLangSmithEnvironmentVariable("ENDPOINT") || + "https://api.smith.langchain.com"; + const defaultBaseUrl = lsEndpoint.replace(/\/$/, ""); + const defaultUrl = `${defaultBaseUrl}/otel/v1/traces`; + // Configure headers with API key and project if available + let defaultHeaderString = + getEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS") ?? ""; + if (!defaultHeaderString) { + const apiKey = getLangSmithEnvironmentVariable("API_KEY"); + if (apiKey) { + defaultHeaderString = `x-api-key=${apiKey}`; + } + + const project = getLangSmithEnvironmentVariable("PROJECT"); + if (project) { + defaultHeaderString += `,Langsmith-Project=${project}`; + } + } + + super({ + url: defaultUrl, + headers: parseHeadersString(defaultHeaderString), + ...config, + }); + } + + export( + spans: ReadableSpan[], + resultCallback: Parameters[1] + ): void { + if (!isTracingEnabled()) { + return resultCallback({ code: 0 }); + } + for (const span of spans) { + if (!span.attributes[constants.GENAI_PROMPT]) { + if (span.attributes["ai.prompt"]) { + span.attributes[constants.GENAI_PROMPT] = + span.attributes["ai.prompt"]; + } + if ( + span.attributes["ai.prompt.messages"] && + typeof span.attributes["ai.prompt.messages"] === "string" + ) { + let messages; + try { + messages = JSON.parse(span.attributes["ai.prompt.messages"]); + } catch (e) { + console.error("Failed to parse ai.prompt.messages", e); + } + if (messages && Array.isArray(messages)) { + span.attributes[constants.GENAI_PROMPT] = JSON.stringify({ + input: messages, + }); + } + } + if (span.attributes["ai.toolCall.args"]) { + span.attributes[constants.GENAI_PROMPT] = + span.attributes["ai.toolCall.args"]; + } + } + if (!span.attributes[constants.GENAI_COMPLETION]) { + if (span.attributes["ai.response.text"]) { + span.attributes[constants.GENAI_COMPLETION] = + span.attributes["ai.response.text"]; + } + if (span.attributes["ai.response.choices"]) { + span.attributes[constants.GENAI_COMPLETION] = + span.attributes["ai.response.choices"]; + } + if (span.attributes["ai.response.object"]) { + span.attributes[constants.GENAI_COMPLETION] = + span.attributes["ai.response.object"]; + } + if (span.attributes["ai.response.toolCalls"]) { + span.attributes[constants.GENAI_COMPLETION] = + span.attributes["ai.response.toolCalls"]; + } + if (span.attributes["ai.toolCall.result"]) { + span.attributes[constants.GENAI_COMPLETION] = + span.attributes["ai.toolCall.result"]; + } + } + if ( + typeof span.attributes["ai.operationId"] === "string" && + constants.AI_SDK_LLM_OPERATIONS.includes( + span.attributes["ai.operationId"] + ) + ) { + span.attributes[constants.LANGSMITH_RUN_TYPE] = "llm"; + } else if ( + typeof span.attributes["ai.operationId"] === "string" && + constants.AI_SDK_TOOL_OPERATIONS.includes( + span.attributes["ai.operationId"] + ) + ) { + span.attributes[constants.LANGSMITH_RUN_TYPE] = "tool"; + if (span.attributes["ai.toolCall.name"]) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (span as any).name = span.attributes["ai.toolCall.name"]; + } + } + } + super.export(spans, resultCallback); + } +} diff --git a/js/src/experimental/otel/setup.ts b/js/src/experimental/otel/setup.ts new file mode 100644 index 000000000..1bd2c26cf --- /dev/null +++ b/js/src/experimental/otel/setup.ts @@ -0,0 +1,109 @@ +// Avoid async hooks as even a peer dep to avoid pulling into +// non-node build environments. +// eslint-disable-next-line import/no-extraneous-dependencies +import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; +import { + trace as otel_trace, + context as otel_context, + type TracerProvider, + type ContextManager, +} from "@opentelemetry/api"; +import { + BatchSpanProcessor, + BasicTracerProvider, +} from "@opentelemetry/sdk-trace-base"; +import { LangSmithOTLPTraceExporter } from "./exporter.js"; + +import { + setDefaultOTLPTracerComponents, + setOTELInstances, +} from "../../singletons/otel.js"; + +/** + * Initializes OpenTelemetry with LangSmith-specific configuration for tracing. + * + * Call this once at the start of your application to enable tracing integration. Sets global + * OpenTelemetry components including the tracer provider and context manager. + * + * Requires the following peer dependencies to be installed: + * - @opentelemetry/api + * - @opentelemetry/sdk-trace-base + * - @opentelemetry/exporter-trace-otlp-proto + * - @opentelemetry/context-async-hooks + * + * @param options - Configuration options + * @param options.globalTracerProvider - Optional custom TracerProvider to use instead of creating and globally setting a new one + * @returns Object containing the initialized OTEL components (tracer provider, span processor, exporter) + * + * @example + * ```typescript + * import { initializeOTEL } from "langsmith/experimental/otel/setup"; + * initializeOTEL(); + * ``` + * + * @example With custom tracer provider + * ```typescript + * import { initializeOTEL } from "langsmith/experimental/otel/setup"; + * import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; + * + * const customProvider = new BasicTracerProvider(); + * initializeOTEL({ globalTracerProvider: customProvider }); + * ``` + */ +export const initializeOTEL = ({ + globalTracerProvider, + globalContextManager, +}: { + globalTracerProvider?: TracerProvider; + globalContextManager?: ContextManager; +} = {}) => { + const otel = { + trace: otel_trace, + context: otel_context, + }; + + setOTELInstances(otel); + + if (!globalContextManager) { + const contextManager = new AsyncHooksContextManager(); + contextManager.enable(); + otel_context.setGlobalContextManager(contextManager); + } + + const DEFAULT_LANGSMITH_SPAN_EXPORTER = new LangSmithOTLPTraceExporter({}); + + const DEFAULT_LANGSMITH_SPAN_PROCESSOR = new BatchSpanProcessor( + DEFAULT_LANGSMITH_SPAN_EXPORTER + ); + + if (!globalTracerProvider) { + const DEFAULT_LANGSMITH_TRACER_PROVIDER = new BasicTracerProvider({ + spanProcessors: [DEFAULT_LANGSMITH_SPAN_PROCESSOR], + }); + + const defaultComponents = { + DEFAULT_LANGSMITH_SPAN_PROCESSOR, + DEFAULT_LANGSMITH_TRACER_PROVIDER, + DEFAULT_LANGSMITH_SPAN_EXPORTER, + }; + + // If user has set global tracer before, this fails and returns false + const globalSuccessfullyOverridden = otel_trace.setGlobalTracerProvider( + defaultComponents.DEFAULT_LANGSMITH_TRACER_PROVIDER + ); + + if (globalSuccessfullyOverridden) { + setDefaultOTLPTracerComponents(defaultComponents); + } + + return defaultComponents; + } else { + const defaultComponents = { + DEFAULT_LANGSMITH_TRACER_PROVIDER: globalTracerProvider, + DEFAULT_LANGSMITH_SPAN_PROCESSOR, + DEFAULT_LANGSMITH_SPAN_EXPORTER, + }; + setDefaultOTLPTracerComponents(defaultComponents); + return defaultComponents; + } +}; diff --git a/js/src/experimental/otel/translator.ts b/js/src/experimental/otel/translator.ts new file mode 100644 index 000000000..a8e00d085 --- /dev/null +++ b/js/src/experimental/otel/translator.ts @@ -0,0 +1,574 @@ +import type { OTELContext, OTELSpan } from "./types.js"; +import type { KVMap, RunCreate, RunUpdate } from "../../schemas.js"; +import * as constants from "./constants.js"; +import { getOTELTrace } from "../../singletons/otel.js"; + +const WELL_KNOWN_OPERATION_NAMES: Record = { + llm: "chat", + tool: "execute_tool", + retriever: "embeddings", + embedding: "embeddings", + prompt: "chat", +}; + +function getOperationName(runType: string): string { + return WELL_KNOWN_OPERATION_NAMES[runType] || runType; +} + +export type SerializedRunOperation< + T extends "post" | "patch" = "post" | "patch" +> = { + operation: T; + id: string; + trace_id: string; + run: T extends "post" ? RunCreate : RunUpdate; +}; + +export class LangSmithToOTELTranslator { + private spans: Map = new Map(); + + exportBatch( + operations: SerializedRunOperation[], + otelContextMap: Map + ): void { + for (const op of operations) { + try { + if (!op.run) { + continue; + } + + if (op.operation === "post") { + const span = this.createSpanForRun( + op, + op.run as RunCreate, + otelContextMap.get(op.id) + ); + if (span && !op.run.end_time) { + this.spans.set(op.id, span); + } + } else { + this.updateSpanForRun(op, op.run); + } + } catch (e) { + console.error(`Error processing operation ${op.id}:`, e); + } + } + } + + private createSpanForRun( + op: SerializedRunOperation, + runInfo: RunCreate, + otelContext?: OTELContext + ): OTELSpan | undefined { + const activeSpan = otelContext && getOTELTrace().getSpan(otelContext); + if (!activeSpan) { + return; + } + try { + return this.finishSpanSetup(activeSpan, runInfo, op); + } catch (e) { + console.error(`Failed to create span for run ${op.id}:`, e); + return undefined; + } + } + + private finishSpanSetup( + span: OTELSpan, + runInfo: RunCreate | RunUpdate, + op: SerializedRunOperation + ): OTELSpan { + // Set all attributes + this.setSpanAttributes(span, runInfo, op); + + // Set status based on error + if (runInfo.error) { + span.setStatus({ code: 2 }); // ERROR status + span.recordException(new Error(runInfo.error)); + } else { + span.setStatus({ code: 1 }); // OK status + } + + // End the span if end_time is present + if (runInfo.end_time) { + span.end(runInfo.end_time); + } + + return span; + } + + private updateSpanForRun( + op: SerializedRunOperation, + runInfo: RunUpdate + ): void { + try { + const span = this.spans.get(op.id); + if (!span) { + console.debug(`No span found for run ${op.id} during update`); + return; + } + + // Update attributes + this.setSpanAttributes(span, runInfo, op); + + // Update status based on error + if (runInfo.error) { + span.setStatus({ code: 2 }); // ERROR status + span.recordException(new Error(runInfo.error)); + } else { + span.setStatus({ code: 1 }); // OK status + } + + // End the span if end_time is present + const endTime = runInfo.end_time; + if (endTime) { + span.end(endTime); + this.spans.delete(op.id); + } + } catch (e) { + console.error(`Failed to update span for run ${op.id}:`, e); + } + } + + private extractModelName(runInfo: RunCreate | RunUpdate) { + // Try to get model name from metadata + if (runInfo.extra?.metadata) { + const metadata = runInfo.extra.metadata; + + // First check for ls_model_name in metadata + if (metadata.ls_model_name) { + return metadata.ls_model_name; + } + + // Then check invocation_params for model info + if (metadata.invocation_params) { + const invocationParams = metadata.invocation_params; + if (invocationParams.model) { + return invocationParams.model; + } else if (invocationParams.model_name) { + return invocationParams.model_name; + } + } + } + + return; + } + + private setSpanAttributes( + span: OTELSpan, + runInfo: RunCreate | RunUpdate, + op: SerializedRunOperation + ): void { + if ("run_type" in runInfo && runInfo.run_type) { + span.setAttribute(constants.LANGSMITH_RUN_TYPE, runInfo.run_type); + // Set GenAI attributes according to OTEL semantic conventions + const operationName = getOperationName(runInfo.run_type || "chain"); + span.setAttribute(constants.GEN_AI_OPERATION_NAME, operationName); + } + + if ("name" in runInfo && runInfo.name) { + span.setAttribute(constants.LANGSMITH_NAME, runInfo.name); + } + + if ("session_id" in runInfo && runInfo.session_id) { + span.setAttribute(constants.LANGSMITH_SESSION_ID, runInfo.session_id); + } + + if ("session_name" in runInfo && runInfo.session_name) { + span.setAttribute(constants.LANGSMITH_SESSION_NAME, runInfo.session_name); + } + + // Set gen_ai.system + this.setGenAiSystem(span, runInfo); + + // Set model name if available + const modelName = this.extractModelName(runInfo); + if (modelName) { + span.setAttribute(constants.GEN_AI_REQUEST_MODEL, modelName); + } + + // Set token usage information + if ( + "prompt_tokens" in runInfo && + typeof runInfo.prompt_tokens === "number" + ) { + span.setAttribute( + constants.GEN_AI_USAGE_INPUT_TOKENS, + runInfo.prompt_tokens + ); + } + + if ( + "completion_tokens" in runInfo && + typeof runInfo.completion_tokens === "number" + ) { + span.setAttribute( + constants.GEN_AI_USAGE_OUTPUT_TOKENS, + runInfo.completion_tokens + ); + } + + if ("total_tokens" in runInfo && typeof runInfo.total_tokens === "number") { + span.setAttribute( + constants.GEN_AI_USAGE_TOTAL_TOKENS, + runInfo.total_tokens + ); + } + + // Set other parameters from invocation_params + this.setInvocationParameters(span, runInfo); + + // Set metadata and tags if available + const metadata = runInfo.extra?.metadata || {}; + for (const [key, value] of Object.entries(metadata)) { + if (value !== null && value !== undefined) { + span.setAttribute( + `${constants.LANGSMITH_METADATA}.${key}`, + String(value) + ); + } + } + + const tags = runInfo.tags; + if (tags && Array.isArray(tags)) { + span.setAttribute(constants.LANGSMITH_TAGS, tags.join(", ")); + } else if (tags) { + span.setAttribute(constants.LANGSMITH_TAGS, String(tags)); + } + + // Support additional serialized attributes, if present + if ("serialized" in runInfo && typeof runInfo.serialized === "object") { + const serialized = runInfo.serialized as KVMap; + if (serialized.name) { + span.setAttribute( + constants.GEN_AI_SERIALIZED_NAME, + String(serialized.name) + ); + } + if (serialized.signature) { + span.setAttribute( + constants.GEN_AI_SERIALIZED_SIGNATURE, + String(serialized.signature) + ); + } + if (serialized.doc) { + span.setAttribute( + constants.GEN_AI_SERIALIZED_DOC, + String(serialized.doc) + ); + } + } + + // Set inputs/outputs if available + this.setIOAttributes(span, op); + } + + private setGenAiSystem(span: OTELSpan, runInfo: RunCreate | RunUpdate): void { + // Default to "langchain" if we can't determine the system + let system = "langchain"; + + // Extract model name to determine the system + const modelName = this.extractModelName(runInfo); + if (modelName) { + const modelLower = modelName.toLowerCase(); + if (modelLower.includes("anthropic") || modelLower.startsWith("claude")) { + system = "anthropic"; + } else if (modelLower.includes("bedrock")) { + system = "aws.bedrock"; + } else if ( + modelLower.includes("azure") && + modelLower.includes("openai") + ) { + system = "az.ai.openai"; + } else if ( + modelLower.includes("azure") && + modelLower.includes("inference") + ) { + system = "az.ai.inference"; + } else if (modelLower.includes("cohere")) { + system = "cohere"; + } else if (modelLower.includes("deepseek")) { + system = "deepseek"; + } else if (modelLower.includes("gemini")) { + system = "gemini"; + } else if (modelLower.includes("groq")) { + system = "groq"; + } else if (modelLower.includes("watson") || modelLower.includes("ibm")) { + system = "ibm.watsonx.ai"; + } else if (modelLower.includes("mistral")) { + system = "mistral_ai"; + } else if (modelLower.includes("gpt") || modelLower.includes("openai")) { + system = "openai"; + } else if ( + modelLower.includes("perplexity") || + modelLower.includes("sonar") + ) { + system = "perplexity"; + } else if (modelLower.includes("vertex")) { + system = "vertex_ai"; + } else if (modelLower.includes("xai") || modelLower.includes("grok")) { + system = "xai"; + } + } + + span.setAttribute(constants.GEN_AI_SYSTEM, system); + } + + private setInvocationParameters( + span: OTELSpan, + runInfo: RunCreate | RunUpdate + ): void { + if (!runInfo.extra?.metadata?.invocation_params) { + return; + } + + const invocationParams = runInfo.extra.metadata.invocation_params; + + // Set relevant invocation parameters + if (invocationParams.max_tokens !== undefined) { + span.setAttribute( + constants.GEN_AI_REQUEST_MAX_TOKENS, + invocationParams.max_tokens + ); + } + + if (invocationParams.temperature !== undefined) { + span.setAttribute( + constants.GEN_AI_REQUEST_TEMPERATURE, + invocationParams.temperature + ); + } + + if (invocationParams.top_p !== undefined) { + span.setAttribute(constants.GEN_AI_REQUEST_TOP_P, invocationParams.top_p); + } + + if (invocationParams.frequency_penalty !== undefined) { + span.setAttribute( + constants.GEN_AI_REQUEST_FREQUENCY_PENALTY, + invocationParams.frequency_penalty + ); + } + + if (invocationParams.presence_penalty !== undefined) { + span.setAttribute( + constants.GEN_AI_REQUEST_PRESENCE_PENALTY, + invocationParams.presence_penalty + ); + } + } + + private setIOAttributes(span: OTELSpan, op: SerializedRunOperation): void { + if (op.run.inputs) { + try { + const inputs = op.run.inputs; + + if (typeof inputs === "object" && inputs !== null) { + if (inputs.model && Array.isArray(inputs.messages)) { + span.setAttribute(constants.GEN_AI_REQUEST_MODEL, inputs.model); + } + + // Set additional request attributes if available + if (inputs.stream !== undefined) { + span.setAttribute( + constants.LANGSMITH_REQUEST_STREAMING, + inputs.stream + ); + } + if (inputs.extra_headers) { + span.setAttribute( + constants.LANGSMITH_REQUEST_HEADERS, + JSON.stringify(inputs.extra_headers) + ); + } + if (inputs.extra_query) { + span.setAttribute( + constants.GEN_AI_REQUEST_EXTRA_QUERY, + JSON.stringify(inputs.extra_query) + ); + } + if (inputs.extra_body) { + span.setAttribute( + constants.GEN_AI_REQUEST_EXTRA_BODY, + JSON.stringify(inputs.extra_body) + ); + } + } + + span.setAttribute(constants.GENAI_PROMPT, JSON.stringify(inputs)); + } catch (e) { + console.debug(`Failed to process inputs for run ${op.id}`, e); + } + } + + if (op.run.outputs) { + try { + const outputs = op.run.outputs; + + // Extract token usage from outputs (for LLM runs) + const tokenUsage = this.getUnifiedRunTokens(outputs); + if (tokenUsage) { + span.setAttribute(constants.GEN_AI_USAGE_INPUT_TOKENS, tokenUsage[0]); + span.setAttribute( + constants.GEN_AI_USAGE_OUTPUT_TOKENS, + tokenUsage[1] + ); + span.setAttribute( + constants.GEN_AI_USAGE_TOTAL_TOKENS, + tokenUsage[0] + tokenUsage[1] + ); + } + + if (outputs && typeof outputs === "object") { + if (outputs.model) { + span.setAttribute( + constants.GEN_AI_RESPONSE_MODEL, + String(outputs.model) + ); + } + + // Extract additional response attributes + if (outputs.id) { + span.setAttribute(constants.GEN_AI_RESPONSE_ID, outputs.id); + } + + if (outputs.choices && Array.isArray(outputs.choices)) { + const finishReasons = outputs.choices + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((choice: any) => choice.finish_reason) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((reason: any) => reason) + .map(String); + if (finishReasons.length > 0) { + span.setAttribute( + constants.GEN_AI_RESPONSE_FINISH_REASONS, + finishReasons.join(", ") + ); + } + } + + if (outputs.service_tier) { + span.setAttribute( + constants.GEN_AI_RESPONSE_SERVICE_TIER, + outputs.service_tier + ); + } + + if (outputs.system_fingerprint) { + span.setAttribute( + constants.GEN_AI_RESPONSE_SYSTEM_FINGERPRINT, + outputs.system_fingerprint + ); + } + + if ( + outputs.usage_metadata && + typeof outputs.usage_metadata === "object" + ) { + const usageMetadata = outputs.usage_metadata; + if (usageMetadata.input_token_details) { + span.setAttribute( + constants.GEN_AI_USAGE_INPUT_TOKEN_DETAILS, + JSON.stringify(usageMetadata.input_token_details) + ); + } + if (usageMetadata.output_token_details) { + span.setAttribute( + constants.GEN_AI_USAGE_OUTPUT_TOKEN_DETAILS, + JSON.stringify(usageMetadata.output_token_details) + ); + } + } + } + + span.setAttribute(constants.GENAI_COMPLETION, JSON.stringify(outputs)); + } catch (e) { + console.debug(`Failed to process outputs for run ${op.id}`, e); + } + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private getUnifiedRunTokens(outputs: any): [number, number] | null { + if (!outputs) { + return null; + } + + // Search in non-generations lists + let tokenUsage = this.extractUnifiedRunTokens(outputs.usage_metadata); + if (tokenUsage) { + return tokenUsage; + } + + // Find if direct kwarg in outputs + const keys = Object.keys(outputs); + for (const key of keys) { + const haystack = outputs[key]; + if (!haystack || typeof haystack !== "object") { + continue; + } + + tokenUsage = this.extractUnifiedRunTokens(haystack.usage_metadata); + if (tokenUsage) { + return tokenUsage; + } + + if ( + haystack.lc === 1 && + haystack.kwargs && + typeof haystack.kwargs === "object" + ) { + tokenUsage = this.extractUnifiedRunTokens( + haystack.kwargs.usage_metadata + ); + if (tokenUsage) { + return tokenUsage; + } + } + } + + // Find in generations + const generations = outputs.generations || []; + if (!Array.isArray(generations)) { + return null; + } + + const flatGenerations = Array.isArray(generations[0]) + ? generations.flat() + : generations; + + for (const generation of flatGenerations) { + if ( + typeof generation === "object" && + generation.message && + typeof generation.message === "object" && + generation.message.kwargs && + typeof generation.message.kwargs === "object" + ) { + tokenUsage = this.extractUnifiedRunTokens( + generation.message.kwargs.usage_metadata + ); + if (tokenUsage) { + return tokenUsage; + } + } + } + + return null; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private extractUnifiedRunTokens(outputs: any): [number, number] | null { + if (!outputs || typeof outputs !== "object") { + return null; + } + + if ( + typeof outputs.input_tokens !== "number" || + typeof outputs.output_tokens !== "number" + ) { + return null; + } + + return [outputs.input_tokens, outputs.output_tokens]; + } +} diff --git a/js/src/experimental/otel/types.ts b/js/src/experimental/otel/types.ts new file mode 100644 index 000000000..19b906cad --- /dev/null +++ b/js/src/experimental/otel/types.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Local type definitions to avoid importing @opentelemetry/api +// These match the OpenTelemetry API interfaces but don't require the package to be installed + +export interface OTELSpan { + setAttribute: (key: string, value: any) => void; + setStatus: (status: { code: number; message?: string }) => void; + recordException: (exception: Error | string) => void; + end: (endTime?: number) => void; +} + +export interface OTELTracer { + startSpan: (name: string, options?: any) => OTELSpan; + startActiveSpan: (name: string, options: any, fn: () => any) => any; +} + +export interface OTELContext { + // Context type - opaque object in OpenTelemetry +} + +export interface OTELTracerProvider { + getTracer: (name: string, version?: string) => OTELTracer; +} + +export interface OTELSpanContext { + traceId: string; + spanId: string; + isRemote?: boolean; + traceFlags: number; + traceState?: any; +} + +export interface OTELTraceFlags { + NONE: number; + SAMPLED: number; +} diff --git a/js/src/experimental/otel/utils.ts b/js/src/experimental/otel/utils.ts new file mode 100644 index 000000000..8638e2a1c --- /dev/null +++ b/js/src/experimental/otel/utils.ts @@ -0,0 +1,37 @@ +import type { OTELSpanContext } from "./types.js"; + +/** + * Get OpenTelemetry trace ID as hex string from UUID. + * @param uuidStr - The UUID string to convert + * @returns Hex string representation of the trace ID + */ +export function getOtelTraceIdFromUuid(uuidStr: string): string { + // Use full UUID hex (like Python's uuid_val.hex) + return uuidStr.replace(/-/g, ""); +} + +/** + * Get OpenTelemetry span ID as hex string from UUID. + * @param uuidStr - The UUID string to convert + * @returns Hex string representation of the span ID + */ +export function getOtelSpanIdFromUuid(uuidStr: string): string { + // Convert UUID string to bytes equivalent (first 8 bytes for span ID) + // Like Python's uuid_val.bytes[:8].hex() + const cleanUuid = uuidStr.replace(/-/g, ""); + return cleanUuid.substring(0, 16); // First 8 bytes (16 hex chars) +} + +export function createOtelSpanContextFromRun(run: { + trace_id?: string; + id: string; +}): OTELSpanContext { + const traceId = getOtelTraceIdFromUuid(run.trace_id ?? run.id); + const spanId = getOtelSpanIdFromUuid(run.id); + return { + traceId, + spanId, + isRemote: false, + traceFlags: 1, // SAMPLED + }; +} diff --git a/js/src/index.ts b/js/src/index.ts index d875bade7..2ec009ea1 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -20,4 +20,4 @@ export { overrideFetchImplementation } from "./singletons/fetch.js"; export { getDefaultProjectName } from "./utils/project.js"; // Update using yarn bump-version -export const __version__ = "0.3.34"; +export const __version__ = "0.3.35"; diff --git a/js/src/singletons/otel.ts b/js/src/singletons/otel.ts new file mode 100644 index 000000000..49da7e0ad --- /dev/null +++ b/js/src/singletons/otel.ts @@ -0,0 +1,200 @@ +// Should not import any OTEL packages to avoid pulling in optional deps. + +import { getEnvironmentVariable } from "../utils/env.js"; + +interface OTELTraceInterface { + getTracer: (name: string, version?: string) => any; + getActiveSpan: () => any; + setSpan: (context: any, span: any) => any; + getSpan: (context: any) => any; + setSpanContext: (context: any, spanContext: any) => any; + getTracerProvider: () => any; + setGlobalTracerProvider: (tracerProvider: any) => boolean; +} + +interface OTELContextInterface { + active: () => any; + with: (context: any, fn: () => T) => T; +} + +interface OTELInterface { + trace: OTELTraceInterface; + context: OTELContextInterface; +} + +class MockTracer { + private hasWarned = false; + + startActiveSpan(_name: string, ...args: any[]): T | undefined { + if (!this.hasWarned && getEnvironmentVariable("OTEL_ENABLED") === "true") { + console.warn( + "You have enabled OTEL export via the `OTEL_ENABLED` environment variable, but have not initialized the required OTEL instances. " + + 'Please add:\n```\nimport { initializeOTEL } from "langsmith/experimental/otel/setup";\ninitializeOTEL();\n```\nat the beginning of your code.' + ); + this.hasWarned = true; + } + + // Handle different overloads: + // startActiveSpan(name, fn) + // startActiveSpan(name, options, fn) + // startActiveSpan(name, options, context, fn) + let fn: ((...args: any[]) => T) | undefined; + + if (args.length === 1 && typeof args[0] === "function") { + fn = args[0]; + } else if (args.length === 2 && typeof args[1] === "function") { + fn = args[1]; + } else if (args.length === 3 && typeof args[2] === "function") { + fn = args[2]; + } + + if (typeof fn === "function") { + return fn(); + } + return undefined; + } +} + +class MockOTELTrace implements OTELTraceInterface { + private mockTracer = new MockTracer(); + + getTracer(_name: string, _version?: string) { + return this.mockTracer; + } + + getActiveSpan() { + return undefined; + } + + setSpan(context: any, _span: any) { + return context; + } + + getSpan(_context: any) { + return undefined; + } + + setSpanContext(context: any, _spanContext: any) { + return context; + } + + getTracerProvider() { + return undefined; + } + + setGlobalTracerProvider(_tracerProvider: any) { + return false; + } +} + +class MockOTELContext implements OTELContextInterface { + active() { + return {}; + } + + with(_context: any, fn: () => T): T { + return fn(); + } +} + +const OTEL_TRACE_KEY = Symbol.for("ls:otel_trace"); +const OTEL_CONTEXT_KEY = Symbol.for("ls:otel_context"); +const OTEL_GET_DEFAULT_OTLP_TRACER_PROVIDER_KEY = Symbol.for( + "ls:otel_get_default_otlp_tracer_provider" +); + +const mockOTELTrace = new MockOTELTrace(); +const mockOTELContext = new MockOTELContext(); + +class OTELProvider { + getTraceInstance(): OTELTraceInterface { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (globalThis as any)[OTEL_TRACE_KEY] ?? mockOTELTrace; + } + + getContextInstance(): OTELContextInterface { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (globalThis as any)[OTEL_CONTEXT_KEY] ?? mockOTELContext; + } + + initializeGlobalInstances(otel: OTELInterface) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((globalThis as any)[OTEL_TRACE_KEY] === undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any)[OTEL_TRACE_KEY] = otel.trace; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((globalThis as any)[OTEL_CONTEXT_KEY] === undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any)[OTEL_CONTEXT_KEY] = otel.context; + } + } + + setDefaultOTLPTracerComponents(components: { + DEFAULT_LANGSMITH_SPAN_PROCESSOR: any; + DEFAULT_LANGSMITH_TRACER_PROVIDER: any; + DEFAULT_LANGSMITH_SPAN_EXPORTER: any; + }) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any)[OTEL_GET_DEFAULT_OTLP_TRACER_PROVIDER_KEY] = components; + } + + getDefaultOTLPTracerComponents() { + return ( + (globalThis as any)[OTEL_GET_DEFAULT_OTLP_TRACER_PROVIDER_KEY] ?? + undefined + ); + } +} + +export const OTELProviderSingleton = new OTELProvider(); + +/** + * Get the current OTEL trace instance. + * Returns a mock implementation if OTEL is not available. + */ +export function getOTELTrace(): OTELTraceInterface { + return OTELProviderSingleton.getTraceInstance(); +} + +/** + * Get the current OTEL context instance. + * Returns a mock implementation if OTEL is not available. + */ +export function getOTELContext(): OTELContextInterface { + return OTELProviderSingleton.getContextInstance(); +} + +/** + * Initialize the global OTEL instances. + * Should be called once when OTEL packages are available. + */ +export function setOTELInstances(otel: OTELInterface): void { + OTELProviderSingleton.initializeGlobalInstances(otel); +} + +/** + * Set a getter function for the default OTLP tracer provider. + * This allows lazy initialization of the tracer provider. + */ +export function setDefaultOTLPTracerComponents(components: { + DEFAULT_LANGSMITH_SPAN_PROCESSOR: any; + DEFAULT_LANGSMITH_TRACER_PROVIDER: any; + DEFAULT_LANGSMITH_SPAN_EXPORTER: any; +}): void { + OTELProviderSingleton.setDefaultOTLPTracerComponents(components); +} + +/** + * Get the default OTLP tracer provider instance. + * Returns undefined if not set. + */ +export function getDefaultOTLPTracerComponents(): + | { + DEFAULT_LANGSMITH_SPAN_PROCESSOR: any; + DEFAULT_LANGSMITH_TRACER_PROVIDER: any; + DEFAULT_LANGSMITH_SPAN_EXPORTER: any; + } + | undefined { + return OTELProviderSingleton.getDefaultOTLPTracerComponents(); +} diff --git a/js/src/tests/otel/ai_sdk_extended.int.test.ts b/js/src/tests/otel/ai_sdk_extended.int.test.ts new file mode 100644 index 000000000..490cfce41 --- /dev/null +++ b/js/src/tests/otel/ai_sdk_extended.int.test.ts @@ -0,0 +1,194 @@ +/* eslint-disable no-process-env */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { z } from "zod"; +import { openai } from "@ai-sdk/openai"; +import { generateObject, streamObject, streamText } from "ai"; +import { v4 as uuidv4 } from "uuid"; +import { Client } from "../../client.js"; +import { traceable } from "../../traceable.js"; +import { getLangSmithEnvironmentVariable } from "../../utils/env.js"; +import { toArray, waitUntilRunFoundByMetaField } from "../utils.js"; + +// Initialize basic OTEL setup +import { initializeOTEL } from "../../experimental/otel/setup.js"; + +initializeOTEL(); + +describe("AI SDK Streaming Integration", () => { + beforeAll(() => { + process.env.LANGCHAIN_TRACING = "true"; + }); + + afterAll(() => { + delete process.env.OTEL_ENABLED; + delete process.env.LANGCHAIN_TRACING; + }); + + it("works with streamText", async () => { + process.env.OTEL_ENABLED = "true"; + const meta = uuidv4(); + const client = new Client(); + + const wrappedStreamText = traceable( + async (prompt: string) => { + const { textStream } = await streamText({ + model: openai("gpt-4.1-nano"), + prompt, + experimental_telemetry: { + isEnabled: true, + }, + }); + + let fullText = ""; + for await (const textPart of textStream) { + fullText += textPart; + } + + return { text: fullText }; + }, + { name: "streamTextTest", metadata: { testKey: meta }, client } + ); + + const result = await wrappedStreamText("Say hello in exactly 3 words"); + expect(result.text).toBeTruthy(); + + await client.awaitPendingTraceBatches(); + const projectName = getLangSmithEnvironmentVariable("PROJECT") ?? "default"; + await waitUntilRunFoundByMetaField(client, projectName, "testKey", meta); + + const storedRuns = await toArray( + client.listRuns({ + projectName, + filter: `and(eq(metadata_key, "testKey"), eq(metadata_value, "${meta}"))`, + }) + ); + expect(storedRuns.length).toBe(1); + + const runWithChildren = await client.readRun(storedRuns[0].id, { + loadChildRuns: true, + }); + expect(runWithChildren.child_runs?.length).toBeGreaterThan(0); + expect( + runWithChildren.child_runs?.some((run) => run.name === "ai.streamText") + ).toBe(true); + }); + + it("works with generateObject", async () => { + process.env.OTEL_ENABLED = "true"; + const meta = uuidv4(); + const client = new Client(); + + const schema = z.object({ + name: z.string(), + age: z.number(), + city: z.string(), + }); + + const wrappedGenerateObject = traceable( + async (prompt: string) => { + const { object } = await generateObject({ + model: openai("gpt-4.1-nano"), + prompt, + schema, + experimental_telemetry: { + isEnabled: true, + }, + }); + + return object; + }, + { name: "generateObjectTest", metadata: { testKey: meta }, client } + ); + + const result = await wrappedGenerateObject( + "Generate a person with name John, age 30, and city New York" + ); + expect(result.name).toBe("John"); + expect(result.age).toBe(30); + expect(result.city).toBe("New York"); + + await client.awaitPendingTraceBatches(); + const projectName = getLangSmithEnvironmentVariable("PROJECT") ?? "default"; + await waitUntilRunFoundByMetaField(client, projectName, "testKey", meta); + + const storedRuns = await toArray( + client.listRuns({ + projectName, + filter: `and(eq(metadata_key, "testKey"), eq(metadata_value, "${meta}"))`, + }) + ); + expect(storedRuns.length).toBe(1); + + const runWithChildren = await client.readRun(storedRuns[0].id, { + loadChildRuns: true, + }); + expect(runWithChildren.child_runs?.length).toBeGreaterThan(0); + expect( + runWithChildren.child_runs?.some( + (run) => run.name === "ai.generateObject" + ) + ).toBe(true); + }); + + it("works with streamObject", async () => { + process.env.OTEL_ENABLED = "true"; + const meta = uuidv4(); + const client = new Client(); + + const schema = z.object({ + characters: z.array( + z.object({ + name: z.string(), + trait: z.string(), + }) + ), + }); + + const wrappedStreamObject = traceable( + async (prompt: string) => { + const { partialObjectStream } = await streamObject({ + model: openai("gpt-4.1-nano"), + prompt, + schema, + experimental_telemetry: { + isEnabled: true, + }, + }); + + let finalObject: any = {}; + for await (const partialObject of partialObjectStream) { + finalObject = partialObject; + } + + return finalObject; + }, + { name: "streamObjectTest", metadata: { testKey: meta }, client } + ); + + const result = await wrappedStreamObject( + "Generate 2 fantasy characters with names and traits" + ); + expect(result.characters).toBeDefined(); + expect(Array.isArray(result.characters)).toBe(true); + + await client.awaitPendingTraceBatches(); + const projectName = getLangSmithEnvironmentVariable("PROJECT") ?? "default"; + await waitUntilRunFoundByMetaField(client, projectName, "testKey", meta); + + const storedRuns = await toArray( + client.listRuns({ + projectName, + filter: `and(eq(metadata_key, "testKey"), eq(metadata_value, "${meta}"))`, + }) + ); + expect(storedRuns.length).toBe(1); + + const runWithChildren = await client.readRun(storedRuns[0].id, { + loadChildRuns: true, + }); + expect(runWithChildren.child_runs?.length).toBeGreaterThan(0); + expect( + runWithChildren.child_runs?.some((run) => run.name === "ai.streamObject") + ).toBe(true); + }); +}); diff --git a/js/src/tests/otel/traceable_otel.int.test.ts b/js/src/tests/otel/traceable_otel.int.test.ts new file mode 100644 index 000000000..bbd210745 --- /dev/null +++ b/js/src/tests/otel/traceable_otel.int.test.ts @@ -0,0 +1,153 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable no-process-env */ +import { generateText, tool } from "ai"; +import { openai } from "@ai-sdk/openai"; +import { z } from "zod"; +import { v4 as uuidv4 } from "uuid"; + +import { Client } from "../../client.js"; +import { traceable } from "../../traceable.js"; +import { toArray, waitUntilRunFoundByMetaField } from "../utils.js"; +import { getLangSmithEnvironmentVariable } from "../../utils/env.js"; + +// Initialize basic OTEL setup +import { initializeOTEL } from "../../experimental/otel/setup.js"; + +initializeOTEL(); + +describe("Traceable OTEL Integration Tests", () => { + beforeEach(() => { + process.env.LANGCHAIN_TRACING = "true"; + }); + + afterEach(() => { + delete process.env.OTEL_ENABLED; + delete process.env.LANGCHAIN_TRACING; + }); + + it("handles nested calls with OTEL context", async () => { + process.env.OTEL_ENABLED = "true"; + + const meta = uuidv4(); + const client = new Client(); + + const childFunction = traceable( + async (input: string) => { + return { content: `child: ${input}`, role: "assistant" }; + }, + { name: "child-function", run_type: "llm", client } + ); + + const parentFunction = traceable( + async (input: string) => { + const childResult = await childFunction(input); + return `parent: ${childResult.content}`; + }, + { name: "parent-function", metadata: { hackyKey: meta } } + ); + + const result = await parentFunction("test"); + expect(result).toBe("parent: child: test"); + + await client.awaitPendingTraceBatches(); + + const projectName = getLangSmithEnvironmentVariable("PROJECT") ?? "default"; + await waitUntilRunFoundByMetaField( + client, + projectName, + "hackyKey", + meta, + true + ); + const storedRun = await toArray( + client.listRuns({ + projectName, + filter: `and(eq(metadata_key, "hackyKey"), eq(metadata_value, "${meta}"))`, + }) + ); + expect(storedRun.length).toBe(1); + const runWithChildren = await client.readRun(storedRun[0].id, { + loadChildRuns: true, + }); + expect(runWithChildren.child_runs?.length).toBe(1); + expect(runWithChildren.child_runs?.[0].name).toBe("child-function"); + }); + + it("works with AI SDK", async () => { + process.env.OTEL_ENABLED = "true"; + + const meta = uuidv4(); + const client = new Client(); + const wrappedText = traceable( + async (content: string) => { + const { text } = await generateText({ + model: openai("gpt-4.1-nano"), + messages: [{ role: "user", content }], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => { + const getOrderNumber = traceable( + async () => { + return "1234"; + }, + { name: "getOrderNumber" } + ); + const orderNumber = await getOrderNumber(); + return `User ${userId} has the following orders: ${orderNumber}`; + }, + }), + viewTrackingInformation: tool({ + description: "view tracking information for a specific order", + parameters: z.object({ orderId: z.string() }), + execute: async ({ orderId }) => + `Here is the tracking information for ${orderId}`, + }), + }, + experimental_telemetry: { + isEnabled: true, + }, + maxSteps: 10, + }); + + return { text }; + }, + { name: "parentTraceable", metadata: { hackyKey: meta }, client } + ); + + await wrappedText( + "What are my orders and where are they? My user ID is 123. Use available tools." + ); + await client.awaitPendingTraceBatches(); + const projectName = getLangSmithEnvironmentVariable("PROJECT") ?? "default"; + await waitUntilRunFoundByMetaField( + client, + projectName, + "hackyKey", + meta, + true + ); + const storedRun = await toArray( + client.listRuns({ + projectName, + filter: `and(eq(metadata_key, "hackyKey"), eq(metadata_value, "${meta}"))`, + }) + ); + expect(storedRun.length).toBe(1); + const runWithChildren = await client.readRun(storedRun[0].id, { + loadChildRuns: true, + }); + expect(runWithChildren.child_runs?.length).toBe(1); + expect(runWithChildren.child_runs?.[0].name).toBe("ai.generateText"); + expect(runWithChildren.child_runs?.[0].child_runs?.[0].name).toBe( + "ai.generateText.doGenerate" + ); + const listToolRuns = runWithChildren.child_runs?.[0].child_runs?.filter( + (run) => run.name === "listOrders" + ); + expect(listToolRuns?.length).toBeGreaterThan(0); + expect(listToolRuns?.[0].name).toBe("listOrders"); + expect(listToolRuns?.[0].child_runs?.[0].name).toBe("getOrderNumber"); + }); +}); diff --git a/js/src/tests/traceable.int.test.ts b/js/src/tests/traceable.int.test.ts index 84b353238..1941e3b1b 100644 --- a/js/src/tests/traceable.int.test.ts +++ b/js/src/tests/traceable.int.test.ts @@ -749,7 +749,7 @@ test("Test upload attachments and process inputs.", async () => { multipartIngestRunsSpy.mockRestore(); }, 60000); -test.only("Test trace to multiple projects with replicas", async () => { +test("Test trace to multiple projects with replicas", async () => { const client = new Client({ callerOptions: { maxRetries: 6 } }); const datasetName = `__test_trace_to_multiple_projects${uuidv4().slice( 0, diff --git a/js/src/tests/utils.ts b/js/src/tests/utils.ts index faf458542..6c2da2e94 100644 --- a/js/src/tests/utils.ts +++ b/js/src/tests/utils.ts @@ -105,6 +105,44 @@ export async function waitUntilRunFound( ); } +export async function waitUntilRunFoundByMetaField( + client: Client, + projectName: string, + metaKey: string, + metaValue: string, + checkOutputs = false +) { + return waitUntil( + async () => { + try { + const runs = await toArray( + client.listRuns({ + filter: `and(eq(metadata_key, "${metaKey}"), eq(metadata_value, "${metaValue}"))`, + projectName, + }) + ); + if (runs.length === 0) { + return false; + } + const run = runs[0]; + if (checkOutputs) { + return ( + run.outputs !== null && + run.outputs !== undefined && + Object.keys(run.outputs).length !== 0 + ); + } + return true; + } catch (e) { + return false; + } + }, + 30_000, + 5_000, + `Waiting for run with metadata.${metaKey} = "${metaValue}"` + ); +} + export async function waitUntilProjectFound( client: Client, projectName: string diff --git a/js/src/traceable.ts b/js/src/traceable.ts index 3de912b58..02585457e 100644 --- a/js/src/traceable.ts +++ b/js/src/traceable.ts @@ -29,11 +29,56 @@ import { isGenerator, isPromiseMethod, } from "./utils/asserts.js"; +import { getEnvironmentVariable } from "./utils/env.js"; +import { __version__ } from "./index.js"; +import { getOTELTrace, getOTELContext } from "./singletons/otel.js"; +import { createOtelSpanContextFromRun } from "./experimental/otel/utils.js"; +import { OTELTracer } from "./experimental/otel/types.js"; AsyncLocalStorageProviderSingleton.initializeGlobalInstance( new AsyncLocalStorage() ); +/** + * Create OpenTelemetry context manager from RunTree if OTEL is enabled. + */ +function maybeCreateOtelContext( + runTree?: RunTree, + tracer?: OTELTracer + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): ((fn: (...args: any[]) => T) => T) | undefined { + if (!runTree || getEnvironmentVariable("OTEL_ENABLED") !== "true") { + return; + } + + const otel_trace = getOTELTrace(); + const otel_context = getOTELContext(); + + try { + const spanContext = createOtelSpanContextFromRun(runTree); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (fn: (...args: any[]) => T) => { + const resolvedTracer = + tracer ?? otel_trace.getTracer("langsmith", __version__); + return resolvedTracer.startActiveSpan( + runTree.name, + { + attributes: { + "langsmith.traceable": "true", + }, + }, + () => { + otel_trace.setSpanContext(otel_context.active(), spanContext); + return fn(); + } + ); + }; + } catch { + // Silent failure if OTEL setup is incomplete + return; + } +} + const runInputsToMap = (rawInputs: unknown[]) => { const firstInput = rawInputs[0]; let inputs: KVMap; @@ -396,6 +441,7 @@ export function traceable any>( // eslint-disable-next-line @typescript-eslint/no-explicit-any aggregator?: (args: any[]) => any; argsConfigPath?: [number] | [number, string]; + tracer?: OTELTracer; __finalTracedIteratorKey?: string; /** @@ -598,7 +644,13 @@ export function traceable any>( return [currentRunTree, processedArgs as Inputs]; })(); - return asyncLocalStorage.run(currentRunTree, () => { + const otelContextManager = maybeCreateOtelContext( + currentRunTree, + config?.tracer + ); + const otel_context = getOTELContext(); + + const runWithContext = () => { const postRunPromise = currentRunTree?.postRun(); async function handleChunks(chunks: unknown[]) { @@ -620,14 +672,17 @@ export function traceable any>( const reader = stream.getReader(); let finished = false; const chunks: unknown[] = []; + const capturedOtelContext = otel_context.active(); const tappedStream = new ReadableStream({ async start(controller) { // eslint-disable-next-line no-constant-condition while (true) { const result = await (snapshot - ? snapshot(() => reader.read()) - : reader.read()); + ? snapshot(() => + otel_context.with(capturedOtelContext, () => reader.read()) + ) + : otel_context.with(capturedOtelContext, () => reader.read())); if (result.done) { finished = true; const processedOutputs = handleRunOutputs({ @@ -673,11 +728,14 @@ export function traceable any>( ) { let finished = false; const chunks: unknown[] = []; + const capturedOtelContext = otel_context.active(); try { while (true) { const { value, done } = await (snapshot - ? snapshot(() => iterator.next()) - : iterator.next()); + ? snapshot(() => + otel_context.with(capturedOtelContext, () => iterator.next()) + ) + : otel_context.with(capturedOtelContext, () => iterator.next())); if (done) { finished = true; break; @@ -875,7 +933,16 @@ export function traceable any>( return Reflect.get(target, prop, receiver); }, }); - }); + }; + + // Wrap with OTEL context if available, similar to Python's implementation + if (otelContextManager) { + return asyncLocalStorage.run(currentRunTree, () => + otelContextManager(runWithContext) + ); + } else { + return asyncLocalStorage.run(currentRunTree, runWithContext); + } }; Object.defineProperty(traceableFunc, "langsmith:traceable", { diff --git a/js/src/utils/_uuid.ts b/js/src/utils/_uuid.ts index 51d71f020..65491f80b 100644 --- a/js/src/utils/_uuid.ts +++ b/js/src/utils/_uuid.ts @@ -1,7 +1,11 @@ -import * as uuid from "uuid"; +// Relaxed UUID validation regex (allows any valid UUID format including nil UUIDs) +const UUID_REGEX = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; export function assertUuid(str: string, which?: string): string { - if (!uuid.validate(str)) { + // Use relaxed regex validation instead of strict uuid.validate() + // This allows edge cases like nil UUIDs or test UUIDs that might not pass strict validation + if (!UUID_REGEX.test(str)) { const msg = which !== undefined ? `Invalid UUID for ${which}: ${str}` diff --git a/js/tsconfig.json b/js/tsconfig.json index 47fb12c55..15712677b 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -50,7 +50,9 @@ "src/wrappers/openai.ts", "src/wrappers/vercel.ts", "src/singletons/traceable.ts", - "src/utils/jestlike/index.ts" + "src/utils/jestlike/index.ts", + "src/experimental/otel/setup.ts", + "src/experimental/otel/exporter.ts" ] } }