feat(js): Add OTEL support

js/.gitignore

@@ -103,6 +103,14 @@ Chinook_Sqlite.sql
 /utils/jestlike.js
 /utils/jestlike.d.ts
 /utils/jestlike.d.cts
+/experimental/otel/setup.cjs
+/experimental/otel/setup.js
+/experimental/otel/setup.d.ts
+/experimental/otel/setup.d.cts
+/experimental/otel/exporter.cjs
+/experimental/otel/exporter.js
+/experimental/otel/exporter.d.ts
+/experimental/otel/exporter.d.cts
 /index.cjs
 /index.js
 /index.d.ts

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.34",
+  "version": "0.3.35",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -77,6 +77,14 @@
     "utils/jestlike.js",
     "utils/jestlike.d.ts",
     "utils/jestlike.d.cts",
+    "experimental/otel/setup.cjs",
+    "experimental/otel/setup.js",
+    "experimental/otel/setup.d.ts",
+    "experimental/otel/setup.d.cts",
+    "experimental/otel/exporter.cjs",
+    "experimental/otel/exporter.js",
+    "experimental/otel/exporter.d.ts",
+    "experimental/otel/exporter.d.cts",
     "index.cjs",
     "index.js",
     "index.d.ts",
@@ -172,11 +180,23 @@
     "zod": "^3.23.8"
   },
   "peerDependencies": {
-    "openai": "*"
+    "openai": "*",
+    "@opentelemetry/api": "*",
+    "@opentelemetry/exporter-trace-otlp-proto": "*",
+    "@opentelemetry/sdk-trace-base": "*"
   },
   "peerDependenciesMeta": {
     "openai": {
       "optional": true
+    },
+    "@opentelemetry/api": {
+      "optional": true
+    },
+    "@opentelemetry/exporter-trace-otlp-proto": {
+      "optional": true
+    },
+    "@opentelemetry/sdk-trace-base": {
+      "optional": true
     }
   },
   "lint-staged": {
@@ -357,6 +377,24 @@
       "import": "./utils/jestlike.js",
       "require": "./utils/jestlike.cjs"
     },
+    "./experimental/otel/setup": {
+      "types": {
+        "import": "./experimental/otel/setup.d.ts",
+        "require": "./experimental/otel/setup.d.cts",
+        "default": "./experimental/otel/setup.d.ts"
+      },
+      "import": "./experimental/otel/setup.js",
+      "require": "./experimental/otel/setup.cjs"
+    },
+    "./experimental/otel/exporter": {
+      "types": {
+        "import": "./experimental/otel/exporter.d.ts",
+        "require": "./experimental/otel/exporter.d.cts",
+        "default": "./experimental/otel/exporter.d.ts"
+      },
+      "import": "./experimental/otel/exporter.js",
+      "require": "./experimental/otel/exporter.cjs"
+    },
     "./package.json": "./package.json"
   }
 }

js/scripts/create-entrypoints.js

@@ -25,6 +25,8 @@ const entrypoints = {
   "wrappers/vercel": "wrappers/vercel",
   "singletons/traceable": "singletons/traceable",
   "utils/jestlike": "utils/jestlike/index",
+  "experimental/otel/setup": "experimental/otel/setup",
+  "experimental/otel/exporter": "experimental/otel/exporter",
 };
 
 const defaultEntrypoints = [

js/src/client.ts

@@ -1,5 +1,14 @@
 import * as uuid from "uuid";
-
+import type { OTELContext } from "./experimental/otel/types.js";
+import {
+  LangSmithToOTELTranslator,
+  SerializedRunOperation,
+} from "./experimental/otel/translator.js";
+import {
+  getDefaultOTLPTracerComponents,
+  getOTELTrace,
+  getOTELContext,
+} from "./singletons/otel.js";
 import { AsyncCaller, AsyncCallerParams } from "./utils/async_caller.js";
 import {
   ComparativeExperiment,
@@ -373,6 +382,7 @@
 type AutoBatchQueueItem = {
   action: "create" | "update";
   item: RunCreate | RunUpdate;
+  otelContext?: OTELContext;
 };
 
 type MultipartPart = {
@@ -389,7 +399,7 @@
   max_start_time: string;
   latency_p50: number;
   latency_p99: number;
   feedback_stats: any | null;
   group_key: string;
   first_inputs: string;
   last_outputs: string;
@@ -487,6 +497,7 @@
   items: {
     action: "create" | "update";
     payload: RunCreate | RunUpdate;
+    otelContext?: OTELContext;
     itemPromiseResolve: () => void;
     itemPromise: Promise<void>;
     size: number;
@@ -512,6 +523,7 @@
     this.items.push({
       action: item.action,
       payload: item.item,
+      otelContext: item.otelContext,
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       itemPromiseResolve: itemPromiseResolve!,
       itemPromise,
@@ -542,13 +554,17 @@
     // If there is an item on the queue we were unable to pop,
     // just return it as a single batch.
     if (popped.length === 0 && this.items.length > 0) {
       const item = this.items.shift()!;
       popped.push(item);
       poppedSizeBytes += item.size;
       this.sizeBytes -= item.size;
     }
     return [
-      popped.map((it) => ({ action: it.action, item: it.payload })),
+      popped.map((it) => ({
+        action: it.action,
+        item: it.payload,
+        otelContext: it.otelContext,
+      })),
       () => popped.forEach((it) => it.itemPromiseResolve()),
     ];
   }
@@ -608,6 +624,8 @@
 
   private manualFlushMode = false;
 
+  private langSmithToOTELTranslator?: LangSmithToOTELTranslator;
+
   debug = getEnvironmentVariable("LANGSMITH_DEBUG") === "true";
 
   constructor(config: ClientConfig = {}) {
@@ -653,6 +671,9 @@
     this.batchSizeBytesLimit = config.batchSizeBytesLimit;
     this.fetchOptions = config.fetchOptions || {};
     this.manualFlushMode = config.manualFlushMode ?? this.manualFlushMode;
+    if (getEnvironmentVariable("OTEL_ENABLED") === "true") {
+      this.langSmithToOTELTranslator = new LangSmithToOTELTranslator();
+    }
   }
 
   public static getDefaultClientConfig(): {
@@ -958,25 +979,59 @@
       return;
     }
     try {
-      const ingestParams = {
-        runCreates: batch
-          .filter((item) => item.action === "create")
-          .map((item) => item.item) as RunCreate[],
-        runUpdates: batch
-          .filter((item) => item.action === "update")
-          .map((item) => item.item) as RunUpdate[],
-      };
-      const serverInfo = await this._ensureServerInfo();
-      if (serverInfo?.batch_ingest_config?.use_multipart_endpoint) {
-        await this.multipartIngestRuns(ingestParams);
+      if (this.langSmithToOTELTranslator !== undefined) {
+        this._sendBatchToOTELTranslator(batch);
       } else {
-        await this.batchIngestRuns(ingestParams);
+        const ingestParams = {
+          runCreates: batch
+            .filter((item) => item.action === "create")
+            .map((item) => item.item) as RunCreate[],
+          runUpdates: batch
+            .filter((item) => item.action === "update")
+            .map((item) => item.item) as RunUpdate[],
+        };
+        const serverInfo = await this._ensureServerInfo();
+        if (serverInfo?.batch_ingest_config?.use_multipart_endpoint) {
+          await this.multipartIngestRuns(ingestParams);
+        } else {
+          await this.batchIngestRuns(ingestParams);
+        }
       }
+    } catch (e) {
+      console.error("Error exporting batch:", e);
     } finally {
       done();
     }
   }
 
+  private _sendBatchToOTELTranslator(batch: AutoBatchQueueItem[]) {
+    if (this.langSmithToOTELTranslator !== undefined) {
+      const otelContextMap = new Map<string, OTELContext>();
+      const operations: SerializedRunOperation[] = [];
+      for (const item of batch) {
+        if (item.item.id && item.otelContext) {
+          otelContextMap.set(item.item.id, item.otelContext);
+          if (item.action === "create") {
+            operations.push({
+              operation: "post",
+              id: item.item.id,
+              trace_id: item.item.trace_id ?? item.item.id,
+              run: item.item as RunCreate,
+            });
+          } else {
+            operations.push({
+              operation: "patch",
+              id: item.item.id,
+              trace_id: item.item.trace_id ?? item.item.id,
+              run: item.item as RunUpdate,
+            });
+          }
+        }
+      }
+      this.langSmithToOTELTranslator.exportBatch(operations, otelContextMap);
+    }
+  }
+
   private async processRunOperation(item: AutoBatchQueueItem) {
     clearTimeout(this.autoBatchTimeout);
     this.autoBatchTimeout = undefined;
@@ -1030,7 +1085,7 @@
         if (this._serverInfo === undefined) {
           try {
             this._serverInfo = await this._getServerInfo();
           } catch (e: any) {
             console.warn(
               `[WARNING]: LangSmith failed to fetch info on supported operations with status code ${e.status}. Falling back to batch operations and default limits.`
             );
@@ -1063,6 +1118,18 @@
     await this.drainAutoBatchQueue(sizeLimitBytes);
   }
 
+  private _cloneCurrentOTELContext() {
+    const otel_trace = getOTELTrace();
+    const otel_context = getOTELContext();
+    if (this.langSmithToOTELTranslator !== undefined) {
+      const currentSpan = otel_trace.getActiveSpan();
+      if (currentSpan) {
+        return otel_trace.setSpan(otel_context.active(), currentSpan);
+      }
+    }
+    return undefined;
+  }
+
   public async createRun(run: CreateRunParams): Promise<void> {
     if (!this._filterForSampling([run]).length) {
       return;
@@ -1081,9 +1148,11 @@
       runCreate.trace_id !== undefined &&
       runCreate.dotted_order !== undefined
     ) {
+      const otelContext = this._cloneCurrentOTELContext();
       void this.processRunOperation({
         action: "create",
         item: runCreate,
+        otelContext,
       }).catch(console.error);
       return;
     }
@@ -1173,7 +1242,7 @@
       let batchItem = batchItems.pop();
       while (batchItem !== undefined) {
         // Type is wrong but this is a deprecated code path anyway
         batchChunks[key].push(batchItem as any);
         batchItem = batchItems.pop();
       }
     }
@@ -1511,6 +1580,7 @@
       data.trace_id !== undefined &&
       data.dotted_order !== undefined
     ) {
+      const otelContext = this._cloneCurrentOTELContext();
       if (
         run.end_time !== undefined &&
         data.parent_run_id === undefined &&
@@ -1519,14 +1589,18 @@
       ) {
         // Trigger batches as soon as a root trace ends and wait to ensure trace finishes
         // in serverless environments.
-        await this.processRunOperation({ action: "update", item: data }).catch(
-          console.error
-        );
+        await this.processRunOperation({
+          action: "update",
+          item: data,
+          otelContext,
+        }).catch(console.error);
         return;
       } else {
-        void this.processRunOperation({ action: "update", item: data }).catch(
-          console.error
-        );
+        void this.processRunOperation({
+          action: "update",
+          item: data,
+          otelContext,
+        }).catch(console.error);
       }
       return;
     }
@@ -1932,7 +2006,7 @@
     treeFilter?: string;
     isRoot?: boolean;
     dataSourceType?: string;
   }): Promise<any> {
     let projectIds_ = projectIds || [];
     if (projectNames) {
       projectIds_ = [
@@ -2224,7 +2298,7 @@
         `Failed to list shared examples: ${response.status} ${response.statusText}`
       );
     }
     return result.map((example: any) => ({
       ...example,
       _hostUrl: this.getHostUrl(),
     }));
@@ -2354,7 +2428,7 @@
       }
       // projectId querying
       return true;
     } catch (e) {
       return false;
     }
   }
@@ -3941,7 +4015,7 @@
       | EvaluationResult[]
       | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]> {
     const evalResults: Array<EvaluationResult> =
       this._selectEvalResults(evaluatorResponse);
@@ -3983,7 +4057,7 @@
       | EvaluationResult[]
       | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<EvaluationResult[]> {
     const [results] = await this._logEvaluationFeedback(
       evaluatorResponse,
@@ -4493,7 +4567,7 @@
 
   public async createCommit(
     promptIdentifier: string,
     object: any,
     options?: {
       parentCommitHash?: string;
     }
@@ -5055,17 +5129,20 @@
    *
    * @returns A promise that resolves once all currently pending traces have sent.
    */
-  public awaitPendingTraceBatches() {
+  public async awaitPendingTraceBatches() {
     if (this.manualFlushMode) {
       console.warn(
         "[WARNING]: When tracing in manual flush mode, you must call `await client.flush()` manually to submit trace batches."
       );
       return Promise.resolve();
     }
-    return Promise.all([
+    await Promise.all([
       ...this.autoBatchQueue.items.map(({ itemPromise }) => itemPromise),
       this.batchIngestCaller.queue.onIdle(),
     ]);
+    if (this.langSmithToOTELTranslator !== undefined) {
+      await getDefaultOTLPTracerComponents()?.DEFAULT_LANGSMITH_SPAN_PROCESSOR?.forceFlush();
+    }
   }
 }
 

js/src/experimental/otel/constants.ts

@@ -0,0 +1,62 @@
+// OpenTelemetry GenAI semantic convention attribute names
+export const GEN_AI_OPERATION_NAME = "gen_ai.operation.name";
+export const GEN_AI_SYSTEM = "gen_ai.system";
+export const GEN_AI_REQUEST_MODEL = "gen_ai.request.model";
+export const GEN_AI_RESPONSE_MODEL = "gen_ai.response.model";
+export const GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens";
+export const GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens";
+export const GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens";
+export const GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens";
+export const GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature";
+export const GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p";
+export const GEN_AI_REQUEST_FREQUENCY_PENALTY =
+  "gen_ai.request.frequency_penalty";
+export const GEN_AI_REQUEST_PRESENCE_PENALTY =
+  "gen_ai.request.presence_penalty";
+export const GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons";
+export const GENAI_PROMPT = "gen_ai.prompt";
+export const GENAI_COMPLETION = "gen_ai.completion";
+
+export const GEN_AI_REQUEST_EXTRA_QUERY = "gen_ai.request.extra_query";
+export const GEN_AI_REQUEST_EXTRA_BODY = "gen_ai.request.extra_body";
+export const GEN_AI_SERIALIZED_NAME = "gen_ai.serialized.name";
+export const GEN_AI_SERIALIZED_SIGNATURE = "gen_ai.serialized.signature";
+export const GEN_AI_SERIALIZED_DOC = "gen_ai.serialized.doc";
+export const GEN_AI_RESPONSE_ID = "gen_ai.response.id";
+export const GEN_AI_RESPONSE_SERVICE_TIER = "gen_ai.response.service_tier";
+export const GEN_AI_RESPONSE_SYSTEM_FINGERPRINT =
+  "gen_ai.response.system_fingerprint";
+export const GEN_AI_USAGE_INPUT_TOKEN_DETAILS =
+  "gen_ai.usage.input_token_details";
+export const GEN_AI_USAGE_OUTPUT_TOKEN_DETAILS =
+  "gen_ai.usage.output_token_details";
+
+// LangSmith custom attributes
+export const LANGSMITH_SESSION_ID = "langsmith.trace.session_id";
+export const LANGSMITH_SESSION_NAME = "langsmith.trace.session_name";
+export const LANGSMITH_RUN_TYPE = "langsmith.span.kind";
+export const LANGSMITH_NAME = "langsmith.trace.name";
+export const LANGSMITH_METADATA = "langsmith.metadata";
+export const LANGSMITH_TAGS = "langsmith.span.tags";
+export const LANGSMITH_RUNTIME = "langsmith.span.runtime";
+export const LANGSMITH_REQUEST_STREAMING = "langsmith.request.streaming";
+export const LANGSMITH_REQUEST_HEADERS = "langsmith.request.headers";
+export const LANGSMITH_RUN_ID = "langsmith.span.id";
+export const LANGSMITH_TRACE_ID = "langsmith.trace.id";
+export const LANGSMITH_DOTTED_ORDER = "langsmith.span.dotted_order";
+export const LANGSMITH_PARENT_RUN_ID = "langsmith.span.parent_id";
+
+// GenAI event names
+export const GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message";
+export const GEN_AI_USER_MESSAGE = "gen_ai.user.message";
+export const GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message";
+export const GEN_AI_CHOICE = "gen_ai.choice";
+
+export const AI_SDK_LLM_OPERATIONS = [
+  "ai.generateText.doGenerate",
+  "ai.streamText.doStream",
+  "ai.generateObject.doGenerate",
+  "ai.streamObject.doStream",
+];
+
+export const AI_SDK_TOOL_OPERATIONS = ["ai.toolCall"];