From 145a1a5acb32a5c0c8996cbb456063a2834b97e1 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Fri, 31 Jul 2026 21:04:48 -0700 Subject: [PATCH 1/4] feat(otel): export traces to PostHog in production Mirrors the existing PostHog log export in the OpenTelemetry pipeline so spans land on the PostHog tracing page alongside logs and errors. --- src/instrumentation.test.ts | 34 +++++++++++++++++++++++++++++++++ src/instrumentation.ts | 38 ++++++++++++++++++++++++++++++++----- src/lib/posthog-config.ts | 2 ++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/instrumentation.test.ts b/src/instrumentation.test.ts index a3e941e673..c32302d155 100644 --- a/src/instrumentation.test.ts +++ b/src/instrumentation.test.ts @@ -205,6 +205,40 @@ describe("instrumentation", () => { ); }); + it("adds a PostHog trace exporter in production even without Axiom endpoints", async () => { + const { startInstrumentation } = await import("./instrumentation.ts"); + + startInstrumentation({ + DEPLOY_ENVIRONMENT: "production", + }); + + const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + expect(config?.spanProcessors).toHaveLength(1); + expect(config?.instrumentations).toEqual([mockAutoInstrumentations]); + expect(BatchSpanProcessor).toHaveBeenCalledWith(expect.any(OTLPTraceExporter)); + expect(OTLPTraceExporter).toHaveBeenCalledWith( + expect.objectContaining({ + url: expect.stringContaining("/i/v1/traces"), + headers: expect.objectContaining({ + Authorization: expect.stringContaining("Bearer "), + }), + }), + ); + }); + + it("keeps Axiom and PostHog trace exporters alongside each other in production with an Axiom endpoint", async () => { + const { startInstrumentation } = await import("./instrumentation.ts"); + + startInstrumentation({ + DEPLOY_ENVIRONMENT: "production", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + }); + + const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + expect(config?.spanProcessors).toHaveLength(2); + expect(BatchSpanProcessor).toHaveBeenCalledTimes(2); + }); + it("only configures trace processors and auto instrumentations when only traces endpoint exists", async () => { const { startInstrumentation } = await import("./instrumentation.ts"); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 1d28b389b5..c42c262d6e 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -6,14 +6,42 @@ import { resourceFromAttributes } from "@opentelemetry/resources"; import { BatchLogRecordProcessor, type LogRecordProcessor } from "@opentelemetry/sdk-logs"; import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; import { NodeSDK } from "@opentelemetry/sdk-node"; -import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; +import { BatchSpanProcessor, type SpanProcessor } from "@opentelemetry/sdk-trace-node"; import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; -import { POSTHOG_API_KEY, POSTHOG_LOGS_URL } from "./lib/posthog-config.ts"; +import { POSTHOG_API_KEY, POSTHOG_LOGS_URL, POSTHOG_TRACES_URL } from "./lib/posthog-config.ts"; function isProductionDeployment(environment: string | undefined): boolean { return environment === "prod" || environment === "production"; } +function createSpanProcessors(env: Record): SpanProcessor[] { + const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT; + const tracesEndpoint = env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + const hasAxiomTraceExport = Boolean(endpoint || tracesEndpoint); + const hasPostHogTraceExport = isProductionDeployment(env.DEPLOY_ENVIRONMENT); + + const processors: SpanProcessor[] = []; + + if (hasAxiomTraceExport) { + processors.push(new BatchSpanProcessor(new OTLPTraceExporter())); + } + + if (hasPostHogTraceExport) { + processors.push( + new BatchSpanProcessor( + new OTLPTraceExporter({ + url: POSTHOG_TRACES_URL, + headers: { + Authorization: `Bearer ${POSTHOG_API_KEY}`, + }, + }), + ), + ); + } + + return processors; +} + function createLogRecordProcessors(env: Record): LogRecordProcessor[] { const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT; const logsEndpoint = env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; @@ -54,10 +82,10 @@ export function startInstrumentation( env: Record = process.env, ): NodeSDK | undefined { const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT; - const tracesEndpoint = env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; const metricsEndpoint = env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT; - const hasTraceExport = Boolean(endpoint || tracesEndpoint); + const spanProcessors = createSpanProcessors(env); + const hasTraceExport = spanProcessors.length > 0; const logRecordProcessors = createLogRecordProcessors(env); const hasLogExport = logRecordProcessors.length > 0; const hasMetricExport = Boolean(endpoint || metricsEndpoint); @@ -71,7 +99,7 @@ export function startInstrumentation( resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName, }), - spanProcessors: hasTraceExport ? [new BatchSpanProcessor(new OTLPTraceExporter())] : [], + spanProcessors, logRecordProcessors, metricReader: hasMetricExport ? new PeriodicExportingMetricReader({ diff --git a/src/lib/posthog-config.ts b/src/lib/posthog-config.ts index a8cc28df79..99c373604a 100644 --- a/src/lib/posthog-config.ts +++ b/src/lib/posthog-config.ts @@ -4,3 +4,5 @@ export const POSTHOG_API_KEY = "phc_GsvyihTLSXrWGKYYGz84m44nuT59kYEwEXNnI0JICtg" export const POSTHOG_HOST = "https://us.i.posthog.com"; export const POSTHOG_LOGS_URL = `${POSTHOG_HOST}/i/v1/logs`; + +export const POSTHOG_TRACES_URL = `${POSTHOG_HOST}/i/v1/traces`; From 4ceaf6e8ed926b43ecc7cbd2643fc58c33025994 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Sat, 1 Aug 2026 09:52:21 -0700 Subject: [PATCH 2/4] fix(otel): address PR review on PostHog trace export - Robustify NodeSDK config assertions (use last call, not first) so tests are order-independent against module auto-start. - Assert exact PostHog traces URL and bearer key in tests. - Parameterize production exporter test over both prod and production DEPLOY_ENVIRONMENT values. - Set DEPLOY_ENVIRONMENT for cdc-health and metric-stream-clickhouse-sink so all instrumented prod services export traces and logs to PostHog. --- deploy/stack.yml | 2 ++ src/instrumentation.test.ts | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/deploy/stack.yml b/deploy/stack.yml index b7d2a3e663..180f595458 100644 --- a/deploy/stack.yml +++ b/deploy/stack.yml @@ -181,6 +181,7 @@ services: command: ["cdc-health"] env_file: ${INFISICAL_ENV_FILE:?INFISICAL_ENV_FILE is required} environment: + DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT:-production} NODE_ENV: production DATABASE_URL: postgres://health:${POSTGRES_PASSWORD}@db:5432/health CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD_ENCODED:?CLICKHOUSE_PASSWORD_ENCODED is required}@clickhouse:8123 @@ -338,6 +339,7 @@ services: command: ["metric-stream-clickhouse-sink"] env_file: ${INFISICAL_ENV_FILE:?INFISICAL_ENV_FILE is required} environment: + DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT:-production} NODE_ENV: production CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD_ENCODED:?CLICKHOUSE_PASSWORD_ENCODED is required}@clickhouse:8123 REDPANDA_BROKERS: ${REDPANDA_BROKERS:?REDPANDA_BROKERS is required} diff --git a/src/instrumentation.test.ts b/src/instrumentation.test.ts index c32302d155..0dc2714228 100644 --- a/src/instrumentation.test.ts +++ b/src/instrumentation.test.ts @@ -7,6 +7,7 @@ import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { POSTHOG_API_KEY, POSTHOG_TRACES_URL } from "./lib/posthog-config.ts"; const mockStart = vi.fn(); const mockShutdown = vi.fn().mockResolvedValue(undefined); @@ -166,7 +167,7 @@ describe("instrumentation", () => { startInstrumentation({ OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318" }); expect(NodeSDK).toHaveBeenCalledOnce(); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(1); expect(config?.logRecordProcessors).toHaveLength(1); expect(config?.metricReader).toBeDefined(); @@ -190,7 +191,7 @@ describe("instrumentation", () => { DEPLOY_ENVIRONMENT: "production", }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.logRecordProcessors).toHaveLength(1); expect(BatchLogRecordProcessor).toHaveBeenCalledWith({ exporter: expect.any(OTLPLogExporter), @@ -205,22 +206,25 @@ describe("instrumentation", () => { ); }); - it("adds a PostHog trace exporter in production even without Axiom endpoints", async () => { + it.each([ + "prod", + "production", + ])("adds a PostHog trace exporter in production even without Axiom endpoints (DEPLOY_ENVIRONMENT=%s)", async (deploymentEnvironment) => { const { startInstrumentation } = await import("./instrumentation.ts"); startInstrumentation({ - DEPLOY_ENVIRONMENT: "production", + DEPLOY_ENVIRONMENT: deploymentEnvironment, }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(1); expect(config?.instrumentations).toEqual([mockAutoInstrumentations]); expect(BatchSpanProcessor).toHaveBeenCalledWith(expect.any(OTLPTraceExporter)); expect(OTLPTraceExporter).toHaveBeenCalledWith( expect.objectContaining({ - url: expect.stringContaining("/i/v1/traces"), + url: POSTHOG_TRACES_URL, headers: expect.objectContaining({ - Authorization: expect.stringContaining("Bearer "), + Authorization: `Bearer ${POSTHOG_API_KEY}`, }), }), ); @@ -234,7 +238,7 @@ describe("instrumentation", () => { OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(2); expect(BatchSpanProcessor).toHaveBeenCalledTimes(2); }); @@ -246,7 +250,7 @@ describe("instrumentation", () => { OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://localhost:4318/v1/traces", }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(1); expect(config?.logRecordProcessors).toHaveLength(0); expect(config?.metricReader).toBeUndefined(); @@ -261,7 +265,7 @@ describe("instrumentation", () => { OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://localhost:4318/v1/logs", }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(0); expect(config?.logRecordProcessors).toHaveLength(1); expect(config?.instrumentations).toEqual([]); @@ -275,7 +279,7 @@ describe("instrumentation", () => { OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://localhost:4318/v1/metrics", }); - const config = vi.mocked(NodeSDK).mock.calls[0]?.[0]; + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; expect(config?.spanProcessors).toHaveLength(0); expect(config?.logRecordProcessors).toHaveLength(0); expect(config?.metricReader).toBeDefined(); From 96eb7f97251ba40deb8bab9c489cc05424e4c20c Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Sat, 1 Aug 2026 09:53:34 -0700 Subject: [PATCH 3/4] docs(otel): clarify when instrumentation starts --- src/instrumentation.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/instrumentation.ts b/src/instrumentation.ts index c42c262d6e..1577e33afe 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -75,8 +75,9 @@ function createLogRecordProcessors(env: Record): Log } /** - * Starts OpenTelemetry instrumentation when OTLP export env vars are set. - * Returns the SDK instance for shutdown, or undefined if OTel is disabled. + * Starts OpenTelemetry instrumentation when a telemetry export is configured + * (OTLP env vars or a production PostHog export). Returns the SDK instance for + * shutdown, or undefined if OTel is disabled. */ export function startInstrumentation( env: Record = process.env, From 9d80f828ebb0cd5ef584204070425c25c71321fc Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Sat, 1 Aug 2026 10:44:53 -0700 Subject: [PATCH 4/4] fix(otel): keep service export trace-only Use the existing production runtime plus explicit OTLP trace setup for helper services, avoiding broader PostHog log export. --- deploy/stack.yml | 2 -- src/instrumentation.test.ts | 23 +++++++++++++++++++++++ src/instrumentation.ts | 4 +++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/deploy/stack.yml b/deploy/stack.yml index 180f595458..b7d2a3e663 100644 --- a/deploy/stack.yml +++ b/deploy/stack.yml @@ -181,7 +181,6 @@ services: command: ["cdc-health"] env_file: ${INFISICAL_ENV_FILE:?INFISICAL_ENV_FILE is required} environment: - DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT:-production} NODE_ENV: production DATABASE_URL: postgres://health:${POSTGRES_PASSWORD}@db:5432/health CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD_ENCODED:?CLICKHOUSE_PASSWORD_ENCODED is required}@clickhouse:8123 @@ -339,7 +338,6 @@ services: command: ["metric-stream-clickhouse-sink"] env_file: ${INFISICAL_ENV_FILE:?INFISICAL_ENV_FILE is required} environment: - DEPLOY_ENVIRONMENT: ${DEPLOY_ENVIRONMENT:-production} NODE_ENV: production CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD_ENCODED:?CLICKHOUSE_PASSWORD_ENCODED is required}@clickhouse:8123 REDPANDA_BROKERS: ${REDPANDA_BROKERS:?REDPANDA_BROKERS is required} diff --git a/src/instrumentation.test.ts b/src/instrumentation.test.ts index 0dc2714228..d964af9136 100644 --- a/src/instrumentation.test.ts +++ b/src/instrumentation.test.ts @@ -243,6 +243,29 @@ describe("instrumentation", () => { expect(BatchSpanProcessor).toHaveBeenCalledTimes(2); }); + it("adds PostHog traces without PostHog logs for production services configured with explicit OTLP endpoints", async () => { + const { startInstrumentation } = await import("./instrumentation.ts"); + + startInstrumentation({ + NODE_ENV: "production", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://collector:4318/v1/traces", + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://collector:4318/v1/logs", + }); + + const config = vi.mocked(NodeSDK).mock.calls.at(-1)?.[0]; + expect(config?.spanProcessors).toHaveLength(2); + expect(config?.logRecordProcessors).toHaveLength(1); + expect(OTLPTraceExporter).toHaveBeenCalledTimes(2); + expect(OTLPTraceExporter).toHaveBeenLastCalledWith({ + url: POSTHOG_TRACES_URL, + headers: { + Authorization: `Bearer ${POSTHOG_API_KEY}`, + }, + }); + expect(OTLPLogExporter).toHaveBeenCalledOnce(); + expect(OTLPLogExporter).toHaveBeenCalledWith(); + }); + it("only configures trace processors and auto instrumentations when only traces endpoint exists", async () => { const { startInstrumentation } = await import("./instrumentation.ts"); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 1577e33afe..2d0b6fc74d 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -18,7 +18,9 @@ function createSpanProcessors(env: Record): SpanProc const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT; const tracesEndpoint = env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; const hasAxiomTraceExport = Boolean(endpoint || tracesEndpoint); - const hasPostHogTraceExport = isProductionDeployment(env.DEPLOY_ENVIRONMENT); + const hasPostHogTraceExport = + isProductionDeployment(env.DEPLOY_ENVIRONMENT) || + (env.NODE_ENV === "production" && hasAxiomTraceExport); const processors: SpanProcessor[] = [];