Add OTLP Proxy Endpoint for Frontend Telemetry (#2245)

* Add OTLP proxy endpoint

* Fix OtlpProxyOptions

* Refactor

* Fix CodeQL

* Refactor and dispose disposable

Author: reakaleek

Directory.Packages.props

@@ -27,6 +27,7 @@
     <PackageVersion Include="AWSSDK.S3" Version="4.0.7.14" />
     <PackageVersion Include="Elastic.OpenTelemetry" Version="1.1.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="10.0.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.13.0" />
     <PackageVersion Include="Generator.Equals" Version="3.2.1" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
@@ -41,6 +42,7 @@
     <PackageVersion Include="Microsoft.OpenApi" Version="3.0.1" />
     <PackageVersion Include="TUnit" Version="0.25.21" />
     <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
+    <PackageVersion Include="WireMock.Net" Version="1.6.11" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -106,4 +108,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj

@@ -9,8 +9,10 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
       <PackageReference Include="Microsoft.Extensions.Logging" />
       <PackageReference Include="Microsoft.Extensions.Telemetry.Abstractions" />
+      <PackageReference Include="NetEscapades.EnumGenerators" />
     </ItemGroup>
 
 </Project>

src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.Api.Core.Telemetry;
+
+/// <summary>
+/// Gateway for forwarding OTLP telemetry to a collector.
+/// </summary>
+public interface IOtlpGateway
+{
+	/// <summary>
+	/// Forwards OTLP telemetry data to the collector.
+	/// </summary>
+	/// <param name="signalType">The OTLP signal type (traces, logs, or metrics)</param>
+	/// <param name="requestBody">The raw OTLP payload stream</param>
+	/// <param name="contentType">Content-Type of the payload</param>
+	/// <param name="ctx">Cancellation token</param>
+	/// <returns>HTTP status code and response content</returns>
+	Task<(int StatusCode, string? Content)> ForwardOtlp(
+		OtlpSignalType signalType,
+		Stream requestBody,
+		string contentType,
+		Cancel ctx = default);
+}

src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Microsoft.Extensions.Configuration;
+
+namespace Elastic.Documentation.Api.Core.Telemetry;
+
+/// <summary>
+/// Configuration options for the OTLP proxy.
+/// The proxy forwards telemetry to a local OTLP collector (typically ADOT Lambda Layer).
+/// </summary>
+/// <remarks>
+/// ADOT Lambda Layer runs a local OpenTelemetry Collector that accepts OTLP/HTTP on:
+/// - localhost:4318 (HTTP/JSON and HTTP/protobuf)
+/// - localhost:4317 (gRPC)
+/// 
+/// Configuration priority:
+/// 1. OtlpProxy:Endpoint in IConfiguration (for tests/overrides)
+/// 2. OTEL_EXPORTER_OTLP_ENDPOINT environment variable
+/// 3. Default: http://localhost:4318
+/// 
+/// The proxy will return 503 if the collector is not available.
+/// </remarks>
+public class OtlpProxyOptions
+{
+	/// <summary>
+	/// OTLP endpoint URL for the local ADOT collector.
+	/// Defaults to localhost:4318 when running in Lambda with ADOT layer.
+	/// </summary>
+	public string Endpoint { get; }
+
+	public OtlpProxyOptions(IConfiguration configuration)
+	{
+		// Check for explicit configuration override first (for tests or custom deployments)
+		var configEndpoint = configuration["OtlpProxy:Endpoint"];
+		if (!string.IsNullOrEmpty(configEndpoint))
+		{
+			Endpoint = configEndpoint;
+			return;
+		}
+
+		// Default to localhost:4318 - this is where ADOT Lambda Layer collector runs
+		// If ADOT layer is not present, the proxy will fail gracefully and return 503
+		Endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT")
+			?? "http://localhost:4318";
+	}
+}

src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs

@@ -0,0 +1,46 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.ComponentModel.DataAnnotations;
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Documentation.Api.Core.Telemetry;
+
+/// <summary>
+/// OTLP signal types supported by the proxy.
+/// The Display names match the OTLP path segments (lowercase).
+/// </summary>
+[EnumExtensions]
+public enum OtlpSignalType
+{
+	/// <summary>
+	/// Distributed traces - maps to /v1/traces
+	/// </summary>
+	[Display(Name = "traces")]
+	Traces,
+
+	/// <summary>
+	/// Log records - maps to /v1/logs
+	/// </summary>
+	[Display(Name = "logs")]
+	Logs,
+
+	/// <summary>
+	/// Metrics data - maps to /v1/metrics
+	/// </summary>
+	[Display(Name = "metrics")]
+	Metrics
+}
+
+/// <summary>
+/// Request model for OTLP proxy endpoint.
+/// Accepts raw OTLP payload from frontend and forwards to configured OTLP endpoint.
+/// </summary>
+public class OtlpProxyRequest
+{
+	/// <summary>
+	/// The OTLP signal type: traces, logs, or metrics
+	/// </summary>
+	public required string SignalType { get; init; }
+}

src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Diagnostics;
+
+namespace Elastic.Documentation.Api.Core.Telemetry;
+
+/// <summary>
+/// Proxies OTLP telemetry from the frontend to the local ADOT Lambda Layer collector.
+/// The ADOT layer handles authentication and forwarding to the backend.
+/// </summary>
+public class OtlpProxyUsecase(IOtlpGateway gateway)
+{
+	private static readonly ActivitySource ActivitySource = new(TelemetryConstants.OtlpProxySourceName);
+
+	/// <summary>
+	/// Proxies OTLP data from the frontend to the local ADOT collector.
+	/// </summary>
+	/// <param name="signalType">The OTLP signal type (traces, logs, or metrics)</param>
+	/// <param name="requestBody">The raw OTLP payload (JSON or protobuf)</param>
+	/// <param name="contentType">Content-Type header from the original request</param>
+	/// <param name="ctx">Cancellation token</param>
+	/// <returns>HTTP status code and response content</returns>
+	public async Task<(int StatusCode, string? Content)> ProxyOtlp(
+		OtlpSignalType signalType,
+		Stream requestBody,
+		string contentType,
+		Cancel ctx = default)
+	{
+		using var activity = ActivitySource.StartActivity("ProxyOtlp", ActivityKind.Client);
+
+		// Forward to gateway
+		return await gateway.ForwardOtlp(signalType, requestBody, contentType, ctx);
+	}
+}

src/api/Elastic.Documentation.Api.Core/TelemetryConstants.cs

@@ -25,4 +25,10 @@ public static class TelemetryConstants
 	/// Tag/baggage name used to annotate spans with the user's EUID value.
 	/// </summary>
 	public const string UserEuidAttributeName = "user.euid";
+
+	/// <summary>
+	/// ActivitySource name for OTLP proxy operations.
+	/// Used to trace frontend telemetry proxying.
+	/// </summary>
+	public const string OtlpProxySourceName = "Elastic.Documentation.Api.OtlpProxy";
 }

src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/AgentBuilderAskAiGateway.cs

@@ -37,15 +37,15 @@ public async Task<Stream> AskAi(AskAiRequest askAiRequest, Cancel ctx = default)
 		var kibanaUrl = await parameterProvider.GetParam("docs-kibana-url", false, ctx);
 		var kibanaApiKey = await parameterProvider.GetParam("docs-kibana-apikey", true, ctx);
 
-		var request = new HttpRequestMessage(HttpMethod.Post,
+		using var request = new HttpRequestMessage(HttpMethod.Post,
 			$"{kibanaUrl}/api/agent_builder/converse/async")
 		{
 			Content = new StringContent(requestBody, Encoding.UTF8, "application/json")
 		};
 		request.Headers.Add("kbn-xsrf", "true");
 		request.Headers.Authorization = new AuthenticationHeaderValue("ApiKey", kibanaApiKey);
 
-		var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx);
+		using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx);
 
 		// Ensure the response is successful before streaming
 		if (!response.IsSuccessStatusCode)

src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs

@@ -25,7 +25,7 @@ public async Task<Stream> AskAi(AskAiRequest askAiRequest, Cancel ctx = default)
 	{
 		var llmGatewayRequest = LlmGatewayRequest.CreateFromRequest(askAiRequest);
 		var requestBody = JsonSerializer.Serialize(llmGatewayRequest, LlmGatewayContext.Default.LlmGatewayRequest);
-		var request = new HttpRequestMessage(HttpMethod.Post, options.FunctionUrl)
+		using var request = new HttpRequestMessage(HttpMethod.Post, options.FunctionUrl)
 		{
 			Content = new StringContent(requestBody, Encoding.UTF8, "application/json")
 		};
@@ -37,7 +37,7 @@ public async Task<Stream> AskAi(AskAiRequest askAiRequest, Cancel ctx = default)
 
 		// Use HttpCompletionOption.ResponseHeadersRead to get headers immediately
 		// This allows us to start streaming as soon as headers are received
-		var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx);
+		using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx);
 
 		// Ensure the response is successful before streaming
 		if (!response.IsSuccessStatusCode)

src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs

@@ -0,0 +1,74 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Api.Core.Telemetry;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Documentation.Api.Infrastructure.Adapters.Telemetry;
+
+/// <summary>
+/// Gateway that forwards OTLP telemetry to the ADOT Lambda Layer collector.
+/// </summary>
+public class AdotOtlpGateway(
+	IHttpClientFactory httpClientFactory,
+	OtlpProxyOptions options,
+	ILogger<AdotOtlpGateway> logger) : IOtlpGateway
+{
+	public const string HttpClientName = "OtlpProxy";
+	private readonly HttpClient _httpClient = httpClientFactory.CreateClient(HttpClientName);
+
+	/// <inheritdoc />
+	public async Task<(int StatusCode, string? Content)> ForwardOtlp(
+		OtlpSignalType signalType,
+		Stream requestBody,
+		string contentType,
+		Cancel ctx = default)
+	{
+		try
+		{
+			// Build the target URL: http://localhost:4318/v1/{signalType}
+			// Use ToStringFast(true) from generated enum extensions (returns Display name: "traces", "logs", "metrics")
+			var targetUrl = $"{options.Endpoint.TrimEnd('/')}/v1/{signalType.ToStringFast(true)}";
+
+			logger.LogDebug("Forwarding OTLP {SignalType} to ADOT collector at {TargetUrl}", signalType, targetUrl);
+
+			using var request = new HttpRequestMessage(HttpMethod.Post, targetUrl);
+
+			// Forward the content with the original content type
+			request.Content = new StreamContent(requestBody);
+			_ = request.Content.Headers.TryAddWithoutValidation("Content-Type", contentType);
+
+			// No need to add authentication headers - ADOT layer handles auth to backend
+			// Just forward the telemetry to the local collector
+
+			// Forward to ADOT collector
+			using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ctx);
+			var responseContent = response.Content.Headers.ContentLength > 0
+				? await response.Content.ReadAsStringAsync(ctx)
+				: string.Empty;
+
+			if (!response.IsSuccessStatusCode)
+			{
+				logger.LogError("OTLP forward to ADOT failed with status {StatusCode}: {Content}",
+					response.StatusCode, responseContent);
+			}
+			else
+			{
+				logger.LogDebug("Successfully forwarded OTLP {SignalType} to ADOT collector", signalType);
+			}
+
+			return ((int)response.StatusCode, responseContent);
+		}
+		catch (HttpRequestException ex) when (ex.Message.Contains("Connection refused") || ex.InnerException?.Message?.Contains("Connection refused") == true)
+		{
+			logger.LogError(ex, "Failed to connect to ADOT collector at {Endpoint}. Is ADOT Lambda Layer enabled?", options.Endpoint);
+			return (503, "ADOT collector not available. Ensure AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument is set");
+		}
+		catch (Exception ex)
+		{
+			logger.LogError(ex, "Error forwarding OTLP {SignalType}", signalType);
+			return (500, $"Error forwarding OTLP: {ex.Message}");
+		}
+	}
+}