From bfe8dbce580de0f3226ac4ae634a6154eca930c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fede=20Fern=C3=A1ndez?= <720923+fedefernandez@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:02:09 +0200 Subject: [PATCH 1/3] Adapts the open-telemetry example for using agents --- .../xef/conversation/conversations/Animal.kt | 4 +- .../xef/opentelemetry/OpenTelemetryConfig.kt | 43 +++++++++++++---- .../docker/opentelemetry/docker-compose.yml | 47 ++++++++++++++++--- .../opentelemetry/otel-collector-config.yaml | 41 ++++++++++++++++ server/docker/opentelemetry/prometheus.yaml | 6 +++ 5 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 server/docker/opentelemetry/otel-collector-config.yaml create mode 100644 server/docker/opentelemetry/prometheus.yaml diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt index 685d63873..e05732e7c 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt @@ -18,7 +18,9 @@ suspend fun main() { // To run the example with OpenTelemetry, you can execute the following commands: // - # docker compose-up server/docker/opentelemetry - // OpenAI.conversation(LocalVectorStore(OpenAI().DEFAULT_EMBEDDING), OpenTelemetryMetric()) +// OpenAI.conversation( +// com.xebia.functional.xef.store.LocalVectorStore(OpenAI().DEFAULT_EMBEDDING), +// com.xebia.functional.xef.opentelemetry.OpenTelemetryMetric()) { OpenAI.conversation { val animal: Animal = prompt("A unique animal species.") diff --git a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt index 3a650423d..f5dcd5713 100644 --- a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt +++ b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt @@ -2,14 +2,22 @@ package com.xebia.functional.xef.opentelemetry import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.common.AttributeKey -import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator +import io.opentelemetry.context.propagation.ContextPropagators +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.logs.SdkLoggerProvider +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader import io.opentelemetry.sdk.resources.Resource import io.opentelemetry.sdk.trace.SdkTracerProvider import io.opentelemetry.sdk.trace.export.BatchSpanProcessor import java.util.concurrent.TimeUnit + data class OpenTelemetryConfig( val endpointConfig: String, val defaultScopeName: String, @@ -17,24 +25,43 @@ data class OpenTelemetryConfig( ) { fun newInstance(): OpenTelemetry { - val jaegerOtlpExporter: OtlpGrpcSpanExporter = + val otlpGrpcSpanExporter: OtlpGrpcSpanExporter = OtlpGrpcSpanExporter.builder() .setEndpoint(endpointConfig) .setTimeout(30, TimeUnit.SECONDS) .build() - val serviceNameResource: Resource = - Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName)) + val resource = + Resource.getDefault().toBuilder() + .put(AttributeKey.stringKey("service.name"), serviceName) + .build() val tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(BatchSpanProcessor.builder(jaegerOtlpExporter).build()) - .setResource(Resource.getDefault().merge(serviceNameResource)) + .addSpanProcessor(BatchSpanProcessor.builder(otlpGrpcSpanExporter).build()) + .setResource(resource) .build() - val openTelemetry = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build() + val meterProvider = SdkMeterProvider.builder() + .registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build()) + .setResource(resource) + .build() + + val loggerProvider = SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()).build() + ) + .setResource(resource) + .build() + + val openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setMeterProvider(meterProvider) + .setLoggerProvider(loggerProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal() - Runtime.getRuntime().addShutdownHook(Thread { tracerProvider.close() }) + Runtime.getRuntime().addShutdownHook(Thread { openTelemetry.close() }) return openTelemetry } diff --git a/server/docker/opentelemetry/docker-compose.yml b/server/docker/opentelemetry/docker-compose.yml index 49ba02528..3309c730e 100644 --- a/server/docker/opentelemetry/docker-compose.yml +++ b/server/docker/opentelemetry/docker-compose.yml @@ -1,9 +1,44 @@ -version: "3" +version: "4" services: - jaeger: + + # Jaeger + jaeger-all-in-one: image: jaegertracing/all-in-one:latest - environment: - COLLECTOR_OTLP_ENABLED: true + restart: always ports: - - 4317:4317 - - 16686:16686 + - "16686:16686" + - "14268" + - "14250" + + # Zipkin + zipkin-all-in-one: + image: openzipkin/zipkin:latest + restart: always + ports: + - "9411:9411" + + # Collector + otel-collector: + image: otel/opentelemetry-collector:0.85.0 + restart: always + command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "55679:55679" # zpages extension + depends_on: + - jaeger-all-in-one + - zipkin-all-in-one + + prometheus: + image: prom/prometheus:latest + restart: always + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" \ No newline at end of file diff --git a/server/docker/opentelemetry/otel-collector-config.yaml b/server/docker/opentelemetry/otel-collector-config.yaml new file mode 100644 index 000000000..16b9aa533 --- /dev/null +++ b/server/docker/opentelemetry/otel-collector-config.yaml @@ -0,0 +1,41 @@ +receivers: + otlp: + protocols: + grpc: + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + const_labels: + label1: value1 + + zipkin: + endpoint: "http://zipkin-all-in-one:9411/api/v2/spans" + format: proto + + otlp: + endpoint: jaeger-all-in-one:4317 + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [zipkin, otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] \ No newline at end of file diff --git a/server/docker/opentelemetry/prometheus.yaml b/server/docker/opentelemetry/prometheus.yaml new file mode 100644 index 000000000..eb9d8ccc6 --- /dev/null +++ b/server/docker/opentelemetry/prometheus.yaml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['otel-collector:8889'] + - targets: ['otel-collector:8888'] \ No newline at end of file From 281a52baa16cc28d7c130b0dc8da8d77f625a651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fede=20Fern=C3=A1ndez?= <720923+fedefernandez@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:59:18 +0200 Subject: [PATCH 2/3] Spotless apply --- .../xef/conversation/conversations/Animal.kt | 6 +-- .../xef/opentelemetry/OpenTelemetryConfig.kt | 41 +++++++++++-------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt index e05732e7c..0d662eb42 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt @@ -18,9 +18,9 @@ suspend fun main() { // To run the example with OpenTelemetry, you can execute the following commands: // - # docker compose-up server/docker/opentelemetry -// OpenAI.conversation( -// com.xebia.functional.xef.store.LocalVectorStore(OpenAI().DEFAULT_EMBEDDING), -// com.xebia.functional.xef.opentelemetry.OpenTelemetryMetric()) { + // OpenAI.conversation( + // com.xebia.functional.xef.store.LocalVectorStore(OpenAI().DEFAULT_EMBEDDING), + // com.xebia.functional.xef.opentelemetry.OpenTelemetryMetric()) { OpenAI.conversation { val animal: Animal = prompt("A unique animal species.") diff --git a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt index f5dcd5713..dffc7d2bf 100644 --- a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt +++ b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt @@ -17,7 +17,6 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider import io.opentelemetry.sdk.trace.export.BatchSpanProcessor import java.util.concurrent.TimeUnit - data class OpenTelemetryConfig( val endpointConfig: String, val defaultScopeName: String, @@ -32,7 +31,8 @@ data class OpenTelemetryConfig( .build() val resource = - Resource.getDefault().toBuilder() + Resource.getDefault() + .toBuilder() .put(AttributeKey.stringKey("service.name"), serviceName) .build() @@ -42,24 +42,29 @@ data class OpenTelemetryConfig( .setResource(resource) .build() - val meterProvider = SdkMeterProvider.builder() - .registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build()) - .setResource(resource) - .build() + val meterProvider = + SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()).build() + ) + .setResource(resource) + .build() - val loggerProvider = SdkLoggerProvider.builder() - .addLogRecordProcessor( - BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()).build() - ) - .setResource(resource) - .build() + val loggerProvider = + SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()).build() + ) + .setResource(resource) + .build() - val openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(tracerProvider) - .setMeterProvider(meterProvider) - .setLoggerProvider(loggerProvider) - .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) - .buildAndRegisterGlobal() + val openTelemetry = + OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setMeterProvider(meterProvider) + .setLoggerProvider(loggerProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal() Runtime.getRuntime().addShutdownHook(Thread { openTelemetry.close() }) return openTelemetry From 399219fdaff81ea6590bdf128d8d25d122674953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fede=20Fern=C3=A1ndez?= <720923+fedefernandez@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:16:08 +0200 Subject: [PATCH 3/3] Bumps opentelemetry --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9b214d0ff..3fd8706f3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,7 +45,7 @@ suspendApp = "0.4.0" flyway = "9.22.3" resources-kmp = "0.4.0" detekt = "1.23.1" -opentelemetry="1.30.1" +opentelemetry="1.31.0" opentelemetry-alpha="1.30.1-alpha" [libraries]