- * Using the mutable StringBuilder instead of plain String. - */ - public static final class StringFormat implements Format { - } - -} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java index b8560c46853a..95e735b8965b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java @@ -19,9 +19,6 @@ import static java.util.Collections.emptyMap; -import io.opentracing.Scope; -import io.opentracing.Span; -import io.opentracing.util.GlobalTracer; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.util.Arrays; @@ -67,10 +64,7 @@ public Object invoke(Object proxy, Method method, Object[] args) method.getName()); } - Span span = GlobalTracer.get().buildSpan( - name + "." + method.getName()) - .start(); - try (Scope ignored = GlobalTracer.get().activateSpan(span)) { + try (TracingUtil.TraceCloseable ignored = TracingUtil.createActivatedSpan(name + "." + method.getName())) { try { return delegateMethod.invoke(delegate, args); } catch (Exception ex) { @@ -79,8 +73,6 @@ public Object invoke(Object proxy, Method method, Object[] args) } else { throw ex; } - } finally { - span.finish(); } } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java index 867802120ef2..560f3876c114 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java @@ -17,27 +17,44 @@ package org.apache.hadoop.hdds.tracing; -import io.jaegertracing.Configuration; -import io.jaegertracing.internal.JaegerTracer; -import io.opentracing.Scope; -import io.opentracing.Span; -import io.opentracing.SpanContext; -import io.opentracing.Tracer; -import io.opentracing.util.GlobalTracer; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; import java.lang.reflect.Proxy; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.ratis.util.function.CheckedRunnable; import org.apache.ratis.util.function.CheckedSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utility class to collect all the tracing helper methods. */ public final class TracingUtil { - + private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class); private static final String NULL_SPAN_AS_STRING = ""; + private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"; + private static final String OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT = "http://localhost:4317"; + private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG"; + private static final double OTEL_TRACES_SAMPLER_RATIO_DEFAULT = 1.0; private static volatile boolean isInit = false; + private static Tracer tracer = OpenTelemetry.noop().getTracer("noop"); private TracingUtil() { } @@ -47,38 +64,68 @@ private TracingUtil() { */ public static void initTracing( String serviceName, ConfigurationSource conf) { - if (!GlobalTracer.isRegistered() && isTracingEnabled(conf)) { - Configuration config = Configuration.fromEnv(serviceName); - JaegerTracer tracer = config.getTracerBuilder() - .registerExtractor(StringCodec.FORMAT, new StringCodec()) - .registerInjector(StringCodec.FORMAT, new StringCodec()) - .build(); - GlobalTracer.registerIfAbsent(tracer); + if (!isTracingEnabled(conf) || isInit) { + return; + } + + try { + initialize(serviceName); isInit = true; + LOG.info("Initialized tracing service: {}", serviceName); + } catch (Exception e) { + LOG.error("Failed to initialize tracing", e); } } - /** - * Export the active tracing span as a string. - * - * @return encoded tracing context. - */ - public static String exportCurrentSpan() { - return exportSpan(GlobalTracer.get().activeSpan()); + private static void initialize(String serviceName) { + String otelEndPoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT); + if (otelEndPoint == null || otelEndPoint.isEmpty()) { + otelEndPoint = OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT; + } + + double samplerRatio = OTEL_TRACES_SAMPLER_RATIO_DEFAULT; + try { + String sampleStrRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG); + if (sampleStrRatio != null && !sampleStrRatio.isEmpty()) { + samplerRatio = Double.parseDouble(System.getenv(OTEL_TRACES_SAMPLER_ARG)); + } + } catch (NumberFormatException ex) { + // ignore and use the default value. + } + + Resource resource = Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName)); + OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder() + .setEndpoint(otelEndPoint) + .build(); + + SimpleSpanProcessor spanProcessor = SimpleSpanProcessor.builder(spanExporter).build(); + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor) + .setResource(resource) + .setSampler(Sampler.traceIdRatioBased(samplerRatio)) + .build(); + OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build(); + tracer = openTelemetry.getTracer(serviceName); } /** - * Export the specific span as a string. + * Export the active tracing span as a string. * * @return encoded tracing context. */ - public static String exportSpan(Span span) { - if (span != null && isInit) { - StringBuilder builder = new StringBuilder(); - GlobalTracer.get().inject(span.context(), StringCodec.FORMAT, builder); - return builder.toString(); + public static String exportCurrentSpan() { + Span currentSpan = Span.current(); + if (!currentSpan.getSpanContext().isValid()) { + return NULL_SPAN_AS_STRING; } - return NULL_SPAN_AS_STRING; + + StringBuilder builder = new StringBuilder(); + W3CTraceContextPropagator propagator = W3CTraceContextPropagator.getInstance(); + propagator.inject(Context.current(), builder, + (carrier, key, value) -> carrier.append(key).append('=').append(value).append(';')); + return builder.toString(); } /** @@ -87,25 +134,18 @@ public static String exportSpan(Span span) { * @param name name of the newly created scope * @param encodedParent Encoded parent span (could be null or empty) * - * @return OpenTracing scope. + * @return Tracing scope. */ public static Span importAndCreateSpan(String name, String encodedParent) { - Tracer tracer = GlobalTracer.get(); - return tracer.buildSpan(name) - .asChildOf(extractParent(encodedParent, tracer)) - .start(); - } - - private static SpanContext extractParent(String parent, Tracer tracer) { - if (!GlobalTracer.isRegistered()) { - return null; + if (encodedParent == null || encodedParent.isEmpty()) { + return tracer.spanBuilder(name).setNoParent().startSpan(); } - if (parent == null || parent.isEmpty()) { - return null; - } - - return tracer.extract(StringCodec.FORMAT, new StringBuilder(parent)); + W3CTraceContextPropagator propagator = W3CTraceContextPropagator.getInstance(); + Context extract = propagator.extract(Context.current(), encodedParent, new TextExtractor()); + return tracer.spanBuilder(name) + .setParent(extract) + .startSpan(); } /** @@ -142,8 +182,7 @@ public static boolean isTracingEnabled( */ public static void executeInNewSpan(String spanName, CheckedRunnable runnable) throws E { - Span span = GlobalTracer.get() - .buildSpan(spanName).start(); + Span span = tracer.spanBuilder(spanName).setNoParent().startSpan(); executeInSpan(span, runnable); } @@ -152,8 +191,7 @@ public static void executeInNewSpan(String spanName, */ public static R executeInNewSpan(String spanName, CheckedSupplier supplier) throws E { - Span span = GlobalTracer.get() - .buildSpan(spanName).start(); + Span span = tracer.spanBuilder(spanName).setNoParent().startSpan(); return executeInSpan(span, supplier); } @@ -163,13 +201,14 @@ public static R executeInNewSpan(String spanName, */ private static R executeInSpan(Span span, CheckedSupplier supplier) throws E { - try (Scope ignored = GlobalTracer.get().activateSpan(span)) { + try (Scope ignored = span.makeCurrent()) { return supplier.get(); } catch (Exception ex) { - span.setTag("failed", true); + span.addEvent("Failed with exception: " + ex.getMessage()); + span.setStatus(StatusCode.ERROR); throw ex; } finally { - span.finish(); + span.end(); } } @@ -178,13 +217,14 @@ private static R executeInSpan(Span span, */ private static void executeInSpan(Span span, CheckedRunnable runnable) throws E { - try (Scope ignored = GlobalTracer.get().activateSpan(span)) { + try (Scope ignored = span.makeCurrent()) { runnable.run(); } catch (Exception ex) { - span.setTag("failed", true); + span.addEvent("Failed with exception: " + ex.getMessage()); + span.setStatus(StatusCode.ERROR); throw ex; } finally { - span.finish(); + span.end(); } } @@ -203,12 +243,60 @@ public static void executeAsChildSpan(String spanName, * This is a simplified way to use span as there is no way to add any tag * in case of Exceptions. */ - public static AutoCloseable createActivatedSpan(String spanName) { - Span span = GlobalTracer.get().buildSpan(spanName).start(); - Scope scope = GlobalTracer.get().activateSpan(span); + public static TraceCloseable createActivatedSpan(String spanName) { + Span span = tracer.spanBuilder(spanName).setNoParent().startSpan(); + Scope scope = span.makeCurrent(); return () -> { scope.close(); - span.finish(); + span.end(); }; } + + public static Span getActiveSpan() { + return Span.current(); + } + + /** + * AutoCloseable interface for tracing span but no exception is thrown in close. + */ + public interface TraceCloseable extends AutoCloseable { + @Override + void close(); + } + + /** + * A TextMapGetter implementation to extract tracing info from String. + */ + public static class TextExtractor implements io.opentelemetry.context.propagation.TextMapGetter { + private Map map = new HashMap<>(); + + @Override + public Iterable keys(String carrier) { + if (map.isEmpty()) { + parse(carrier); + } + return map.keySet(); + } + + @Override + public String get(String carrier, String key) { + if (map.isEmpty()) { + parse(carrier); + } + return map.get(key); + } + + private void parse(String carrier) { + if (carrier == null || carrier.isEmpty()) { + return; + } + String[] parts = carrier.split(";"); + for (String part : parts) { + String[] kv = part.split("="); + if (kv.length == 2) { + map.put(kv[0].trim(), kv[1].trim()); + } + } + } + } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestStringCodec.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestStringCodec.java deleted file mode 100644 index aab23d5da007..000000000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestStringCodec.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.tracing; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import io.jaegertracing.internal.JaegerSpanContext; -import io.jaegertracing.internal.exceptions.EmptyTracerStateStringException; -import io.jaegertracing.internal.exceptions.MalformedTracerStateStringException; -import org.junit.jupiter.api.Test; - -class TestStringCodec { - - @Test - void testExtract() { - StringCodec codec = new StringCodec(); - - assertThrows(EmptyTracerStateStringException.class, - () -> codec.extract(null)); - - StringBuilder sb = new StringBuilder().append("123"); - MalformedTracerStateStringException malformedException = - assertThrows(MalformedTracerStateStringException.class, - () -> codec.extract(sb)); - assertEquals("String does not match tracer state format: 123", - malformedException.getMessage()); - - sb.append(":456:789"); - malformedException = - assertThrows(MalformedTracerStateStringException.class, - () -> codec.extract(sb)); - assertEquals("String does not match tracer state format: 123:456:789", - malformedException.getMessage()); - - sb.append(":66"); - JaegerSpanContext context = codec.extract(sb); - StringBuilder injected = new StringBuilder(); - codec.inject(context, injected); - - String expectedTraceId = pad("123"); - assertEquals(expectedTraceId, context.getTraceId()); - assertEquals(expectedTraceId + ":456:789:66", injected.toString()); - } - - private static String pad(String s) { - return "0000000000000000".substring(s.length()) + s; - } -} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java index 6519031b2d31..bc500ddddab2 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java @@ -22,9 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; -import io.jaegertracing.Configuration; -import io.jaegertracing.internal.JaegerTracer; -import io.opentracing.util.GlobalTracer; import org.apache.hadoop.hdds.conf.InMemoryConfiguration; import org.apache.hadoop.hdds.conf.MutableConfigurationSource; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -47,10 +44,8 @@ public void testDefaultMethod() { @Test public void testInitTracing() { - Configuration config = Configuration.fromEnv("testInitTracing"); - JaegerTracer tracer = config.getTracerBuilder().build(); - GlobalTracer.registerIfAbsent(tracer); - try (AutoCloseable ignored = TracingUtil.createActivatedSpan("initTracing")) { + TracingUtil.initTracing("testInitTracing", tracingEnabled()); + try (TracingUtil.TraceCloseable ignored = TracingUtil.createActivatedSpan("initTracing")) { exportCurrentSpan(); } catch (Exception e) { fail("Should not get exception"); diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index b46e46e4f2a1..0c455d269591 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -79,12 +79,12 @@ netty-transport