feat: migrate built in metrics to OTEL

google-cloud-bigtable/clirr-ignored-differences.xml

@@ -156,6 +156,12 @@
         <differenceType>8001</differenceType>
         <className>com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerBatchedUnaryCallable</className>
     </difference>
+    <!-- InternalApi constructor was updated -->
+    <difference>
+        <differenceType>7004</differenceType>
+        <className>com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory</className>
+        <method>*</method>
+    </difference>
     <!-- InternalApi was updated -->
     <difference>
         <differenceType>6001</differenceType>

google-cloud-bigtable/pom.xml

@@ -339,6 +339,10 @@
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-metrics</artifactId>
@@ -347,6 +351,11 @@
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-monitoring</artifactId>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java

@@ -25,16 +25,14 @@
 import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
 import com.google.api.gax.rpc.UnaryCallSettings;
 import com.google.auth.Credentials;
-import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.bigtable.data.v2.models.Query;
 import com.google.cloud.bigtable.data.v2.models.Row;
 import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
 import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
-import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter;
-import com.google.cloud.bigtable.stats.BuiltinViews;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import io.grpc.ManagedChannelBuilder;
+import io.opentelemetry.api.OpenTelemetry;
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -197,24 +195,25 @@ public static void enableGfeOpenCensusStats() {
     com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews();
   }
 
-  /** Register built in metrics. */
-  public static void enableBuiltinMetrics() throws IOException {
-    if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
-      BuiltinViews.registerBigtableBuiltinViews();
-      BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault());
-    }
-  }
+  /**
+   * Register built in metrics.
+   *
+   * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+   *     now. Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)} to
+   *     enable or disable built-in metrics.
+   */
+  @Deprecated
+  public static void enableBuiltinMetrics() throws IOException {}
 
   /**
    * Register built in metrics with credentials. The credentials need to have metric write access
    * for all the projects you're publishing to.
+   *
+   * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+   *     now. Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)} to
+   *     enable or disable built-in metrics.
    */
-  public static void enableBuiltinMetrics(Credentials credentials) throws IOException {
-    if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
-      BuiltinViews.registerBigtableBuiltinViews();
-      BigtableStackdriverStatsExporter.register(credentials);
-    }
-  }
+  public static void enableBuiltinMetrics(Credentials credentials) throws IOException {}
 
   /** Returns the target project id. */
   public String getProjectId() {
@@ -278,6 +277,20 @@ public boolean isBulkMutationFlowControlEnabled() {
     return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
   }
 
+  /** Gets if built-in metrics is enabled. */
+  public boolean isBuiltinMetricsEnabled() {
+    return stubSettings.isBuiltinMetricsEnabled();
+  }
+
+  /**
+   * Gets the custom OpenTelemetry instance. If no custom OTEL instance is set, the client uses a
+   * default instance with builtin metrics enabled. Use {@link
+   * Builder#setBuiltinMetricsEnabled(boolean)} to disable the builtin metrics.
+   */
+  public OpenTelemetry getOpenTelemetry() {
+    return stubSettings.getOpenTelemetry();
+  }
+
   /** Returns the underlying RPC settings. */
   public EnhancedBigtableStubSettings getStubSettings() {
     return stubSettings;
@@ -527,6 +540,54 @@ public boolean isBulkMutationFlowControlEnabled() {
       return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
     }
 
+    /** Set enable to true to enable builtin metrics. */
+    public Builder setBuiltinMetricsEnabled(boolean enable) {
+      stubSettings.setBuiltinMetricsEnabled(enable);
+      return this;
+    }
+
+    /** Gets if built-in metrics is enabled. */
+    public boolean isBuiltinMetricsEnabled() {
+      return stubSettings.isBuiltinMetricsEnabled();
+    }
+
+    /**
+     * Set a custom OpenTelemetry instance.
+     *
+     * <p>To register builtin metrics on the custom OpenTelemetry:
+     *
+     * <pre>{@code
+     * SdkMeterProviderBuilder sdkMeterProvider = SdkMeterProvider.builder();
+     *
+     * // register Builtin metrics on your meter provider
+     * BuiltinMetricsViews.registerBuiltinMetrics("project-id", sdkMeterProvider);
+     *
+     * // register other metrics reader and views
+     * sdkMeterProvider.registerMetricReader(..);
+     * sdkMeterProvider.registerView(..);
+     *
+     * // create the OTEL instance
+     * OpenTelemetry openTelemetry = OpenTelemetrySdk
+     *     .builder()
+     *     .setMeterProvider(sdkMeterProvider().build());
+     *
+     * settings.setOpenTelemetry(openTelemetry);
+     * }</pre>
+     */
+    public Builder setOpenTelemetry(OpenTelemetry openTelemetry) {
+      stubSettings.setOpenTelemetry(openTelemetry);
+      return this;
+    }
+
+    /**
+     * Gets the custom OpenTelemetry instance. If no custom OTEL instance is set, the client uses a
+     * default instance with builtin metrics enabled. Use {@link #setBuiltinMetricsEnabled(boolean)}
+     * to disable the builtin metrics.
+     */
+    public OpenTelemetry getOpenTelemetry() {
+      return stubSettings.getOpenTelemetry();
+    }
+
     /**
      * Returns the underlying settings for making RPC calls. The settings should be changed with
      * care.

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -15,6 +15,10 @@
  */
 package com.google.cloud.bigtable.data.v2.stub;
 
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PROJECT_ID;
+
 import com.google.api.core.BetaApi;
 import com.google.api.core.InternalApi;
 import com.google.api.gax.batching.Batcher;
@@ -93,6 +97,7 @@
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsView;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants;
@@ -123,6 +128,11 @@
 import io.opencensus.tags.TagValue;
 import io.opencensus.tags.Tagger;
 import io.opencensus.tags.Tags;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -236,14 +246,15 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
     return ClientContext.create(builder.build());
   }
 
-  public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings) {
+  public static ApiTracerFactory createBigtableTracerFactory(EnhancedBigtableStubSettings settings)
+      throws IOException {
     return createBigtableTracerFactory(settings, Tags.getTagger(), Stats.getStatsRecorder());
   }
 
   @VisibleForTesting
   public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats) {
+      EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats)
+      throws IOException {
     String projectId = settings.getProjectId();
     String instanceId = settings.getInstanceId();
     String appProfileId = settings.getAppProfileId();
@@ -254,32 +265,61 @@ public static ApiTracerFactory createBigtableTracerFactory(
             .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(instanceId))
             .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId))
             .build();
-    ImmutableMap<String, String> builtinAttributes =
-        ImmutableMap.<String, String>builder()
-            .put("project_id", projectId)
-            .put("instance", instanceId)
-            .put("app_profile", appProfileId)
-            .build();
 
-    return new CompositeTracerFactory(
-        ImmutableList.of(
+    ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
+    tracerFactories
+        .add(
             // Add OpenCensus Tracing
             new OpencensusTracerFactory(
                 ImmutableMap.<String, String>builder()
                     // Annotate traces with the same tags as metrics
-                    .put(RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), projectId)
-                    .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), instanceId)
-                    .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), appProfileId)
+                    .put(RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), settings.getProjectId())
+                    .put(
+                        RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(),
+                        settings.getInstanceId())
+                    .put(
+                        RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(),
+                        settings.getAppProfileId())
                     // Also annotate traces with library versions
                     .put("gax", GaxGrpcProperties.getGaxGrpcVersion())
                     .put("grpc", GaxGrpcProperties.getGrpcVersion())
                     .put("gapic", Version.VERSION)
-                    .build()),
-            // Add OpenCensus Metrics
-            MetricsTracerFactory.create(tagger, stats, attributes),
-            BuiltinMetricsTracerFactory.create(builtinAttributes),
-            // Add user configured tracer
-            settings.getTracerFactory()));
+                    .build()))
+        // Add OpenCensus Metrics
+        .add(MetricsTracerFactory.create(tagger, stats, attributes))
+        // Add user configured tracer
+        .add(settings.getTracerFactory());
+    Attributes otelAttributes =
+        Attributes.of(
+            PROJECT_ID,
+            settings.getProjectId(),
+            INSTANCE_ID,
+            settings.getInstanceId(),
+            APP_PROFILE,
+            settings.getAppProfileId());
+    BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
+        setupBuiltinMetricsTracerFactory(settings.toBuilder(), otelAttributes);
+    if (builtinMetricsTracerFactory != null) {
+      tracerFactories.add(builtinMetricsTracerFactory);
+    }
+    return new CompositeTracerFactory(tracerFactories.build());
+  }
+
+  private static BuiltinMetricsTracerFactory setupBuiltinMetricsTracerFactory(
+      EnhancedBigtableStubSettings.Builder settings, Attributes attributes) throws IOException {
+    if (settings.getOpenTelemetry() != null) {
+      return BuiltinMetricsTracerFactory.create(settings.getOpenTelemetry(), attributes);
+    } else if (settings.isBuiltinMetricsEnabled()) {
+      SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder();
+      BuiltinMetricsView.registerBuiltinMetrics(
+          settings.getProjectId(),
+          settings.getCredentialsProvider().getCredentials(),
+          meterProvider);
+      OpenTelemetry openTelemetry =
+          OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
+      return BuiltinMetricsTracerFactory.create(openTelemetry, attributes);
+    }
+    return null;
   }
 
   private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)