feat: migrate built in metrics to OTEL (#1796)

* feat: migrate exporter to OTEL

* address comments

* filter out only bigtable metrics

* fix test

* use the bom

* update

* update

* feat: migrate builtin metrics to OTEl

* update completeResultCode

* add a comment

* udpate

* fix tests

* remove unrelated changes

* fix tests

* add documentation

* fix test

* merge exporter changes

* address comments

* rebase on otel

* revert changes in stats

* fix import

* update

* merge back the endpoint change

* refactor constants and settings

* refactor and fix tests

* remove unused dependency

* add some javadoc

* address part of the comments

* update test

* test with nano

* measure everything in nanos and publish with double histogram

* address comments

* fix test

* add toString

Author: mutianf

google-cloud-bigtable/clirr-ignored-differences.xml

@@ -156,6 +156,12 @@
         <differenceType>8001</differenceType>
         <className>com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerBatchedUnaryCallable</className>
     </difference>
+    <!-- InternalApi constructor was updated -->
+    <difference>
+        <differenceType>7004</differenceType>
+        <className>com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory</className>
+        <method>*</method>
+    </difference>
     <!-- InternalApi was updated -->
     <difference>
         <differenceType>6001</differenceType>

google-cloud-bigtable/pom.xml

@@ -64,19 +64,6 @@
   <dependencies>
     <!-- NOTE: Dependencies are organized into two groups, production and test.
          Within a group, dependencies are sorted by (groupId, artifactId) -->
-    <dependency>
-      <groupId>com.google.cloud</groupId>
-      <artifactId>google-cloud-bigtable-stats</artifactId>
-      <!-- Exclude all dependencies that have been shaded. This is to workaround maven's immutable project structure:
-           after shading, the maven-shade-plugin tries to remove shaded dependencies, but it can't since the project
-           structure is immutable. So we have to manually exclude the shaded transitive dependencies manually. -->
-      <exclusions>
-        <exclusion>
-          <groupId>io.opencensus</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <!-- Production dependencies -->
     <dependency>
       <groupId>com.google.api</groupId>
@@ -339,6 +326,10 @@
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-metrics</artifactId>
@@ -347,6 +338,11 @@
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-monitoring</artifactId>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java

@@ -25,19 +25,16 @@
 import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
 import com.google.api.gax.rpc.UnaryCallSettings;
 import com.google.auth.Credentials;
-import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.bigtable.data.v2.models.Query;
 import com.google.cloud.bigtable.data.v2.models.Row;
 import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
 import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
-import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter;
-import com.google.cloud.bigtable.stats.BuiltinViews;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import io.grpc.ManagedChannelBuilder;
 import java.io.IOException;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -77,7 +74,10 @@ public final class BigtableDataSettings {
 
   private static final Logger LOGGER = Logger.getLogger(BigtableDataSettings.class.getName());
   private static final String BIGTABLE_EMULATOR_HOST_ENV_VAR = "BIGTABLE_EMULATOR_HOST";
-  private static final AtomicBoolean BUILTIN_METRICS_REGISTERED = new AtomicBoolean(false);
+  // This is the legacy credential override used in the deprecated enableBuiltinMetrics method to
+  // override the default credentials set on the Bigtable client. Keeping it for backward
+  // compatibility.
+  @Deprecated @Nullable private static Credentials legacyMetricCredentialOverride;
 
   private final EnhancedBigtableStubSettings stubSettings;
 
@@ -197,23 +197,34 @@ public static void enableGfeOpenCensusStats() {
     com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews();
   }
 
-  /** Register built in metrics. */
-  public static void enableBuiltinMetrics() throws IOException {
-    if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
-      BuiltinViews.registerBigtableBuiltinViews();
-      BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault());
-    }
-  }
+  /**
+   * Register built in metrics.
+   *
+   * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+   *     now. Please refer to {@link
+   *     BigtableDataSettings.Builder#setMetricsProvider(MetricsProvider)} on how to enable or
+   *     disable built-in metrics.
+   */
+  @Deprecated
+  public static void enableBuiltinMetrics() throws IOException {}
 
   /**
    * Register built in metrics with credentials. The credentials need to have metric write access
    * for all the projects you're publishing to.
+   *
+   * @deprecated This is a no-op that doesn't do anything. Builtin metrics are enabled by default
+   *     now. Please refer {@link BigtableDataSettings.Builder#setMetricsProvider(MetricsProvider)}
+   *     on how to enable or disable built-in metrics.
    */
+  @Deprecated
   public static void enableBuiltinMetrics(Credentials credentials) throws IOException {
-    if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
-      BuiltinViews.registerBigtableBuiltinViews();
-      BigtableStackdriverStatsExporter.register(credentials);
-    }
+    BigtableDataSettings.legacyMetricCredentialOverride = credentials;
+  }
+
+  /** Get the metrics credentials if it's set by {@link #enableBuiltinMetrics(Credentials)}. */
+  @InternalApi
+  public static Credentials getMetricsCredentials() {
+    return legacyMetricCredentialOverride;
   }
 
   /** Returns the target project id. */
@@ -278,6 +289,11 @@ public boolean isBulkMutationFlowControlEnabled() {
     return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
   }
 
+  /** Gets the {@link MetricsProvider}. * */
+  public MetricsProvider getMetricsProvider() {
+    return stubSettings.getMetricsProvider();
+  }
+
   /** Returns the underlying RPC settings. */
   public EnhancedBigtableStubSettings getStubSettings() {
     return stubSettings;
@@ -527,6 +543,30 @@ public boolean isBulkMutationFlowControlEnabled() {
       return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
     }
 
+    /**
+     * Sets the {@link MetricsProvider}.
+     *
+     * <p>By default, this is set to {@link
+     * com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider#INSTANCE} which will
+     * collect and export client side metrics.
+     *
+     * <p>To disable client side metrics, set it to {@link
+     * com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider#INSTANCE}.
+     *
+     * <p>To use a custom OpenTelemetry instance, refer to {@link
+     * com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider} on how to
+     * set it up.
+     */
+    public Builder setMetricsProvider(MetricsProvider metricsProvider) {
+      stubSettings.setMetricsProvider(metricsProvider);
+      return this;
+    }
+
+    /** Gets the {@link MetricsProvider}. */
+    public MetricsProvider getMetricsProvider() {
+      return stubSettings.getMetricsProvider();
+    }
+
     /**
      * Returns the underlying settings for making RPC calls. The settings should be changed with
      * care.

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -15,6 +15,10 @@
  */
 package com.google.cloud.bigtable.data.v2.stub;
 
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PROJECT_ID_KEY;
+
 import com.google.api.core.BetaApi;
 import com.google.api.core.InternalApi;
 import com.google.api.gax.batching.Batcher;
@@ -68,6 +72,7 @@
 import com.google.bigtable.v2.SampleRowKeysRequest;
 import com.google.bigtable.v2.SampleRowKeysResponse;
 import com.google.cloud.bigtable.Version;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
 import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience;
 import com.google.cloud.bigtable.data.v2.internal.RequestContext;
 import com.google.cloud.bigtable.data.v2.models.BulkMutation;
@@ -93,8 +98,13 @@
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsView;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
+import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory;
+import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants;
 import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersServerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersUnaryCallable;
@@ -123,6 +133,11 @@
 import io.opencensus.tags.TagValue;
 import io.opencensus.tags.Tagger;
 import io.opencensus.tags.Tags;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -241,7 +256,7 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
   }
 
   public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings, ClientContext clientContext) {
+      EnhancedBigtableStubSettings settings, ClientContext clientContext) throws IOException {
     return createBigtableTracerFactory(
         settings, Tags.getTagger(), Stats.getStatsRecorder(), clientContext);
   }
@@ -251,7 +266,8 @@ public static ApiTracerFactory createBigtableTracerFactory(
       EnhancedBigtableStubSettings settings,
       Tagger tagger,
       StatsRecorder stats,
-      ClientContext clientContext) {
+      ClientContext clientContext)
+      throws IOException {
     String projectId = settings.getProjectId();
     String instanceId = settings.getInstanceId();
     String appProfileId = settings.getAppProfileId();
@@ -262,16 +278,11 @@ public static ApiTracerFactory createBigtableTracerFactory(
             .put(RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(instanceId))
             .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId))
             .build();
-    ImmutableMap<String, String> builtinAttributes =
-        ImmutableMap.<String, String>builder()
-            .put("project_id", projectId)
-            .put("instance", instanceId)
-            .put("app_profile", appProfileId)
-            .build();
 
     ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
     tracerFactories
         .add(
+            // Add OpenCensus Tracing
             new OpencensusTracerFactory(
                 ImmutableMap.<String, String>builder()
                     // Annotate traces with the same tags as metrics
@@ -285,13 +296,47 @@ public static ApiTracerFactory createBigtableTracerFactory(
                     .build()))
         // Add OpenCensus Metrics
         .add(MetricsTracerFactory.create(tagger, stats, attributes))
-        .add(BuiltinMetricsTracerFactory.create(builtinAttributes))
         // Add user configured tracer
         .add(settings.getTracerFactory());
-
+    Attributes otelAttributes =
+        Attributes.of(
+            PROJECT_ID_KEY, projectId, INSTANCE_ID_KEY, instanceId, APP_PROFILE_KEY, appProfileId);
+    BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
+        createBuiltinMetricsTracerFactory(
+            projectId, settings.getMetricsProvider(), otelAttributes, clientContext);
+    if (builtinMetricsTracerFactory != null) {
+      tracerFactories.add(builtinMetricsTracerFactory);
+    }
     return new CompositeTracerFactory(tracerFactories.build());
   }
 
+  private static BuiltinMetricsTracerFactory createBuiltinMetricsTracerFactory(
+      String projectId,
+      MetricsProvider metricsProvider,
+      Attributes attributes,
+      ClientContext clientContext)
+      throws IOException {
+    if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) {
+      CustomOpenTelemetryMetricsProvider customMetricsProvider =
+          (CustomOpenTelemetryMetricsProvider) metricsProvider;
+      return BuiltinMetricsTracerFactory.create(
+          customMetricsProvider.getOpenTelemetry(), attributes);
+    } else if (metricsProvider instanceof DefaultMetricsProvider) {
+      SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder();
+      Credentials credentials =
+          BigtableDataSettings.getMetricsCredentials() != null
+              ? BigtableDataSettings.getMetricsCredentials()
+              : clientContext.getCredentials();
+      BuiltinMetricsView.registerBuiltinMetrics(projectId, credentials, meterProvider);
+      OpenTelemetry openTelemetry =
+          OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
+      return BuiltinMetricsTracerFactory.create(openTelemetry, attributes);
+    } else if (metricsProvider instanceof NoopMetricsProvider) {
+      return null;
+    }
+    throw new IOException("Invalid MetricsProvider type " + metricsProvider);
+  }
+
   private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)
       throws IOException {
     int i = settings.getEndpoint().lastIndexOf(":");