feat: migrate per connection error count metric to otel (#2133)

* feat: migrate per connection error count metric to otel

* update test

* address comments

* remove unnecessary check

* clean up statsRecorder

* remove dependency

* address comments

Author: mutianf

google-cloud-bigtable/clirr-ignored-differences.xml

@@ -186,4 +186,10 @@
         <className>com/google/cloud/bigtable/data/v2/models/MutateRowsException</className>
         <method>*</method>
     </difference>
+    <!-- InternalApi was updated -->
+    <difference>
+        <differenceType>7004</differenceType>
+        <className>com/google/cloud/bigtable/data/v2/stub/metrics/ErrorCountPerConnectionMetricTracker</className>
+        <method>*</method>
+    </difference>
 </differences>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java

@@ -19,6 +19,7 @@
 import com.google.api.gax.core.BackgroundResource;
 import com.google.api.gax.rpc.ClientContext;
 import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub;
+import io.opentelemetry.api.OpenTelemetry;
 import java.io.IOException;
 import javax.annotation.Nonnull;
 
@@ -64,6 +65,7 @@
 public final class BigtableDataClientFactory implements AutoCloseable {
   private final BigtableDataSettings defaultSettings;
   private final ClientContext sharedClientContext;
+  private final OpenTelemetry openTelemetry;
 
   /**
    * Create a instance of this factory.
@@ -75,13 +77,21 @@ public static BigtableDataClientFactory create(BigtableDataSettings defaultSetti
       throws IOException {
     ClientContext sharedClientContext =
         EnhancedBigtableStub.createClientContext(defaultSettings.getStubSettings());
-    return new BigtableDataClientFactory(sharedClientContext, defaultSettings);
+    OpenTelemetry openTelemetry =
+        EnhancedBigtableStub.getOpenTelemetry(
+            defaultSettings.getProjectId(),
+            defaultSettings.getMetricsProvider(),
+            sharedClientContext.getCredentials());
+    return new BigtableDataClientFactory(sharedClientContext, defaultSettings, openTelemetry);
   }
 
   private BigtableDataClientFactory(
-      ClientContext sharedClientContext, BigtableDataSettings defaultSettings) {
+      ClientContext sharedClientContext,
+      BigtableDataSettings defaultSettings,
+      OpenTelemetry openTelemetry) {
     this.sharedClientContext = sharedClientContext;
     this.defaultSettings = defaultSettings;
+    this.openTelemetry = openTelemetry;
   }
 
   /**
@@ -112,7 +122,7 @@ public BigtableDataClient createDefault() {
               .toBuilder()
               .setTracerFactory(
                   EnhancedBigtableStub.createBigtableTracerFactory(
-                      defaultSettings.getStubSettings(), sharedClientContext))
+                      defaultSettings.getStubSettings(), openTelemetry))
               .build();
 
       return BigtableDataClient.createWithClientContext(defaultSettings, clientContext);
@@ -141,7 +151,7 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro
             .toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext))
+                    settings.getStubSettings(), openTelemetry))
             .build();
     return BigtableDataClient.createWithClientContext(settings, clientContext);
   }
@@ -170,7 +180,7 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull
             .toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext))
+                    settings.getStubSettings(), openTelemetry))
             .build();
 
     return BigtableDataClient.createWithClientContext(settings, clientContext);
@@ -200,7 +210,7 @@ public BigtableDataClient createForInstance(
             .toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext))
+                    settings.getStubSettings(), openTelemetry))
             .build();
     return BigtableDataClient.createWithClientContext(settings, clientContext);
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -16,6 +16,7 @@
 package com.google.cloud.bigtable.data.v2.stub;
 
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_NAME_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PROJECT_ID_KEY;
 
@@ -99,12 +100,11 @@
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory;
-import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsView;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
-import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.ErrorCountPerConnectionMetricTracker;
+import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory;
 import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
 import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants;
@@ -138,9 +138,6 @@
 import io.opencensus.tags.Tags;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.sdk.OpenTelemetrySdk;
-import io.opentelemetry.sdk.metrics.SdkMeterProvider;
-import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -194,10 +191,13 @@ public class EnhancedBigtableStub implements AutoCloseable {
   public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
       throws IOException {
     ClientContext clientContext = createClientContext(settings);
+    OpenTelemetry openTelemetry =
+        getOpenTelemetry(
+            settings.getProjectId(), settings.getMetricsProvider(), clientContext.getCredentials());
     ClientContext contextWithTracer =
         clientContext
             .toBuilder()
-            .setTracerFactory(createBigtableTracerFactory(settings, clientContext))
+            .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry))
             .build();
 
     return new EnhancedBigtableStub(settings, contextWithTracer);
@@ -219,15 +219,26 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
     // workaround JWT audience issues
     patchCredentials(builder);
 
+    // Fix the credentials so that they can be shared
+    Credentials credentials = null;
+    if (builder.getCredentialsProvider() != null) {
+      credentials = builder.getCredentialsProvider().getCredentials();
+    }
+    builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
+
     InstantiatingGrpcChannelProvider.Builder transportProvider =
         builder.getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider
             ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder()
             : null;
 
+    OpenTelemetry openTelemetry =
+        getOpenTelemetry(settings.getProjectId(), settings.getMetricsProvider(), credentials);
     ErrorCountPerConnectionMetricTracker errorCountPerConnectionMetricTracker;
-    if (transportProvider != null) {
+    // Skip setting up ErrorCountPerConnectionMetricTracker if openTelemetry is null
+    if (openTelemetry != null && transportProvider != null) {
       errorCountPerConnectionMetricTracker =
-          new ErrorCountPerConnectionMetricTracker(createBuiltinAttributes(builder));
+          new ErrorCountPerConnectionMetricTracker(
+              openTelemetry, createBuiltinAttributes(settings));
       ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> oldChannelConfigurator =
           transportProvider.getChannelConfigurator();
       transportProvider.setChannelConfigurator(
@@ -249,12 +260,6 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
 
     // Inject channel priming
     if (settings.isRefreshingChannel()) {
-      // Fix the credentials so that they can be shared
-      Credentials credentials = null;
-      if (builder.getCredentialsProvider() != null) {
-        credentials = builder.getCredentialsProvider().getCredentials();
-      }
-      builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
 
       if (transportProvider != null) {
         transportProvider.setChannelPrimer(
@@ -279,17 +284,17 @@ public static ClientContext createClientContext(EnhancedBigtableStubSettings set
   }
 
   public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings, ClientContext clientContext) throws IOException {
+      EnhancedBigtableStubSettings settings, OpenTelemetry openTelemetry) throws IOException {
     return createBigtableTracerFactory(
-        settings, Tags.getTagger(), Stats.getStatsRecorder(), clientContext);
+        settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry);
   }
 
   @VisibleForTesting
   public static ApiTracerFactory createBigtableTracerFactory(
       EnhancedBigtableStubSettings settings,
       Tagger tagger,
       StatsRecorder stats,
-      ClientContext clientContext)
+      OpenTelemetry openTelemetry)
       throws IOException {
     String projectId = settings.getProjectId();
     String instanceId = settings.getInstanceId();
@@ -302,8 +307,6 @@ public static ApiTracerFactory createBigtableTracerFactory(
             .put(RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, TagValue.create(appProfileId))
             .build();
 
-    ImmutableMap<String, String> builtinAttributes = createBuiltinAttributes(settings.toBuilder());
-
     ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
     tracerFactories
         .add(
@@ -323,53 +326,47 @@ public static ApiTracerFactory createBigtableTracerFactory(
         .add(MetricsTracerFactory.create(tagger, stats, attributes))
         // Add user configured tracer
         .add(settings.getTracerFactory());
-    Attributes otelAttributes =
-        Attributes.of(
-            PROJECT_ID_KEY, projectId, INSTANCE_ID_KEY, instanceId, APP_PROFILE_KEY, appProfileId);
     BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
-        createBuiltinMetricsTracerFactory(
-            projectId, settings.getMetricsProvider(), otelAttributes, clientContext);
+        openTelemetry != null
+            ? BuiltinMetricsTracerFactory.create(openTelemetry, createBuiltinAttributes(settings))
+            : null;
     if (builtinMetricsTracerFactory != null) {
       tracerFactories.add(builtinMetricsTracerFactory);
     }
     return new CompositeTracerFactory(tracerFactories.build());
   }
 
-  private static BuiltinMetricsTracerFactory createBuiltinMetricsTracerFactory(
-      String projectId,
-      MetricsProvider metricsProvider,
-      Attributes attributes,
-      ClientContext clientContext)
+  @Nullable
+  public static OpenTelemetry getOpenTelemetry(
+      String projectId, MetricsProvider metricsProvider, @Nullable Credentials defaultCredentials)
       throws IOException {
     if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) {
       CustomOpenTelemetryMetricsProvider customMetricsProvider =
           (CustomOpenTelemetryMetricsProvider) metricsProvider;
-      return BuiltinMetricsTracerFactory.create(
-          customMetricsProvider.getOpenTelemetry(), attributes);
+      return customMetricsProvider.getOpenTelemetry();
     } else if (metricsProvider instanceof DefaultMetricsProvider) {
-      SdkMeterProviderBuilder meterProvider = SdkMeterProvider.builder();
       Credentials credentials =
           BigtableDataSettings.getMetricsCredentials() != null
               ? BigtableDataSettings.getMetricsCredentials()
-              : clientContext.getCredentials();
-      BuiltinMetricsView.registerBuiltinMetrics(projectId, credentials, meterProvider);
-      OpenTelemetry openTelemetry =
-          OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build();
-      return BuiltinMetricsTracerFactory.create(openTelemetry, attributes);
+              : defaultCredentials;
+      DefaultMetricsProvider defaultMetricsProvider = (DefaultMetricsProvider) metricsProvider;
+      return defaultMetricsProvider.getOpenTelemetry(projectId, credentials);
     } else if (metricsProvider instanceof NoopMetricsProvider) {
       return null;
     }
     throw new IOException("Invalid MetricsProvider type " + metricsProvider);
   }
 
-  private static ImmutableMap<String, String> createBuiltinAttributes(
-      EnhancedBigtableStubSettings.Builder builder) {
-    return ImmutableMap.<String, String>builder()
-        .put("project_id", builder.getProjectId())
-        .put("instance", builder.getInstanceId())
-        .put("app_profile", builder.getAppProfileId())
-        .put("client_name", "bigtable-java/" + Version.VERSION)
-        .build();
+  private static Attributes createBuiltinAttributes(EnhancedBigtableStubSettings settings) {
+    return Attributes.of(
+        PROJECT_ID_KEY,
+        settings.getProjectId(),
+        INSTANCE_ID_KEY,
+        settings.getInstanceId(),
+        APP_PROFILE_KEY,
+        settings.getAppProfileId(),
+        CLIENT_NAME_KEY,
+        "bigtable-java/" + Version.VERSION);
   }
 
   private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

@@ -20,7 +20,9 @@
 import com.google.api.core.ApiFutureCallback;
 import com.google.api.core.ApiFutures;
 import com.google.api.core.InternalApi;
+import com.google.api.gax.core.CredentialsProvider;
 import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.api.gax.core.NoCredentialsProvider;
 import com.google.auth.Credentials;
 import com.google.cloud.monitoring.v3.MetricServiceClient;
 import com.google.cloud.monitoring.v3.MetricServiceSettings;
@@ -43,6 +45,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.annotation.Nullable;
 import org.threeten.bp.Duration;
 
 /**
@@ -75,10 +78,16 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter {
 
   private CompletableResultCode lastExportCode;
 
-  public static BigtableCloudMonitoringExporter create(String projectId, Credentials credentials)
-      throws IOException {
+  public static BigtableCloudMonitoringExporter create(
+      String projectId, @Nullable Credentials credentials) throws IOException {
     MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder();
-    settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials));
+    CredentialsProvider credentialsProvider;
+    if (credentials == null) {
+      credentialsProvider = NoCredentialsProvider.create();
+    } else {
+      credentialsProvider = FixedCredentialsProvider.create(credentials);
+    }
+    settingsBuilder.setCredentialsProvider(credentialsProvider);
     settingsBuilder.setEndpoint(MONITORING_ENDPOINT);
 
     org.threeten.bp.Duration timeout = Duration.ofMinutes(1);