feat: migrate builtin metrics to OTEl

Author: mutianf

google-cloud-bigtable/pom.xml

@@ -338,6 +338,17 @@
       <artifactId>opentelemetry-api</artifactId>
       <version>${opentelemetry.version}</version>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+      <version>${opentelemetry.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <version>${opentelemetry.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-metrics</artifactId>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -220,12 +220,6 @@ public static EnhancedBigtableStubSettings finalizeSettings(
                 RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID,
                 TagValue.create(settings.getAppProfileId()))
             .build();
-    ImmutableMap<String, String> builtinAttributes =
-        ImmutableMap.<String, String>builder()
-            .put("project_id", settings.getProjectId())
-            .put("instance", settings.getInstanceId())
-            .put("app_profile", settings.getAppProfileId())
-            .build();
     // Inject Opencensus instrumentation
     builder.setTracerFactory(
         new CompositeTracerFactory(
@@ -250,7 +244,7 @@ public static EnhancedBigtableStubSettings finalizeSettings(
                         .build()),
                 // Add OpenCensus Metrics
                 MetricsTracerFactory.create(tagger, stats, attributes),
-                BuiltinMetricsTracerFactory.create(builtinAttributes),
+                BuiltinMetricsTracerFactory.create(settings),
                 // Add user configured tracer
                 settings.getTracerFactory())));
     return builder.build();

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java

@@ -228,6 +228,8 @@ public class EnhancedBigtableStubSettings extends StubSettings<EnhancedBigtableS
 
   private final FeatureFlags featureFlags;
 
+  private final boolean isBuiltinMetricsEnabled;
+
   private EnhancedBigtableStubSettings(Builder builder) {
     super(builder);
 
@@ -252,6 +254,7 @@ private EnhancedBigtableStubSettings(Builder builder) {
     isRefreshingChannel = builder.isRefreshingChannel;
     primedTableIds = builder.primedTableIds;
     jwtAudienceMapping = builder.jwtAudienceMapping;
+    isBuiltinMetricsEnabled = builder.isBuiltinMetricsEnabled;
 
     // Per method settings.
     readRowsSettings = builder.readRowsSettings.build();
@@ -313,6 +316,10 @@ public Map<String, String> getJwtAudienceMapping() {
     return jwtAudienceMapping;
   }
 
+  public boolean isBuiltinMetricsEnabled() {
+    return isBuiltinMetricsEnabled;
+  }
+
   /** Returns a builder for the default ChannelProvider for this service. */
   public static InstantiatingGrpcChannelProvider.Builder defaultGrpcTransportProviderBuilder() {
     return BigtableStubSettings.defaultGrpcTransportProviderBuilder()
@@ -613,6 +620,8 @@ public static class Builder extends StubSettings.Builder<EnhancedBigtableStubSet
 
     private FeatureFlags.Builder featureFlags;
 
+    private boolean isBuiltinMetricsEnabled;
+
     /**
      * Initializes a new Builder with sane defaults for all settings.
      *
@@ -624,6 +633,7 @@ public static class Builder extends StubSettings.Builder<EnhancedBigtableStubSet
     private Builder() {
       this.appProfileId = SERVER_DEFAULT_APP_PROFILE_ID;
       this.isRefreshingChannel = true;
+      this.isBuiltinMetricsEnabled = true;
       primedTableIds = ImmutableList.of();
       jwtAudienceMapping = DEFAULT_JWT_AUDIENCE_MAPPING;
       setCredentialsProvider(defaultCredentialsProviderBuilder().build());
@@ -886,6 +896,17 @@ public Builder setJwtAudienceMapping(Map<String, String> jwtAudienceMapping) {
       return this;
     }
 
+    /** Returns true if builtin metrics is enabled. */
+    public boolean isBuiltinMetricsEnabled() {
+      return isBuiltinMetricsEnabled;
+    }
+
+    /** Set if builtin metrics will be enabled. */
+    public Builder setBuiltinMetricsEnabled(boolean enable) {
+      this.isBuiltinMetricsEnabled = enable;
+      return this;
+    }
+
     @InternalApi("Used for internal testing")
     public Map<String, String> getJwtAudienceMapping() {
       return jwtAudienceMapping;
@@ -1030,6 +1051,7 @@ public String toString() {
             generateInitialChangeStreamPartitionsSettings)
         .add("readChangeStreamSettings", readChangeStreamSettings)
         .add("pingAndWarmSettings", pingAndWarmSettings)
+        .add("isBuiltinMetricsEnabled", isBuiltinMetricsEnabled)
         .add("parent", super.toString())
         .toString();
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

@@ -30,6 +30,7 @@
 import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
 import io.opentelemetry.sdk.metrics.data.MetricData;
 import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -48,7 +49,7 @@ final class BigtableCloudMonitoringExporter implements MetricExporter {
 
   private static final String RESOURCE_TYPE = "bigtable_client_raw";
 
-  BigtableCloudMonitoringExporter(Credentials credentials) throws Exception {
+  BigtableCloudMonitoringExporter(Credentials credentials) throws IOException {
     MetricServiceSettings.Builder settingsBuilder =
         MetricServiceSettings.newBuilder()
             .setTransportChannelProvider(InstantiatingGrpcChannelProvider.newBuilder().build());
@@ -99,7 +100,9 @@ public CompletableResultCode export(Collection<MetricData> collection) {
           logger.log(
               Level.WARNING,
               "Exception thrown when exporting TimeSeries for projectName="
-                  + projectName.getProject(),
+                  + projectName.getProject()
+                  + " metric="
+                  + metricData.getName(),
               e);
         }
       }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java

@@ -67,6 +67,8 @@ class BigtableExporterUtils {
   private static final Logger logger = Logger.getLogger(BigtableExporterUtils.class.getName());
   private static final long NANO_PER_SECOND = (long) 1e9;
 
+  private static final String METRIC_PREFIX = "bigtable.googleapis.com/internal/client/";
+
   static String getDefaultTaskValue() {
     // Something like '<pid>@<hostname>'
     final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
@@ -113,7 +115,7 @@ static TimeSeries convertPointToTimeSeries(
             .setMetricKind(convertMetricKind(metricData))
             .setMetric(
                 Metric.newBuilder()
-                    .setType(metricData.getName())
+                    .setType(METRIC_PREFIX + metricData.getName())
                     .putAllLabels(metricLabels.build())
                     .build())
             .setValueType(convertValueType(metricData.getType()));

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableInstruments.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.stub.metrics;
+
+import io.opentelemetry.api.common.Attributes;
+
+public class BigtableInstruments {
+
+  void recordOperationLatencies(long value, Attributes attributes) {}
+
+  void recordAttemptLatencies(long value, Attributes attributes) {}
+
+  void recordFirstResponseLatencies(long value, Attributes attributes) {}
+
+  void recordRetryCount(long value, Attributes attributes) {}
+
+  void recordServerLatencies(long value, Attributes attributes) {}
+
+  void recordConnectivityErrorCount(long value, Attributes attributes) {}
+
+  void recordApplicationBlockingLatencies(long value, Attributes attributes) {}
+
+  void recordClientBlockingLatencies(long value, Attributes attributes) {}
+}

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinInstruments.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.stub.metrics;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongHistogram;
+import io.opentelemetry.api.metrics.Meter;
+
+class BuiltinInstruments extends BigtableInstruments {
+
+  private static final String MILLISECOND = "ms";
+  private static final String COUNT = "1";
+
+  private final LongHistogram operationLatencies;
+  private final LongHistogram attemptLatencies;
+  private final LongHistogram serverLatencies;
+  private final LongHistogram firstResponseLatencies;
+  private final LongHistogram clientBlockingLatencies;
+  private final LongHistogram applicationBlockingLatencies;
+  private final LongCounter connectivityErrorCount;
+  private final LongCounter retryCount;
+
+  BuiltinInstruments(Meter meter) {
+    operationLatencies =
+        meter
+            .histogramBuilder("operation_latencies")
+            .ofLongs()
+            .setDescription(
+                "Total time until final operation success or failure, including retries and backoff.")
+            .setUnit(MILLISECOND)
+            .build();
+    attemptLatencies =
+        meter
+            .histogramBuilder("attempt_latencies")
+            .ofLongs()
+            .setDescription("Client observed latency per RPC attempt.")
+            .setUnit(MILLISECOND)
+            .build();
+    serverLatencies =
+        meter
+            .histogramBuilder("server_latencies")
+            .ofLongs()
+            .setDescription(
+                "The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.")
+            .setUnit(MILLISECOND)
+            .build();
+    firstResponseLatencies =
+        meter
+            .histogramBuilder("first_response_latencies")
+            .ofLongs()
+            .setDescription(
+                "Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.")
+            .setUnit(MILLISECOND)
+            .build();
+    clientBlockingLatencies =
+        meter
+            .histogramBuilder("throttling_latencies")
+            .ofLongs()
+            .setDescription(
+                "The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.")
+            .setUnit(MILLISECOND)
+            .build();
+    applicationBlockingLatencies =
+        meter
+            .histogramBuilder("application_latencies")
+            .ofLongs()
+            .setDescription(
+                "The latency of the client application consuming available response data.")
+            .setUnit(MILLISECOND)
+            .build();
+    connectivityErrorCount =
+        meter
+            .counterBuilder("connectivity_error_count")
+            .setDescription(
+                "Number of requests that failed to reach the Google datacenter. (Requests without google response headers")
+            .setUnit(COUNT)
+            .build();
+    retryCount =
+        meter
+            .counterBuilder("retry_count")
+            .setDescription("The number of additional RPCs sent after the initial attempt.")
+            .setUnit(COUNT)
+            .build();
+  }
+
+  @Override
+  void recordOperationLatencies(long value, Attributes attributes) {
+    operationLatencies.record(value, attributes);
+  }
+
+  @Override
+  void recordAttemptLatencies(long value, Attributes attributes) {
+    attemptLatencies.record(value, attributes);
+  }
+
+  @Override
+  void recordFirstResponseLatencies(long value, Attributes attributes) {
+    firstResponseLatencies.record(value, attributes);
+  }
+
+  @Override
+  void recordRetryCount(long value, Attributes attributes) {
+    retryCount.add(value, attributes);
+  }
+
+  @Override
+  void recordServerLatencies(long value, Attributes attributes) {
+    serverLatencies.record(value, attributes);
+  }
+
+  @Override
+  void recordConnectivityErrorCount(long value, Attributes attributes) {
+    connectivityErrorCount.add(value, attributes);
+  }
+
+  @Override
+  void recordApplicationBlockingLatencies(long value, Attributes attributes) {
+    applicationBlockingLatencies.record(value, attributes);
+  }
+
+  @Override
+  void recordClientBlockingLatencies(long value, Attributes attributes) {
+    clientBlockingLatencies.record(value, attributes);
+  }
+}

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsAttributes.java

@@ -17,7 +17,7 @@
 
 import io.opentelemetry.api.common.AttributeKey;
 
-class BuiltinMetricsAttributes {
+public class BuiltinMetricsAttributes {
 
   public static final AttributeKey<String> PROJECT_ID = AttributeKey.stringKey("project_id");
   public static final AttributeKey<String> INSTANCE_ID = AttributeKey.stringKey("instance");
@@ -26,7 +26,7 @@ class BuiltinMetricsAttributes {
   public static final AttributeKey<String> ZONE_ID = AttributeKey.stringKey("zone");
   static final AttributeKey<String> CLIENT_UID = AttributeKey.stringKey("client_uid");
 
-  static final AttributeKey<String> APP_PROFILE = AttributeKey.stringKey("app_profile");
+  public static final AttributeKey<String> APP_PROFILE = AttributeKey.stringKey("app_profile");
   static final AttributeKey<Boolean> STREAMING = AttributeKey.booleanKey("streaming");
   static final AttributeKey<String> METHOD = AttributeKey.stringKey("method");
   static final AttributeKey<String> STATUS = AttributeKey.stringKey("status");