feat(metrics): Mitigate potential performance issues with realm_id tag

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMeterFilterProducer.java

@@ -20,21 +20,42 @@
 
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.config.MeterFilter;
+import io.micrometer.core.instrument.internal.OnlyOnceLoggingDenyMeterFilter;
+import io.quarkus.micrometer.runtime.binder.HttpBinderConfiguration;
 import jakarta.enterprise.inject.Produces;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 import java.util.stream.Collectors;
 
 public class QuarkusMeterFilterProducer {
 
-  @Inject QuarkusMetricsConfiguration configuration;
+  @Inject QuarkusMetricsConfiguration metricsConfiguration;
+  @Inject HttpBinderConfiguration binderConfiguration;
 
   @Produces
   @Singleton
-  public MeterFilter produceGlobalMeterFilter() {
+  public MeterFilter commonTagsFilter() {
     return MeterFilter.commonTags(
-        this.configuration.tags().entrySet().stream()
+        this.metricsConfiguration.tags().entrySet().stream()
             .map(e -> Tag.of(e.getKey(), e.getValue()))
             .collect(Collectors.toSet()));
   }
+
+  @Produces
+  @Singleton
+  public MeterFilter maxRealmIdTagsInHttpMetricsFilter() {
+    MeterFilter denyFilter =
+        new OnlyOnceLoggingDenyMeterFilter(
+            () ->
+                String.format(
+                    "Reached the maximum number (%s) of '%s' tags for '%s'",
+                    metricsConfiguration.realmIdTag().httpMetricsMaxCardinality(),
+                    RealmIdTagContributor.TAG_REALM,
+                    binderConfiguration.getHttpServerRequestsName()));
+    return MeterFilter.maximumAllowableTags(
+        binderConfiguration.getHttpServerRequestsName(),
+        RealmIdTagContributor.TAG_REALM,
+        metricsConfiguration.realmIdTag().httpMetricsMaxCardinality(),
+        denyFilter);
+  }
 }

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusMetricsConfiguration.java

@@ -19,11 +19,50 @@
 package org.apache.polaris.service.quarkus.metrics;
 
 import io.smallrye.config.ConfigMapping;
+import io.smallrye.config.WithDefault;
+import jakarta.validation.constraints.Min;
 import java.util.Map;
 
 @ConfigMapping(prefix = "polaris.metrics")
 public interface QuarkusMetricsConfiguration {
 
   /** Additional tags to include in the metrics. */
   Map<String, String> tags();
+
+  /** Configuration for the Realm ID metric tag. */
+  RealmIdTag realmIdTag();
+
+  interface RealmIdTag {
+
+    /**
+     * Whether to include the Realm ID tag in the API request metrics.
+     *
+     * <p>Beware that if the cardinality of this tag is too high, it can cause performance issues or
+     * even crash the server.
+     */
+    @WithDefault("false")
+    boolean enableInApiMetrics();
+
+    /**
+     * Whether to include the Realm ID tag in the HTTP server request metrics.
+     *
+     * <p>Beware that if the cardinality of this tag is too high, it can cause performance issues or
+     * even crash the server.
+     */
+    @WithDefault("false")
+    boolean enableInHttpMetrics();
+
+    /**
+     * The maximum number of Realm ID tag values allowed for the HTTP server request metrics.
+     *
+     * <p>This is used to prevent the number of tags from growing indefinitely and causing
+     * performance issues or crashing the server.
+     *
+     * <p>If the number of tags exceeds this value, a warning will be logged and no more HTTP server
+     * request metrics will be recorded.
+     */
+    @WithDefault("100")
+    @Min(1)
+    int httpMetricsMaxCardinality();
+  }
 }

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/QuarkusValueExpressionResolver.java

@@ -22,17 +22,23 @@
 import io.micrometer.common.lang.Nullable;
 import jakarta.annotation.Nonnull;
 import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
 import org.apache.polaris.core.context.RealmContext;
 
 @ApplicationScoped
 public class QuarkusValueExpressionResolver implements ValueExpressionResolver {
 
+  @Inject QuarkusMetricsConfiguration metricsConfiguration;
+
   @Override
   public String resolve(@Nonnull String expression, @Nullable Object parameter) {
     // TODO maybe replace with CEL of some expression engine and make this more generic
-    if (parameter instanceof RealmContext realmContext && expression.equals("realmIdentifier")) {
+    if (metricsConfiguration.realmIdTag().enableInApiMetrics()
+        && parameter instanceof RealmContext realmContext
+        && expression.equals("realmIdentifier")) {
       return realmContext.getRealmIdentifier();
     }
-    return null;
+    // FIXME cannot return null here, see https://github.com/quarkusio/quarkus/issues/47891
+    return "";
   }
 }

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagContributor.java

@@ -33,12 +33,16 @@ public class RealmIdTagContributor implements HttpServerMetricsTagsContributor {
 
   private static final Tags UNFINISHED_RESOLUTION_TAGS = Tags.of(TAG_REALM, "???");
 
+  @Inject QuarkusMetricsConfiguration metricsConfiguration;
   @Inject RealmContextResolver realmContextResolver;
 
   @Override
   public Tags contribute(Context context) {
-    // FIXME retrieve the realm context from context.requestContextLocalData() when this PR is in:
-    // https://github.com/quarkusio/quarkus/pull/47887
+    // FIXME retrieve the realm context from context.requestContextLocalData()
+    // after upgrading to Quarkus 3.24
+    if (!metricsConfiguration.realmIdTag().enableInHttpMetrics()) {
+      return Tags.empty();
+    }
     HttpServerRequest request = context.request();
     try {
       return realmContextResolver

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TimedApplicationEventListenerTest.java → quarkus/service/src/test/java/org/apache/polaris/service/quarkus/metrics/MetricsTestBase.java

@@ -16,20 +16,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.polaris.service.quarkus;
+package org.apache.polaris.service.quarkus.metrics;
 
 import static org.apache.polaris.service.context.TestRealmContextResolver.REALM_PROPERTY_KEY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.InstanceOfAssertFactories.type;
 
 import io.micrometer.core.instrument.MeterRegistry;
-import io.quarkus.test.junit.QuarkusTest;
-import io.quarkus.test.junit.QuarkusTestProfile;
-import io.quarkus.test.junit.TestProfile;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.core.Response;
 import java.util.Map;
-import org.apache.polaris.service.quarkus.TimedApplicationEventListenerTest.Profile;
 import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestFixture;
 import org.apache.polaris.service.quarkus.test.PolarisIntegrationTestHelper;
 import org.apache.polaris.service.quarkus.test.TestEnvironment;
@@ -46,27 +42,18 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
-@QuarkusTest
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 @ExtendWith(TestEnvironmentExtension.class)
-@TestProfile(Profile.class)
-public class TimedApplicationEventListenerTest {
-
-  public static class Profile implements QuarkusTestProfile {
-
-    @Override
-    public Map<String, String> getConfigOverrides() {
-      return Map.of(
-          "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test");
-    }
-  }
+public abstract class MetricsTestBase {
 
   private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode();
   private static final String ENDPOINT = "api/management/v1/principals";
-  private static final String METRIC_NAME = "polaris_principals_getPrincipal_seconds";
+  private static final String API_METRIC_NAME = "polaris_principals_getPrincipal_seconds";
+  private static final String HTTP_METRIC_NAME = "http_server_requests_seconds";
 
   @Inject PolarisIntegrationTestHelper helper;
   @Inject MeterRegistry registry;
+  @Inject QuarkusMetricsConfiguration metricsConfiguration;
 
   private TestEnvironment testEnv;
   private PolarisIntegrationTestFixture fixture;
@@ -88,15 +75,19 @@ public void testMetricsEmittedOnSuccessfulRequest(String endpoint) {
     sendSuccessfulRequest();
     Map<String, MetricFamily> allMetrics =
         TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri(), endpoint);
-    assertThat(allMetrics).containsKey(METRIC_NAME);
-    assertThat(allMetrics.get(METRIC_NAME).getMetrics())
+    assertThat(allMetrics).containsKey(API_METRIC_NAME);
+    assertThat(allMetrics.get(API_METRIC_NAME).getMetrics())
         .satisfiesOnlyOnce(
             metric -> {
               assertThat(metric.getLabels())
                   .contains(
                       Map.entry("application", "Polaris"),
                       Map.entry("environment", "prod"),
-                      Map.entry("realm_id", fixture.realm),
+                      Map.entry(
+                          "realm_id",
+                          metricsConfiguration.realmIdTag().enableInApiMetrics()
+                              ? fixture.realm
+                              : ""),
                       Map.entry(
                           "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"),
                       Map.entry("exception", "none"),
@@ -106,6 +97,28 @@ public void testMetricsEmittedOnSuccessfulRequest(String endpoint) {
                   .extracting(Summary::getSampleCount)
                   .isEqualTo(1L);
             });
+    assertThat(allMetrics).containsKey(HTTP_METRIC_NAME);
+    assertThat(allMetrics.get(HTTP_METRIC_NAME).getMetrics())
+        .satisfiesOnlyOnce(
+            metric -> {
+              assertThat(metric.getLabels())
+                  .contains(
+                      Map.entry("application", "Polaris"),
+                      Map.entry("environment", "prod"),
+                      Map.entry("method", "GET"),
+                      Map.entry("outcome", "SUCCESS"),
+                      Map.entry("status", "200"),
+                      Map.entry("uri", "/api/management/v1/principals/{principalName}"));
+              if (metricsConfiguration.realmIdTag().enableInHttpMetrics()) {
+                assertThat(metric.getLabels()).containsEntry("realm_id", fixture.realm);
+              } else {
+                assertThat(metric.getLabels()).doesNotContainKey("realm_id");
+              }
+              assertThat(metric)
+                  .asInstanceOf(type(Summary.class))
+                  .extracting(Summary::getSampleCount)
+                  .isEqualTo(1L);
+            });
   }
 
   @ParameterizedTest
@@ -114,15 +127,19 @@ public void testMetricsEmittedOnFailedRequest(String endpoint) {
     sendFailingRequest();
     Map<String, MetricFamily> allMetrics =
         TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri(), endpoint);
-    assertThat(allMetrics).containsKey(METRIC_NAME);
-    assertThat(allMetrics.get(METRIC_NAME).getMetrics())
+    assertThat(allMetrics).containsKey(API_METRIC_NAME);
+    assertThat(allMetrics.get(API_METRIC_NAME).getMetrics())
         .satisfiesOnlyOnce(
             metric -> {
               assertThat(metric.getLabels())
                   .contains(
                       Map.entry("application", "Polaris"),
                       Map.entry("environment", "prod"),
-                      Map.entry("realm_id", fixture.realm),
+                      Map.entry(
+                          "realm_id",
+                          metricsConfiguration.realmIdTag().enableInApiMetrics()
+                              ? fixture.realm
+                              : ""),
                       Map.entry(
                           "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"),
                       Map.entry("exception", "NotFoundException"),
@@ -132,6 +149,27 @@ public void testMetricsEmittedOnFailedRequest(String endpoint) {
                   .extracting(Summary::getSampleCount)
                   .isEqualTo(1L);
             });
+    assertThat(allMetrics.get(HTTP_METRIC_NAME).getMetrics())
+        .satisfiesOnlyOnce(
+            metric -> {
+              assertThat(metric.getLabels())
+                  .contains(
+                      Map.entry("application", "Polaris"),
+                      Map.entry("environment", "prod"),
+                      Map.entry("method", "GET"),
+                      Map.entry("outcome", "CLIENT_ERROR"),
+                      Map.entry("status", "404"),
+                      Map.entry("uri", "/api/management/v1/principals/{principalName}"));
+              if (metricsConfiguration.realmIdTag().enableInHttpMetrics()) {
+                assertThat(metric.getLabels()).containsEntry("realm_id", fixture.realm);
+              } else {
+                assertThat(metric.getLabels()).doesNotContainKey("realm_id");
+              }
+              assertThat(metric)
+                  .asInstanceOf(type(Summary.class))
+                  .extracting(Summary::getSampleCount)
+                  .isEqualTo(1L);
+            });
   }
 
   private int sendRequest(String principalName) {

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagDisabledMetricsTest.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.quarkus.metrics;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import java.util.Map;
+
+@QuarkusTest
+@TestProfile(RealmIdTagDisabledMetricsTest.Profile.class)
+public class RealmIdTagDisabledMetricsTest extends MetricsTestBase {
+
+  public static class Profile implements QuarkusTestProfile {
+
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return Map.of(
+          "polaris.metrics.tags.environment", "prod", "polaris.realm-context.type", "test");
+    }
+  }
+}

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/metrics/RealmIdTagEnabledMetricsTest.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.quarkus.metrics;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.QuarkusTestProfile;
+import io.quarkus.test.junit.TestProfile;
+import java.util.Map;
+
+@QuarkusTest
+@TestProfile(RealmIdTagEnabledMetricsTest.Profile.class)
+public class RealmIdTagEnabledMetricsTest extends MetricsTestBase {
+
+  public static class Profile implements QuarkusTestProfile {
+
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return Map.of(
+          "polaris.metrics.tags.environment",
+          "prod",
+          "polaris.realm-context.type",
+          "test",
+          "polaris.metrics.realm-id-tag.enable-in-api-metrics",
+          "true",
+          "polaris.metrics.realm-id-tag.enable-in-http-metrics",
+          "true");
+    }
+  }
+}