Wire up Vert.x metrics in fetcher docker with Prometheus Otel reader

docker/fetcher/start.sh

@@ -18,11 +18,27 @@ else
   ONLINE_CLASS=$GCP_ONLINE_CLASS
 fi
 
-if [ -z "$EXPORTER_OTLP_ENDPOINT" ]; then
-  echo "OpenTelemetry endpoint not configured. Disabling metrics reporting"
-  METRICS_ENABLED="false"
+METRICS_OPTS=""
+
+if [ -n "$CHRONON_METRICS_READER" ]; then
+  METRICS_OPTS="-Dai.chronon.metrics.enabled=true"
+  METRICS_OPTS="$METRICS_OPTS -Dai.chronon.metrics.reader=$CHRONON_METRICS_READER"
+
+  if [ "$CHRONON_METRICS_READER" = "http" ]; then
+    if [ -n "$EXPORTER_OTLP_ENDPOINT" ]; then
+      METRICS_OPTS="$METRICS_OPTS -Dai.chronon.metrics.exporter.url=$EXPORTER_OTLP_ENDPOINT"
+    fi
+  elif [ "$CHRONON_METRICS_READER" = "prometheus" ]; then
+    if [ -n "$CHRONON_PROMETHEUS_SERVER_PORT" ]; then
+      METRICS_OPTS="$METRICS_OPTS -Dai.chronon.metrics.exporter.port=$CHRONON_PROMETHEUS_SERVER_PORT"
+    fi
+    if [ -n "$VERTX_PROMETHEUS_SERVER_PORT" ]; then
+      METRICS_OPTS="$METRICS_OPTS -Dai.chronon.vertx.metrics.exporter.port=$VERTX_PROMETHEUS_SERVER_PORT"
+    fi
+  fi
 else
-  METRICS_ENABLED="true"
+  echo "No CHRONON_METRICS_READER configured. Disabling metrics reporting"
+  METRICS_OPTS="-Dai.chronon.metrics.enabled=false"
 fi
 
 JMX_OPTS="-XX:MaxMetaspaceSize=1g -XX:MaxRAMPercentage=70.0 -XX:MinRAMPercentage=70.0 -XX:InitialRAMPercentage=70.0 -XX:MaxHeapFreeRatio=100 -XX:MinHeapFreeRatio=0"
@@ -33,8 +49,7 @@ if ! java -jar $FETCHER_JAR run ai.chronon.service.FetcherVerticle \
   $JMX_OPTS \
   -Dserver.port=$FETCHER_PORT \
   -Donline.jar=$ONLINE_JAR \
-  -Dai.chronon.metrics.enabled=$METRICS_ENABLED \
-  -Dai.chronon.metrics.exporter.url=$EXPORTER_OTLP_ENDPOINT \
+  $METRICS_OPTS \
   -Donline.class=$ONLINE_CLASS; then
   echo "Error: Fetcher service failed to start"
   exit 1

maven_install.json

@@ -1,7 +1,7 @@
 {
   "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
-  "__INPUT_ARTIFACTS_HASH": 884577385,
-  "__RESOLVED_ARTIFACTS_HASH": -1551679832,
+  "__INPUT_ARTIFACTS_HASH": 845749994,
+  "__RESOLVED_ARTIFACTS_HASH": 1986370254,
   "artifacts": {
     "ant:ant": {
       "shasums": {
@@ -1782,6 +1782,13 @@
       },
       "version": "1.13.6"
     },
+    "io.micrometer:micrometer-registry-prometheus": {
+      "shasums": {
+        "jar": "65da78a5b803cc78da0c550db9d5c32f543cf68768199452de81d9c3682345ac",
+        "sources": "0d4a129b455c158a6ac061d729a4a6197305c90363b7cec5e499617d858f11d9"
+      },
+      "version": "1.10.13"
+    },
     "io.micrometer:micrometer-registry-statsd": {
       "shasums": {
         "jar": "0d57c12a34ab6e87d58d8b0ecde8ec592bcd2818e39a64c7314d8a0728eab5b6",
@@ -2296,6 +2303,41 @@
       },
       "version": "1.3.6"
     },
+    "io.prometheus:simpleclient": {
+      "shasums": {
+        "jar": "22c374f237f7bc4fdb1f0ec2da379c0ba00e69ad2ffe8b6ade543d73062d377f",
+        "sources": "b357c17f5f3c9d0764059641a118447c03495d18b96497d32b8d2d0f021897d5"
+      },
+      "version": "0.16.0"
+    },
+    "io.prometheus:simpleclient_common": {
+      "shasums": {
+        "jar": "eba6ec26ce7e40cbb8725e05fb83247a9f4f44945b9e7522e3375dde67b9f059",
+        "sources": "b9b3c08dd716e3bc7f791f590d413c956115b0c09469d4330edfe33d6c14c673"
+      },
+      "version": "0.16.0"
+    },
+    "io.prometheus:simpleclient_tracer_common": {
+      "shasums": {
+        "jar": "e84a784ac8e24f182ee62da80b6b5c8140774b75bfa4bf9cc3d3b8390db625f6",
+        "sources": "6fa35352284e1a702c44583d55bea576facc1f515ff0a218102117020b9cd7ed"
+      },
+      "version": "0.16.0"
+    },
+    "io.prometheus:simpleclient_tracer_otel": {
+      "shasums": {
+        "jar": "a2a84c59bef3796bb7f9c6f2ae8b7de8b905313ef28180ac3de7ff61dd68df3f",
+        "sources": "25e4fdd07e904b9dd617121cc591a5bf73bdabf15cb741319849aae577e02387"
+      },
+      "version": "0.16.0"
+    },
+    "io.prometheus:simpleclient_tracer_otel_agent": {
+      "shasums": {
+        "jar": "7ad2bb40df74a772d9f5445a3d03579b49d6b37a47d5e90f6cf3f59ecf41ad06",
+        "sources": "3085f16286869e7a58dc04501dd44304949c8a33e01b7edd3dfa2161d79a0227"
+      },
+      "version": "0.16.0"
+    },
     "io.swagger.core.v3:swagger-annotations": {
       "shasums": {
         "jar": "59573c4d6357c2121d40069959879cf008783cc8208dc5123f759b0e6a0077ad",
@@ -6737,6 +6779,10 @@
       "io.micrometer:micrometer-core",
       "io.opentelemetry.proto:opentelemetry-proto"
     ],
+    "io.micrometer:micrometer-registry-prometheus": [
+      "io.micrometer:micrometer-core",
+      "io.prometheus:simpleclient_common"
+    ],
     "io.micrometer:micrometer-registry-statsd": [
       "io.micrometer:micrometer-core"
     ],
@@ -7078,6 +7124,19 @@
       "io.prometheus:prometheus-metrics-config",
       "io.prometheus:prometheus-metrics-model"
     ],
+    "io.prometheus:simpleclient": [
+      "io.prometheus:simpleclient_tracer_otel",
+      "io.prometheus:simpleclient_tracer_otel_agent"
+    ],
+    "io.prometheus:simpleclient_common": [
+      "io.prometheus:simpleclient"
+    ],
+    "io.prometheus:simpleclient_tracer_otel": [
+      "io.prometheus:simpleclient_tracer_common"
+    ],
+    "io.prometheus:simpleclient_tracer_otel_agent": [
+      "io.prometheus:simpleclient_tracer_common"
+    ],
     "io.temporal:temporal-sdk": [
       "com.fasterxml.jackson.core:jackson-databind",
       "com.fasterxml.jackson.datatype:jackson-datatype-jdk8",
@@ -12164,6 +12223,9 @@
     "io.micrometer:micrometer-registry-otlp": [
       "io.micrometer.registry.otlp"
     ],
+    "io.micrometer:micrometer-registry-prometheus": [
+      "io.micrometer.prometheus"
+    ],
     "io.micrometer:micrometer-registry-statsd": [
       "io.micrometer.shaded.io.netty.bootstrap",
       "io.micrometer.shaded.io.netty.buffer",
@@ -12695,6 +12757,22 @@
       "io.prometheus.metrics.model.registry",
       "io.prometheus.metrics.model.snapshots"
     ],
+    "io.prometheus:simpleclient": [
+      "io.prometheus.client",
+      "io.prometheus.client.exemplars"
+    ],
+    "io.prometheus:simpleclient_common": [
+      "io.prometheus.client.exporter.common"
+    ],
+    "io.prometheus:simpleclient_tracer_common": [
+      "io.prometheus.client.exemplars.tracer.common"
+    ],
+    "io.prometheus:simpleclient_tracer_otel": [
+      "io.prometheus.client.exemplars.tracer.otel"
+    ],
+    "io.prometheus:simpleclient_tracer_otel_agent": [
+      "io.prometheus.client.exemplars.tracer.otel_agent"
+    ],
     "io.swagger.core.v3:swagger-annotations": [
       "io.swagger.v3.oas.annotations",
       "io.swagger.v3.oas.annotations.callbacks",
@@ -24620,6 +24698,8 @@
       "io.micrometer:micrometer-observation:jar:sources",
       "io.micrometer:micrometer-registry-otlp",
       "io.micrometer:micrometer-registry-otlp:jar:sources",
+      "io.micrometer:micrometer-registry-prometheus",
+      "io.micrometer:micrometer-registry-prometheus:jar:sources",
       "io.micrometer:micrometer-registry-statsd",
       "io.micrometer:micrometer-registry-statsd:jar:sources",
       "io.netty:netty-all",
@@ -24772,6 +24852,16 @@
       "io.prometheus:prometheus-metrics-exposition-textformats:jar:sources",
       "io.prometheus:prometheus-metrics-model",
       "io.prometheus:prometheus-metrics-model:jar:sources",
+      "io.prometheus:simpleclient",
+      "io.prometheus:simpleclient:jar:sources",
+      "io.prometheus:simpleclient_common",
+      "io.prometheus:simpleclient_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_common",
+      "io.prometheus:simpleclient_tracer_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel",
+      "io.prometheus:simpleclient_tracer_otel:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel_agent",
+      "io.prometheus:simpleclient_tracer_otel_agent:jar:sources",
       "io.swagger.core.v3:swagger-annotations",
       "io.swagger.core.v3:swagger-annotations:jar:sources",
       "io.temporal:temporal-sdk",
@@ -26088,6 +26178,8 @@
       "io.micrometer:micrometer-observation:jar:sources",
       "io.micrometer:micrometer-registry-otlp",
       "io.micrometer:micrometer-registry-otlp:jar:sources",
+      "io.micrometer:micrometer-registry-prometheus",
+      "io.micrometer:micrometer-registry-prometheus:jar:sources",
       "io.micrometer:micrometer-registry-statsd",
       "io.micrometer:micrometer-registry-statsd:jar:sources",
       "io.netty:netty-all",
@@ -26240,6 +26332,16 @@
       "io.prometheus:prometheus-metrics-exposition-textformats:jar:sources",
       "io.prometheus:prometheus-metrics-model",
       "io.prometheus:prometheus-metrics-model:jar:sources",
+      "io.prometheus:simpleclient",
+      "io.prometheus:simpleclient:jar:sources",
+      "io.prometheus:simpleclient_common",
+      "io.prometheus:simpleclient_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_common",
+      "io.prometheus:simpleclient_tracer_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel",
+      "io.prometheus:simpleclient_tracer_otel:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel_agent",
+      "io.prometheus:simpleclient_tracer_otel_agent:jar:sources",
       "io.swagger.core.v3:swagger-annotations",
       "io.swagger.core.v3:swagger-annotations:jar:sources",
       "io.temporal:temporal-sdk",
@@ -27556,6 +27658,8 @@
       "io.micrometer:micrometer-observation:jar:sources",
       "io.micrometer:micrometer-registry-otlp",
       "io.micrometer:micrometer-registry-otlp:jar:sources",
+      "io.micrometer:micrometer-registry-prometheus",
+      "io.micrometer:micrometer-registry-prometheus:jar:sources",
       "io.micrometer:micrometer-registry-statsd",
       "io.micrometer:micrometer-registry-statsd:jar:sources",
       "io.netty:netty-all",
@@ -27708,6 +27812,16 @@
       "io.prometheus:prometheus-metrics-exposition-textformats:jar:sources",
       "io.prometheus:prometheus-metrics-model",
       "io.prometheus:prometheus-metrics-model:jar:sources",
+      "io.prometheus:simpleclient",
+      "io.prometheus:simpleclient:jar:sources",
+      "io.prometheus:simpleclient_common",
+      "io.prometheus:simpleclient_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_common",
+      "io.prometheus:simpleclient_tracer_common:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel",
+      "io.prometheus:simpleclient_tracer_otel:jar:sources",
+      "io.prometheus:simpleclient_tracer_otel_agent",
+      "io.prometheus:simpleclient_tracer_otel_agent:jar:sources",
       "io.swagger.core.v3:swagger-annotations",
       "io.swagger.core.v3:swagger-annotations:jar:sources",
       "io.temporal:temporal-sdk",

online/src/main/scala/ai/chronon/online/metrics/Metrics.scala

@@ -128,19 +128,15 @@ object Metrics {
     }
 
     private val client: MetricsReporter = {
-      // Can disable metrics collection for local / dev environments
-      val metricsEnabled: Boolean = System.getProperty(MetricsEnabled, "true").toBoolean
+      // Metrics collection is turned off by default, we explicitly turn on in serving contexts
+      val metricsEnabled: Boolean = System.getProperty(MetricsEnabled, "false").toBoolean
       val reporter: String = System.getProperty(MetricsReporter, "otel")
 
       reporter.toLowerCase match {
         case "otel" | "opentelemetry" =>
           if (metricsEnabled) {
-            val maybeMetricReader = OtelMetricsReporter.buildOtelMetricReader()
-            val openTelemetry = maybeMetricReader
-              .map { metricReader =>
-                OtelMetricsReporter.buildOpenTelemetryClient(metricReader)
-              }
-              .getOrElse(OpenTelemetry.noop())
+            val metricReader = OtelMetricsReporter.buildOtelMetricReader()
+            val openTelemetry = OtelMetricsReporter.buildOpenTelemetryClient(metricReader)
             new OtelMetricsReporter(openTelemetry)
           } else {
             new OtelMetricsReporter(OpenTelemetry.noop())

online/src/main/scala/ai/chronon/online/metrics/OtelMetricsReporter.scala

@@ -85,33 +85,35 @@ object OtelMetricsReporter {
   val MetricsExporterPrometheusPortKey = "ai.chronon.metrics.exporter.port"
   val MetricsExporterResourceKey = "ai.chronon.metrics.exporter.resources"
 
-  val MetricsReaderDefault = "http"
+  val MetricsReaderHttp = "http"
   val MetricsReaderPrometheus = "prometheus"
   val MetricsExporterInterval = "PT15s"
+  val MetricsExporterUrlDefault = "http://localhost:4318"
   val MetricsExporterPrometheusPortDefault = "8905"
 
-  def getExporterUrl: Option[String] = {
-    Option(System.getProperty(MetricsExporterUrlKey))
+  def getExporterUrl: String = {
+    System.getProperty(MetricsExporterUrlKey, MetricsExporterUrlDefault)
   }
 
-  def buildOtelMetricReader(): Option[MetricReader] = {
-    val metricReader = System.getProperty(MetricsReader, MetricsReaderDefault)
+  def getMetricsReader: String = {
+    System.getProperty(MetricsReader, MetricsReaderHttp)
+  }
+
+  def buildOtelMetricReader(): MetricReader = {
+    val metricReader = getMetricsReader
     metricReader.toLowerCase match {
-      case MetricsReaderDefault =>
-        getExporterUrl.map { url =>
-          val exporterUrl = url + "/v1/metrics"
-          val metricExporter = OtlpHttpMetricExporter.builder.setEndpoint(exporterUrl).build
-          // Configure periodic metric reader// Configure periodic metric reader
-          PeriodicMetricReader.builder(metricExporter).setInterval(Duration.parse(MetricsExporterInterval)).build
-        }
+      case MetricsReaderHttp =>
+        val exporterUrl = getExporterUrl + "/v1/metrics"
+        val metricExporter = OtlpHttpMetricExporter.builder.setEndpoint(exporterUrl).build
+        // Configure periodic metric reader// Configure periodic metric reader
+        PeriodicMetricReader.builder(metricExporter).setInterval(Duration.parse(MetricsExporterInterval)).build
 
       case MetricsReaderPrometheus =>
         val prometheusPort =
           System.getProperty(MetricsExporterPrometheusPortKey, MetricsExporterPrometheusPortDefault).toInt
-        Some(
-          PrometheusHttpServer.builder
-            .setPort(prometheusPort)
-            .build)
+        PrometheusHttpServer.builder
+          .setPort(prometheusPort)
+          .build
       case _ =>
         throw new IllegalArgumentException(s"Unknown metrics reader (only http / prometheus supported): $metricReader")
     }

service/src/main/java/ai/chronon/service/FetcherVerticle.java

@@ -7,6 +7,7 @@
 import ai.chronon.service.handlers.JoinSchemaHandler;
 import io.vertx.core.AbstractVerticle;
 import io.vertx.core.Promise;
+import io.vertx.core.http.Http2Settings;
 import io.vertx.core.http.HttpServer;
 import io.vertx.core.http.HttpServerOptions;
 import io.vertx.ext.web.Router;
@@ -68,11 +69,21 @@ protected void startHttpServer(int port, String configJsonString, JavaFetcher fe
                .end(configJsonString);
         });
 
+        Http2Settings http2Settings = new Http2Settings()
+                .setMaxConcurrentStreams(200)
+                .setInitialWindowSize(1024 * 1024);
+
         // Start HTTP server
         HttpServerOptions httpOptions =
                 new HttpServerOptions()
                         .setTcpKeepAlive(true)
-                        .setIdleTimeout(60);
+                        .setIdleTimeout(60)
+                        // HTTP/2 specific settings - these are currently the default in our Vert.x version
+                        // but we make them explicit for clarity and to guard against future changes
+                        .setUseAlpn(false)           // No need for ALPN in cleartext mode
+                        .setSsl(false)               // No SSL for cleartext
+                        .setHttp2ClearTextEnabled(true)
+                        .setInitialSettings(http2Settings);
         server = vertx.createHttpServer(httpOptions);
         server.requestHandler(router)
                 .listen(port)

service_commons/BUILD.bazel

@@ -10,6 +10,7 @@ java_library(
         maven_artifact("com.typesafe:config"),
         maven_artifact("io.netty:netty-all"),
         maven_artifact("io.micrometer:micrometer-registry-otlp"),
+        maven_artifact("io.micrometer:micrometer-registry-prometheus"),
         maven_artifact("io.micrometer:micrometer-core"),
         maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
         maven_artifact("org.slf4j:slf4j-api"),