diff --git a/implementations/micrometer-registry-prometheus_native/build.gradle b/implementations/micrometer-registry-prometheus_native/build.gradle new file mode 100644 index 0000000000..0d7ff27fc7 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/build.gradle @@ -0,0 +1,15 @@ +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + api project(':micrometer-core') + + api 'io.prometheus:prometheus-metrics-core:1.0.0-alpha-2' + api 'io.prometheus:prometheus-metrics-model:1.0.0-alpha-2' + api 'io.prometheus:prometheus-metrics-exposition-formats:1.0.0-alpha-2' + api 'io.prometheus:prometheus-metrics-exporter-servlet-jakarta:1.0.0-alpha-2' + + testImplementation project(':micrometer-test') +} diff --git a/implementations/micrometer-registry-prometheus_native/gradle.properties b/implementations/micrometer-registry-prometheus_native/gradle.properties new file mode 100644 index 0000000000..a02575112d --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/gradle.properties @@ -0,0 +1 @@ +compatibleVersion=SKIP diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/Max.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/Max.java new file mode 100644 index 0000000000..c36fbdfd05 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/Max.java @@ -0,0 +1,62 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.prometheus.metrics.core.metrics.SlidingWindow; + +/** + * Micrometer distributions have a {@code max} value, which is not provided out-of-the-box + * by Prometheus histograms or summaries. + *

+ * Max is used to track these {@code max} values. + */ +public class Max { + + private static class MaxObserver { + + private double current = Double.NaN; + + public void observe(double value) { + if (Double.isNaN(current) || current < value) { + current = value; + } + } + + public double get() { + return current; + } + + } + + private final SlidingWindow slidingWindow; + + public Max(DistributionStatisticConfig config) { + long maxAgeSeconds = config.getExpiry() != null ? config.getExpiry().toMillis() / 1000L : 60; + int ageBuckets = config.getBufferLength() != null ? config.getBufferLength() : 3; + slidingWindow = new SlidingWindow<>(MaxObserver.class, MaxObserver::new, MaxObserver::observe, maxAgeSeconds, + ageBuckets); + } + + public void observe(double value) { + slidingWindow.observe(value); + } + + public double get() { + return slidingWindow.current().get(); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusConfig.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusConfig.java new file mode 100644 index 0000000000..618346bb0d --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusConfig.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.config.MeterRegistryConfig; +import io.prometheus.metrics.config.PrometheusProperties; + +public interface PrometheusConfig extends MeterRegistryConfig { + + @Override + default String prefix() { + return "prometheus"; + } + + default PrometheusProperties getPrometheusProperties() { + // To be discussed: Make Prometheus properties configurable via Spring's + // application.properties + // under the `management.metrics.export.prometheus` prefix? + return PrometheusProperties.get(); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounter.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounter.java new file mode 100644 index 0000000000..344ccabeae --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounter.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.common.lang.Nullable; +import io.prometheus.metrics.core.datapoints.CounterDataPoint; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.Exemplar; + +public class PrometheusCounter extends PrometheusMeter + implements io.micrometer.core.instrument.Counter { + + private final CounterDataPoint dataPoint; + + PrometheusCounter(Id id, Counter counter, CounterDataPoint dataPoint) { + super(id, counter); + this.dataPoint = dataPoint; + } + + @Override + public void increment(double amount) { + dataPoint.inc(amount); + } + + @Override + public double count() { + return collect().getValue(); + } + + @Nullable + Exemplar exemplar() { + return collect().getExemplar(); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounterWithCallback.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounterWithCallback.java new file mode 100644 index 0000000000..b457fb29c4 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusCounterWithCallback.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.FunctionCounter; +import io.prometheus.metrics.core.metrics.CounterWithCallback; +import io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; + +public class PrometheusCounterWithCallback extends PrometheusMeter + implements FunctionCounter { + + public PrometheusCounterWithCallback(Id id, CounterWithCallback counter) { + super(id, counter); + } + + @Override + public double count() { + return collect().getValue(); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusFunctionTimer.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusFunctionTimer.java new file mode 100644 index 0000000000..1cf4cb5483 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusFunctionTimer.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.FunctionTimer; +import io.micrometer.core.instrument.Meter; +import io.prometheus.metrics.core.metrics.SummaryWithCallback; +import io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot; + +import java.util.concurrent.TimeUnit; + +public class PrometheusFunctionTimer extends PrometheusMeter + implements FunctionTimer { + + public PrometheusFunctionTimer(Meter.Id id, SummaryWithCallback summary) { + super(id, summary); + } + + @Override + public double count() { + return collect().getCount(); + } + + @Override + public double totalTime(TimeUnit unit) { + return toUnit(collect().getSum(), unit); + } + + @Override + public TimeUnit baseTimeUnit() { + return TimeUnit.SECONDS; + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusGaugeWithCallback.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusGaugeWithCallback.java new file mode 100644 index 0000000000..beda2e467b --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusGaugeWithCallback.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.Gauge; +import io.prometheus.metrics.core.metrics.GaugeWithCallback; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; + +public class PrometheusGaugeWithCallback extends PrometheusMeter + implements Gauge { + + public PrometheusGaugeWithCallback(Id id, GaugeWithCallback gauge) { + super(id, gauge); + } + + @Override + public double value() { + return collect().getValue(); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusHistogram.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusHistogram.java new file mode 100644 index 0000000000..44dd7444a6 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusHistogram.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.distribution.HistogramSnapshot; +import io.prometheus.metrics.core.datapoints.DistributionDataPoint; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot; + +public class PrometheusHistogram extends PrometheusMeter + implements DistributionSummary { + + private final Max max; + + private final DistributionDataPoint dataPoint; + + public PrometheusHistogram(Id id, Max max, Histogram histogram, DistributionDataPoint dataPoint) { + super(id, histogram); + this.max = max; + this.dataPoint = dataPoint; + } + + @Override + public void record(double amount) { + dataPoint.observe(amount); + max.observe(amount); + } + + @Override + public long count() { + return collect().getCount(); + } + + @Override + public double totalAmount() { + return collect().getSum(); + } + + @Override + public double max() { + return max.get(); + } + + @Override + public HistogramSnapshot takeSnapshot() { + return HistogramSnapshot.empty(count(), totalAmount(), max()); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusInfoGauge.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusInfoGauge.java new file mode 100644 index 0000000000..4a47f2a586 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusInfoGauge.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Meter; + +/** + * Prometheus Info metric. + *

+ * Micrometer implements Info metrics as Gauge. + */ +public class PrometheusInfoGauge implements Gauge { + + private final Meter.Id id; + + public PrometheusInfoGauge(Id id) { + this.id = id; + } + + @Override + public double value() { + return 1.0; + } + + @Override + public Id getId() { + return id; + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusLongTaskTimer.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusLongTaskTimer.java new file mode 100644 index 0000000000..cef4c199aa --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusLongTaskTimer.java @@ -0,0 +1,116 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.LongTaskTimer; +import io.micrometer.core.instrument.distribution.HistogramSnapshot; +import io.prometheus.metrics.core.datapoints.DistributionDataPoint; +import io.prometheus.metrics.model.registry.Collector; +import io.prometheus.metrics.model.snapshots.DistributionDataPointSnapshot; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.prometheus.metrics.model.snapshots.Unit.nanosToSeconds; + +/** + * Long task timer that can either be backed by a Prometheus histogram or by a Prometheus + * summary. + * + * @param {@link io.prometheus.metrics.core.metrics.Histogram Histogram} or + * {@link io.prometheus.metrics.core.metrics.Summary Summary}. + * @param + * {@link io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot + * HistogramDataPointSnapshot} if {@code T} is + * {@link io.prometheus.metrics.core.metrics.Histogram Histogram}, + * {@link io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot + * SummaryDataPointSnapshot} if {@code T} is + * {@link io.prometheus.metrics.core.metrics.Summary Summary}. + */ +public class PrometheusLongTaskTimer + extends PrometheusMeter implements LongTaskTimer { + + private final DistributionDataPoint dataPoint; + + private final Max max; + + private AtomicInteger activeTasksCount = new AtomicInteger(0); + + public PrometheusLongTaskTimer(Id id, Max max, T histogramOrSummary, DistributionDataPoint dataPoint) { + super(id, histogramOrSummary); + this.max = max; + this.dataPoint = dataPoint; + } + + class Sample extends LongTaskTimer.Sample { + + private final long startTimeNanos; + + private volatile double durationSeconds = 0; + + Sample() { + this.startTimeNanos = System.nanoTime(); + } + + @Override + public long stop() { + long durationNanos = System.nanoTime() - startTimeNanos; + durationSeconds = nanosToSeconds(durationNanos); + dataPoint.observe(durationSeconds); + max.observe(durationSeconds); + activeTasksCount.decrementAndGet(); + return durationNanos; + } + + @Override + public double duration(TimeUnit unit) { + return toUnit(durationSeconds, unit); + } + + } + + @Override + public Sample start() { + activeTasksCount.incrementAndGet(); + return new Sample(); + } + + @Override + public double duration(TimeUnit unit) { + return toUnit(collect().getSum(), unit); + } + + @Override + public int activeTasks() { + return activeTasksCount.get(); + } + + @Override + public double max(TimeUnit unit) { + return toUnit(max.get(), unit); + } + + @Override + public TimeUnit baseTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + public HistogramSnapshot takeSnapshot() { + return HistogramSnapshot.empty(collect().getCount(), duration(baseTimeUnit()), max(baseTimeUnit())); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeter.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeter.java new file mode 100644 index 0000000000..bffdc499cb --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeter.java @@ -0,0 +1,77 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Tag; +import io.prometheus.metrics.model.registry.Collector; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.PrometheusNaming; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +abstract class PrometheusMeter implements Meter { + + private final Meter.Id id; + + private final T collector; + + private final Labels labels; + + /** + * Collect a single data point. + *

+ * This first collects the entire metric, and then returns the data point where the + * labels match {@link Id#getTags()}. + */ + protected S collect() { + MetricSnapshot snapshot = collector.collect(); + for (DataPointSnapshot dataPoint : snapshot.getData()) { + if (labels.equals(dataPoint.getLabels())) { + return (S) dataPoint; + } + } + throw new IllegalStateException( + "No Prometheus labels found for Micrometer's tags. This is a bug in the Prometheus meter registry."); + } + + PrometheusMeter(Meter.Id id, T collector) { + this.id = id; + this.collector = collector; + this.labels = makeLabels(id.getTags()); + } + + private Labels makeLabels(List tags) { + Labels.Builder builder = Labels.newBuilder(); + for (Tag tag : tags) { + builder.addLabel(PrometheusNaming.sanitizeLabelName(tag.getKey()), tag.getValue()); + } + return builder.build(); + } + + @Override + public Id getId() { + return id; + } + + protected double toUnit(double seconds, TimeUnit unit) { + return seconds * TimeUnit.SECONDS.toNanos(1) / (double) unit.toNanos(1); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeterRegistry.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeterRegistry.java new file mode 100644 index 0000000000..4c39565a8c --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusMeterRegistry.java @@ -0,0 +1,446 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.*; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.micrometer.core.instrument.distribution.pause.PauseDetector; +import io.micrometer.core.instrument.internal.DefaultMeter; +import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.metrics.*; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import io.prometheus.metrics.model.snapshots.*; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.ToDoubleFunction; +import java.util.function.ToLongFunction; + +import static io.prometheus.metrics.model.snapshots.Unit.nanosToSeconds; + +/** + * Meter registry for the upcoming Prometheus Java client library version 1.0. + */ +public class PrometheusMeterRegistry extends MeterRegistry { + + private final PrometheusProperties prometheusProperties; + + private final PrometheusRegistry prometheusRegistry; + + // The following maps have Meter.Id.getName() as their key. + + private final Map counters = new ConcurrentHashMap<>(); + + private final Map countersWithCallback = new ConcurrentHashMap<>(); + + private final Map gauges = new ConcurrentHashMap<>(); + + private final Map histograms = new ConcurrentHashMap<>(); + + private final Map summaries = new ConcurrentHashMap<>(); + + private final Map summariesWithCallback = new ConcurrentHashMap<>(); + + private final Map infos = new ConcurrentHashMap<>(); + + private final Map>> counterCallbackMap = new ConcurrentHashMap<>(); + + private final Map>> gaugeCallbackMap = new ConcurrentHashMap<>(); + + private final Map>> summaryCallbackMap = new ConcurrentHashMap<>(); + + public PrometheusMeterRegistry(PrometheusConfig config, PrometheusRegistry prometheusRegistry, Clock clock) { + super(clock); + this.prometheusRegistry = prometheusRegistry; + this.prometheusProperties = config.getPrometheusProperties(); + } + + @Override + protected PrometheusCounter newCounter(Meter.Id id) { + Counter counter = getOrCreateCounter(id); + return new PrometheusCounter(id, counter, counter.withLabelValues(getLabelValues(id.getTags()))); + } + + @Override + protected Gauge newGauge(Meter.Id id, T obj, ToDoubleFunction valueFunction) { + if (id.getName().equals("jvm.info")) { + // Micrometer does not have info metrics, so info metrics are represented as + // Gauge. + // This is a hack to represent JvmInfoMetrics as an Info rather than a Gauge. + return newInfo(id); + } + else { + return newGaugeWithCallback(id, obj, valueFunction); + } + } + + private PrometheusGaugeWithCallback newGaugeWithCallback(Meter.Id id, T obj, + ToDoubleFunction valueFunction) { + GaugeWithCallback gauge = getOrCreateGaugeWithCallback(id, obj, valueFunction); + return new PrometheusGaugeWithCallback(id, gauge); + } + + private PrometheusInfoGauge newInfo(Meter.Id id) { + id = id.withBaseUnit(null); + Info info = getOrCreateInfo(id); + info.infoLabelValues(getLabelValues(id.getTags())); + return new PrometheusInfoGauge(id); + } + + @Override + protected DistributionSummary newDistributionSummary(Meter.Id id, + DistributionStatisticConfig distributionStatisticConfig, double scale) { + Max max = newMax(id, distributionStatisticConfig); + if (distributionStatisticConfig.isPublishingHistogram()) { + Histogram histogram = getOrCreateHistogram(id, distributionStatisticConfig); + return new PrometheusHistogram(id, max, histogram, histogram.withLabelValues(getLabelValues(id.getTags()))); + } + else { + Summary summary = getOrCreateSummary(id, distributionStatisticConfig); + return new PrometheusSummary(id, max, summary, summary.withLabelValues(getLabelValues(id.getTags()))); + } + } + + /** + * In Prometheus _max is not part of Histograms and Summaries. Therefore, newMax + * registers a Prometheus Gauge to track the max value. + */ + private Max newMax(Meter.Id id, DistributionStatisticConfig config) { + Max max = new Max(config); + getOrCreateGaugeWithCallback(id.withName(id.getName() + ".max"), max, Max::get); + return max; + } + + @Override + protected Timer newTimer(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig, + PauseDetector pauseDetector) { + Max max = newMax(id, distributionStatisticConfig); + if (distributionStatisticConfig.isPublishingHistogram()) { + Histogram histogram = getOrCreateHistogram(id, distributionStatisticConfig); + return new PrometheusTimer(id, max, histogram, + histogram.withLabelValues(getLabelValues(id.getTags()))); + } + else { + Summary summary = getOrCreateSummary(id, distributionStatisticConfig); + return new PrometheusTimer(id, max, summary, + summary.withLabelValues(getLabelValues(id.getTags()))); + } + } + + @Override + protected FunctionTimer newFunctionTimer(Meter.Id id, T obj, ToLongFunction countFunction, + ToDoubleFunction totalTimeFunction, TimeUnit totalTimeFunctionUnit) { + SummaryWithCallback summary = getOrCreateSummaryWithCallback(id, obj, countFunction, totalTimeFunction, + totalTimeFunctionUnit); + return new PrometheusFunctionTimer(id, summary); + } + + /** + * Example: This is used to create {@code http.server.requests.active}. + */ + @Override + protected LongTaskTimer newLongTaskTimer(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig) { + Max max = newMax(id, distributionStatisticConfig); + if (distributionStatisticConfig.isPublishingHistogram()) { + Histogram histogram = getOrCreateHistogram(id, distributionStatisticConfig); + return new PrometheusLongTaskTimer<>(id, max, histogram, + histogram.withLabelValues(getLabelValues(id.getTags()))); + } + else { + Summary summary = getOrCreateSummary(id, distributionStatisticConfig); + return new PrometheusLongTaskTimer<>(id, max, summary, + summary.withLabelValues(getLabelValues(id.getTags()))); + } + } + + @Override + protected FunctionCounter newFunctionCounter(Meter.Id id, T obj, ToDoubleFunction countFunction) { + CounterWithCallback counter = getOrCreateCounterWithCallback(id, obj, countFunction); + return new PrometheusCounterWithCallback(id, counter); + } + + @Override + protected TimeUnit getBaseTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + protected DistributionStatisticConfig defaultHistogramConfig() { + return DistributionStatisticConfig.builder().build().merge(DistributionStatisticConfig.DEFAULT); + } + + @Override + protected Meter newMeter(Meter.Id id, Meter.Type type, Iterable measurements) { + // TODO: This is not the correct implementation, the original + // PrometheusMeterRegistry does something different here. + switch (type) { + case COUNTER: + getOrCreateCounterWithCallback(id, measurements, m -> { + for (Measurement measurement : m) { + if (measurement.getStatistic() == Statistic.TOTAL + || measurement.getStatistic() == Statistic.TOTAL_TIME) { + return measurement.getValue(); + } + } + return Double.NaN; + }); + break; + case TIMER: + getOrCreateSummaryWithCallback(id, measurements, m -> { + // count + for (Measurement measurement : measurements) { + if (measurement.getStatistic() == Statistic.COUNT) { + return (long) measurement.getValue(); + } + } + return 0L; + }, m -> { + // sum + for (Measurement measurement : measurements) { + if (measurement.getStatistic() == Statistic.TOTAL + || measurement.getStatistic() == Statistic.TOTAL_TIME) { + return measurement.getValue(); + } + } + return Double.NaN; + }, TimeUnit.SECONDS); // seconds is our base time unit, so SECONDS means + // don't convert the sum. + break; + // TODO: GAUGE, LONG_TASK_TIMER, TIMER, OTHER + default: + throw new UnsupportedOperationException("Meter type " + type + " not implemented yet."); + } + return new DefaultMeter(id, type, measurements); + } + + private Counter getOrCreateCounter(Meter.Id id) { + return counters.computeIfAbsent(id.getName(), + name -> init(id, Counter.newBuilder(prometheusProperties)).register(prometheusRegistry)); + } + + private Info getOrCreateInfo(Meter.Id id) { + return infos.computeIfAbsent(id.getName(), + name -> init(id, Info.newBuilder(prometheusProperties)).register(prometheusRegistry)); + } + + private GaugeWithCallback getOrCreateGaugeWithCallback(Meter.Id id, T obj, ToDoubleFunction valueFunction) { + Map> callbacks = gaugeCallbackMap.computeIfAbsent(id.getName(), + name -> new ConcurrentHashMap<>()); + callbacks.put(id, new GaugeCallback<>(obj, valueFunction, getLabelValues(id.getTags()))); + return gauges.computeIfAbsent(id.getName(), + name -> init(id, GaugeWithCallback.newBuilder(prometheusProperties)).withCallback(cb -> { + for (GaugeCallback callback : gaugeCallbackMap.get(id.getName()).values()) { + callback.accept(cb); + } + }).register(prometheusRegistry)); + } + + private CounterWithCallback getOrCreateCounterWithCallback(Meter.Id id, T obj, + ToDoubleFunction countFunction) { + Map> callbacks = counterCallbackMap.computeIfAbsent(id.getName(), + name -> new ConcurrentHashMap<>()); + callbacks.put(id, new CounterCallback<>(obj, countFunction, getLabelValues(id.getTags()))); + return countersWithCallback.computeIfAbsent(id.getName(), + name -> init(id, CounterWithCallback.newBuilder(prometheusProperties)).withCallback(cb -> { + for (CounterCallback callback : counterCallbackMap.get(id.getName()).values()) { + callback.accept(cb); + } + }).register(prometheusRegistry)); + } + + private Histogram getOrCreateHistogram(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig) { + return histograms.computeIfAbsent(id.getName(), name -> { + Histogram.Builder builder = init(id, Histogram.newBuilder(prometheusProperties)); + double[] classicBuckets = distributionStatisticConfig.getServiceLevelObjectiveBoundaries(); + if (classicBuckets != null && classicBuckets.length == 0) { + builder = builder.classicOnly().withClassicBuckets(classicBuckets); + } + return builder.register(prometheusRegistry); + }); + } + + private Summary getOrCreateSummary(Meter.Id id, DistributionStatisticConfig distributionStatisticConfig) { + return summaries.computeIfAbsent(id.getName(), name -> { + Summary.Builder builder = init(id, Summary.newBuilder(prometheusProperties)); + if (distributionStatisticConfig.isPublishingPercentiles()) { + for (double quantile : distributionStatisticConfig.getPercentiles()) { + builder = builder.withQuantile(quantile); + } + } + if (distributionStatisticConfig.getExpiry() != null) { + builder = builder.withMaxAgeSeconds(distributionStatisticConfig.getExpiry().toMillis() / 1000L); + } + if (distributionStatisticConfig.getBufferLength() != null) { + builder = builder.withNumberOfAgeBuckets(distributionStatisticConfig.getBufferLength()); + } + return builder.register(prometheusRegistry); + }); + } + + private SummaryWithCallback getOrCreateSummaryWithCallback(Meter.Id id, T obj, ToLongFunction countFunction, + ToDoubleFunction sumFunction, TimeUnit timeUnit) { + Map> callbacks = summaryCallbackMap.computeIfAbsent(id.getName(), + name -> new ConcurrentHashMap<>()); + callbacks.put(id, + new SummaryCallback<>(obj, countFunction, sumFunction, timeUnit, getLabelValues(id.getTags()))); + return summariesWithCallback.computeIfAbsent(id.getName(), + name -> init(id, SummaryWithCallback.newBuilder(prometheusProperties)).withCallback(cb -> { + for (SummaryCallback callback : summaryCallbackMap.get(id.getName()).values()) { + callback.accept(cb); + } + }).register(prometheusRegistry)); + } + + @SuppressWarnings("unchecked") + private > T init(Meter.Id id, T builder) { + return (T) builder.withName(PrometheusNaming.sanitizeMetricName(id.getName())) + .withHelp(id.getDescription()) + .withUnit(id.getBaseUnit() != null ? new Unit(id.getBaseUnit()) : null) + .withLabelNames(getLabelNames(id.getTags())); + } + + private String[] getLabelNames(List tags) { + return tags.stream().map(Tag::getKey).map(PrometheusNaming::sanitizeLabelName).toArray(String[]::new); + } + + private String[] getLabelValues(List tags) { + return tags.stream().map(Tag::getValue).toArray(String[]::new); + } + + private static class GaugeCallback { + + /** + * 2nd parameter of + * {@link MeterRegistry#newGauge(Meter.Id, Object, ToDoubleFunction)}. + */ + private final T obj; + + /** + * 3rd parameter of + * {@link MeterRegistry#newGauge(Meter.Id, Object, ToDoubleFunction)}. + */ + private final ToDoubleFunction valueFunction; + + /** + * For Prometheus callbacks you also need the label values to call a callback. + */ + private final String[] labelValues; + + private GaugeCallback(T obj, ToDoubleFunction valueFunction, String[] labelValues) { + this.obj = obj; + this.valueFunction = valueFunction; + this.labelValues = labelValues; + } + + /** + * Call the callback + */ + public void accept(GaugeWithCallback.Callback callback) { + callback.call(valueFunction.applyAsDouble(obj), labelValues); + } + + } + + private static class CounterCallback { + + /** + * 2nd parameter of + * {@link MeterRegistry#newFunctionCounter(Meter.Id, Object, ToDoubleFunction)} + */ + private final T obj; + + /** + * 3rd parameter of + * {@link MeterRegistry#newFunctionCounter(Meter.Id, Object, ToDoubleFunction)} + */ + private final ToDoubleFunction countFunction; + + /** + * For Prometheus callbacks you also need the label values to call a callback. + */ + private final String[] labelValues; + + private CounterCallback(T obj, ToDoubleFunction countFunction, String[] labelValues) { + this.obj = obj; + this.countFunction = countFunction; + this.labelValues = labelValues; + } + + /** + * Call the callback. + */ + public void accept(CounterWithCallback.Callback callback) { + callback.call(countFunction.applyAsDouble(obj), labelValues); + } + + } + + private static class SummaryCallback { + + /** + * 2nd parameter of + * {@link MeterRegistry#newFunctionTimer(Meter.Id, Object, ToLongFunction, ToDoubleFunction, TimeUnit)} + */ + private final T obj; + + /** + * 3rd parameter of + * {@link MeterRegistry#newFunctionTimer(Meter.Id, Object, ToLongFunction, ToDoubleFunction, TimeUnit)} + */ + private final ToLongFunction countFunction; + + /** + * 4th parameter of + * {@link MeterRegistry#newFunctionTimer(Meter.Id, Object, ToLongFunction, ToDoubleFunction, TimeUnit)} + */ + private final ToDoubleFunction totalTimeFunction; + + /** + * Convert the timeUnit from the 5th parameter of + * {@link MeterRegistry#newFunctionTimer(Meter.Id, Object, ToLongFunction, ToDoubleFunction, TimeUnit)} + * to seconds. + */ + private final double toSecondsFactor; + + /** + * For Prometheus callbacks you also need the label values to call a callback. + */ + private final String[] labelValues; + + private SummaryCallback(T obj, ToLongFunction countFunction, ToDoubleFunction totalTimeFunction, + TimeUnit timeUnit, String[] labelValues) { + this.obj = obj; + this.countFunction = countFunction; + this.totalTimeFunction = totalTimeFunction; + this.labelValues = labelValues; + this.toSecondsFactor = nanosToSeconds(timeUnit.toNanos(1)); + } + + /** + * Call the callback. + */ + public void accept(SummaryWithCallback.Callback callback) { + callback.call(countFunction.applyAsLong(obj), totalTimeFunction.applyAsDouble(obj) * toSecondsFactor, + Quantiles.EMPTY, labelValues); + } + + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusSummary.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusSummary.java new file mode 100644 index 0000000000..8310ff08e5 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusSummary.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.distribution.HistogramSnapshot; +import io.prometheus.metrics.core.datapoints.DistributionDataPoint; +import io.prometheus.metrics.core.metrics.Summary; +import io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot; + +public class PrometheusSummary extends PrometheusMeter + implements DistributionSummary { + + private final DistributionDataPoint dataPoint; + + private final Max max; + + public PrometheusSummary(Id id, Max max, Summary summary, DistributionDataPoint dataPoint) { + super(id, summary); + this.dataPoint = dataPoint; + this.max = max; + } + + @Override + public void record(double amount) { + dataPoint.observe(amount); + max.observe(amount); + } + + @Override + public long count() { + return collect().getCount(); + } + + @Override + public double totalAmount() { + return collect().getSum(); + } + + @Override + public double max() { + return max.get(); + } + + @Override + public HistogramSnapshot takeSnapshot() { + return HistogramSnapshot.empty(count(), totalAmount(), max()); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusTimer.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusTimer.java new file mode 100644 index 0000000000..5c2778ea17 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/PrometheusTimer.java @@ -0,0 +1,122 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.distribution.HistogramSnapshot; +import io.prometheus.metrics.core.datapoints.DistributionDataPoint; +import io.prometheus.metrics.model.registry.Collector; +import io.prometheus.metrics.model.snapshots.DistributionDataPointSnapshot; + +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import static io.prometheus.metrics.model.snapshots.Unit.nanosToSeconds; +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * Backed by either a Histogram or a Summary. + * + * @param {@link io.prometheus.metrics.core.metrics.Histogram Histogram} or + * {@link io.prometheus.metrics.core.metrics.Summary Summary}. + * @param + * {@link io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot + * HistogramDataPointSnapshot} if {@code T} is + * {@link io.prometheus.metrics.core.metrics.Histogram Histogram}, + * {@link io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot + * SummaryDataPointSnapshot} if {@code T} is + * {@link io.prometheus.metrics.core.metrics.Summary Summary}. + */ +public class PrometheusTimer extends PrometheusMeter + implements Timer { + + private final DistributionDataPoint dataPoint; + + private final Max max; + + public PrometheusTimer(Id id, Max max, T histogramOrSummary, DistributionDataPoint dataPoint) { + super(id, histogramOrSummary); + this.max = max; + this.dataPoint = dataPoint; + } + + @Override + public void record(long amount, TimeUnit unit) { + double amountSeconds = nanosToSeconds(unit.toNanos(amount)); + dataPoint.observe(amountSeconds); + max.observe(amountSeconds); + } + + @Override + public O record(Supplier f) { + io.prometheus.metrics.core.datapoints.Timer timer = dataPoint.startTimer(); + try { + return f.get(); + } + finally { + max.observe(timer.observeDuration()); + } + } + + @Override + public O recordCallable(Callable f) throws Exception { + io.prometheus.metrics.core.datapoints.Timer timer = dataPoint.startTimer(); + try { + return f.call(); + } + finally { + max.observe(timer.observeDuration()); + } + } + + @Override + public void record(Runnable f) { + io.prometheus.metrics.core.datapoints.Timer timer = dataPoint.startTimer(); + try { + f.run(); + } + finally { + max.observe(timer.observeDuration()); + } + } + + @Override + public long count() { + return collect().getCount(); + } + + @Override + public double totalTime(TimeUnit unit) { + return toUnit(collect().getSum(), unit); + } + + @Override + public double max(TimeUnit unit) { + return toUnit(max.get(), unit); + } + + @Override + public TimeUnit baseTimeUnit() { + return SECONDS; + } + + @Override + public HistogramSnapshot takeSnapshot() { + return HistogramSnapshot.empty(count(), totalTime(baseTimeUnit()), max(baseTimeUnit())); + } + +} diff --git a/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/package-info.java b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/package-info.java new file mode 100644 index 0000000000..4d97c572d9 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/main/java/io/micrometer/prometheusnative/package-info.java @@ -0,0 +1,21 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +@NonNullFields +package io.micrometer.prometheusnative; + +import io.micrometer.common.lang.NonNullApi; +import io.micrometer.common.lang.NonNullFields; diff --git a/implementations/micrometer-registry-prometheus_native/src/test/java/io/micrometer/prometheusnative/PrometheusConfigTest.java b/implementations/micrometer-registry-prometheus_native/src/test/java/io/micrometer/prometheusnative/PrometheusConfigTest.java new file mode 100644 index 0000000000..cd45f55e73 --- /dev/null +++ b/implementations/micrometer-registry-prometheus_native/src/test/java/io/micrometer/prometheusnative/PrometheusConfigTest.java @@ -0,0 +1,37 @@ +/* + * Copyright 2023 VMware, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micrometer.prometheusnative; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +// TODO: This is just a placeholder. No real tests implemented yet. +class PrometheusConfigTest { + + private final Map props = new HashMap<>(); + + private final PrometheusConfig config = props::get; + + @Test + void testNotNull() { + assertThat(config.getPrometheusProperties()).isNotNull(); + } + +} diff --git a/settings.gradle b/settings.gradle index d0b51fb0d3..ce89e4a977 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,7 +32,7 @@ include 'micrometer-commons', 'micrometer-core', 'micrometer-observation' include 'micrometer-test', 'micrometer-observation-test' -['atlas', 'prometheus', 'datadog', 'elastic', 'ganglia', 'graphite', 'health', 'jmx', 'influx', 'otlp', 'statsd', 'new-relic', 'cloudwatch', 'cloudwatch2', 'signalfx', 'wavefront', 'dynatrace', 'azure-monitor', 'humio', 'appoptics', 'kairos', 'stackdriver', 'opentsdb'].each { sys -> +['atlas', 'prometheus', 'prometheus_native', 'datadog', 'elastic', 'ganglia', 'graphite', 'health', 'jmx', 'influx', 'otlp', 'statsd', 'new-relic', 'cloudwatch', 'cloudwatch2', 'signalfx', 'wavefront', 'dynatrace', 'azure-monitor', 'humio', 'appoptics', 'kairos', 'stackdriver', 'opentsdb'].each { sys -> include "micrometer-registry-$sys" project(":micrometer-registry-$sys").projectDir = new File(rootProject.projectDir, "implementations/micrometer-registry-$sys") }