From fd4624edafb70c15688d3551570a2e1d535ad5f7 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 9 Mar 2024 12:12:01 +0100 Subject: [PATCH 1/3] add laravel cache adapter --- composer.json | 5 +- .../Storage/LaravelCacheAdapter.php | 537 ++++++++++++++++++ .../LaravelCache/CollectorRegistryTest.php | 21 + .../Prometheus/LaravelCache/CounterTest.php | 24 + .../Prometheus/LaravelCache/GaugeTest.php | 24 + .../Prometheus/LaravelCache/HistogramTest.php | 24 + .../Prometheus/LaravelCache/SummaryTest.php | 24 + 7 files changed, 658 insertions(+), 1 deletion(-) create mode 100644 src/Prometheus/Storage/LaravelCacheAdapter.php create mode 100644 tests/Test/Prometheus/LaravelCache/CollectorRegistryTest.php create mode 100644 tests/Test/Prometheus/LaravelCache/CounterTest.php create mode 100644 tests/Test/Prometheus/LaravelCache/GaugeTest.php create mode 100644 tests/Test/Prometheus/LaravelCache/HistogramTest.php create mode 100644 tests/Test/Prometheus/LaravelCache/SummaryTest.php diff --git a/composer.json b/composer.json index 320ad059..bf9de2b0 100644 --- a/composer.json +++ b/composer.json @@ -20,6 +20,8 @@ }, "require-dev": { "guzzlehttp/guzzle": "^6.3|^7.0", + "illuminate/cache": "^9.0|^10.0|^11.0", + "illuminate/contracts": "^9.0|^10.0|^11.0", "phpstan/extension-installer": "^1.0", "phpstan/phpstan": "^1.5.4", "phpstan/phpstan-phpunit": "^1.1.0", @@ -32,7 +34,8 @@ "ext-redis": "Required if using Redis.", "ext-apc": "Required if using APCu.", "promphp/prometheus_push_gateway_php": "An easy client for using Prometheus PushGateway.", - "symfony/polyfill-apcu": "Required if you use APCu on PHP8.0+" + "symfony/polyfill-apcu": "Required if you use APCu on PHP8.0+", + "illuminate/contracts": "Required if using the Laravel Cache Adapter." }, "autoload": { "psr-4": { diff --git a/src/Prometheus/Storage/LaravelCacheAdapter.php b/src/Prometheus/Storage/LaravelCacheAdapter.php new file mode 100644 index 00000000..7f30cb99 --- /dev/null +++ b/src/Prometheus/Storage/LaravelCacheAdapter.php @@ -0,0 +1,537 @@ +cache = $cache; + } + + /** + * @return MetricFamilySamples[] + * @throws InvalidArgumentException + */ + public function collect(bool $sortMetrics = true): array + { + $metrics = $this->internalCollect( + $this->fetch(Counter::TYPE), + $sortMetrics + ); + $metrics = array_merge( + $metrics, + $this->internalCollect($this->fetch(Gauge::TYPE), $sortMetrics) + ); + $metrics = array_merge( + $metrics, + $this->collectHistograms($this->fetch(Histogram::TYPE)) + ); + return array_merge( + $metrics, + $this->collectSummaries($this->fetch(Summary::TYPE)) + ); + } + + /** + * @deprecated use replacement method wipeStorage from Adapter interface + */ + public function flushMemory(): void + { + $this->wipeStorage(); + } + + /** + * @inheritDoc + * @throws InvalidArgumentException + */ + public function wipeStorage(): void + { + $this->cache->deleteMultiple([ + $this->cacheKey(Counter::TYPE), + $this->cacheKey(Gauge::TYPE), + $this->cacheKey(Histogram::TYPE), + $this->cacheKey(Summary::TYPE), + ]); + } + + /** + * @param mixed[] $histograms + * + * @return MetricFamilySamples[] + */ + protected function collectHistograms(array $histograms): array + { + $output = []; + foreach ($histograms as $histogram) { + $metaData = $histogram['meta']; + $data = [ + 'name' => $metaData['name'], + 'help' => $metaData['help'], + 'type' => $metaData['type'], + 'labelNames' => $metaData['labelNames'], + 'buckets' => $metaData['buckets'], + ]; + + // Add the Inf bucket so we can compute it later on + $data['buckets'][] = '+Inf'; + + $histogramBuckets = []; + foreach ($histogram['samples'] as $key => $value) { + $parts = explode(':', $key); + $labelValues = $parts[2]; + $bucket = $parts[3]; + // Key by labelValues + $histogramBuckets[$labelValues][$bucket] = $value; + } + + // Compute all buckets + $labels = array_keys($histogramBuckets); + sort($labels); + foreach ($labels as $labelValues) { + $acc = 0; + $decodedLabelValues = $this->decodeLabelValues($labelValues); + foreach ($data['buckets'] as $bucket) { + $bucket = (string) $bucket; + if (!isset($histogramBuckets[$labelValues][$bucket])) { + $data['samples'][] = [ + 'name' => $metaData['name'] . '_bucket', + 'labelNames' => ['le'], + 'labelValues' => array_merge($decodedLabelValues, + [$bucket]), + 'value' => $acc, + ]; + } else { + $acc += $histogramBuckets[$labelValues][$bucket]; + $data['samples'][] = [ + 'name' => $metaData['name'].'_'.'bucket', + 'labelNames' => ['le'], + 'labelValues' => array_merge($decodedLabelValues, + [$bucket]), + 'value' => $acc, + ]; + } + } + + // Add the count + $data['samples'][] = [ + 'name' => $metaData['name'].'_count', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => $acc, + ]; + + // Add the sum + $data['samples'][] = [ + 'name' => $metaData['name'].'_sum', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => $histogramBuckets[$labelValues]['sum'], + ]; + } + $output[] = new MetricFamilySamples($data); + } + return $output; + } + + /** + * @return MetricFamilySamples[] + */ + protected function collectSummaries($summaries): array + { + $math = new Math(); + $output = []; + foreach ($summaries as $metaKey => &$summary) { + $metaData = $summary['meta']; + $data = [ + 'name' => $metaData['name'], + 'help' => $metaData['help'], + 'type' => $metaData['type'], + 'labelNames' => $metaData['labelNames'], + 'maxAgeSeconds' => $metaData['maxAgeSeconds'], + 'quantiles' => $metaData['quantiles'], + 'samples' => [], + ]; + + foreach ($summary['samples'] as $key => &$values) { + $parts = explode(':', $key); + $labelValues = $parts[2]; + $decodedLabelValues = $this->decodeLabelValues($labelValues); + + // Remove old data + $values = array_filter($values, + function (array $value) use ($data): bool { + return time() - $value['time'] + <= $data['maxAgeSeconds']; + }); + if (count($values) === 0) { + unset($summary['samples'][$key]); + continue; + } + + // Compute quantiles + usort($values, function (array $value1, array $value2) { + if ($value1['value'] === $value2['value']) { + return 0; + } + return ($value1['value'] < $value2['value']) ? -1 : 1; + }); + + foreach ($data['quantiles'] as $quantile) { + $data['samples'][] = [ + 'name' => $metaData['name'], + 'labelNames' => ['quantile'], + 'labelValues' => array_merge($decodedLabelValues, + [$quantile]), + 'value' => $math->quantile(array_column($values, + 'value'), $quantile), + ]; + } + + // Add the count + $data['samples'][] = [ + 'name' => $metaData['name'].'_count', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => count($values), + ]; + + // Add the sum + $data['samples'][] = [ + 'name' => $metaData['name'].'_sum', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => array_sum(array_column($values, 'value')), + ]; + } + if (count($data['samples']) > 0) { + $output[] = new MetricFamilySamples($data); + } + } + return $output; + } + + /** + * @param mixed[] $metrics + * + * @return MetricFamilySamples[] + */ + protected function internalCollect( + array $metrics, + bool $sortMetrics = true + ): array { + $result = []; + foreach ($metrics as $metric) { + $metaData = $metric['meta']; + $data = [ + 'name' => $metaData['name'], + 'help' => $metaData['help'], + 'type' => $metaData['type'], + 'labelNames' => $metaData['labelNames'], + 'samples' => [], + ]; + foreach ($metric['samples'] as $key => $value) { + $parts = explode(':', $key); + $labelValues = $parts[2]; + $data['samples'][] = [ + 'name' => $metaData['name'], + 'labelNames' => [], + 'labelValues' => $this->decodeLabelValues($labelValues), + 'value' => $value, + ]; + } + + if ($sortMetrics) { + $this->sortSamples($data['samples']); + } + + $result[] = new MetricFamilySamples($data); + } + return $result; + } + + /** + * @param mixed[] $data + * + * @return void + * @throws InvalidArgumentException + */ + public function updateHistogram(array $data): void + { + $histograms = $this->fetch(Histogram::TYPE); + + // Initialize the sum + $metaKey = $this->metaKey($data); + if (array_key_exists($metaKey, $histograms) === false) { + $histograms[$metaKey] = [ + 'meta' => $this->metaData($data), + 'samples' => [], + ]; + } + $sumKey = $this->histogramBucketValueKey($data, 'sum'); + if (array_key_exists($sumKey, $histograms[$metaKey]['samples']) === false) { + $histograms[$metaKey]['samples'][$sumKey] = 0; + } + + $histograms[$metaKey]['samples'][$sumKey] += $data['value']; + + + $bucketToIncrease = '+Inf'; + foreach ($data['buckets'] as $bucket) { + if ($data['value'] <= $bucket) { + $bucketToIncrease = $bucket; + break; + } + } + + $bucketKey = $this->histogramBucketValueKey($data, $bucketToIncrease); + if (array_key_exists($bucketKey, $histograms[$metaKey]['samples']) + === false + ) { + $histograms[$metaKey]['samples'][$bucketKey] = 0; + } + $histograms[$metaKey]['samples'][$bucketKey] += 1; + + $this->push(Histogram::TYPE, $histograms); + } + + /** + * @param mixed[] $data + * + * @return void + */ + public function updateSummary(array $data): void + { + $summaries = $this->fetch(Summary::TYPE); + + $metaKey = $this->metaKey($data); + if (array_key_exists($metaKey, $summaries) === false) { + $summaries[$metaKey] = [ + 'meta' => $this->metaData($data), + 'samples' => [], + ]; + } + + $valueKey = $this->valueKey($data); + if (array_key_exists($valueKey, $summaries[$metaKey]['samples']) + === false + ) { + $summaries[$metaKey]['samples'][$valueKey] = []; + } + + $summaries[$metaKey]['samples'][$valueKey][] = [ + 'time' => time(), + 'value' => $data['value'], + ]; + + $this->push(Summary::TYPE, $summaries); + } + + /** + * @param mixed[] $data + */ + public function updateGauge(array $data): void + { + $gauges = $this->fetch(Gauge::TYPE); + + $metaKey = $this->metaKey($data); + $valueKey = $this->valueKey($data); + if (array_key_exists($metaKey, $gauges) === false) { + $gauges[$metaKey] = [ + 'meta' => $this->metaData($data), + 'samples' => [], + ]; + } + if (array_key_exists($valueKey, $gauges[$metaKey]['samples']) + === false + ) { + $gauges[$metaKey]['samples'][$valueKey] = 0; + } + if ($data['command'] === Adapter::COMMAND_SET) { + $gauges[$metaKey]['samples'][$valueKey] = $data['value']; + } else { + $gauges[$metaKey]['samples'][$valueKey] += $data['value']; + } + + $this->push(Gauge::TYPE, $gauges); + } + + /** + * @param mixed[] $data + */ + public function updateCounter(array $data): void + { + $counters = $this->fetch(Counter::TYPE); + + $metaKey = $this->metaKey($data); + $valueKey = $this->valueKey($data); + if (array_key_exists($metaKey, $counters) === false) { + $counters[$metaKey] = [ + 'meta' => $this->metaData($data), + 'samples' => [], + ]; + } + if (array_key_exists($valueKey, $counters[$metaKey]['samples']) + === false + ) { + $counters[$metaKey]['samples'][$valueKey] = 0; + } + if ($data['command'] === Adapter::COMMAND_SET) { + $counters[$metaKey]['samples'][$valueKey] = 0; + } else { + $counters[$metaKey]['samples'][$valueKey] += $data['value']; + } + + $this->push(Counter::TYPE, $counters); + } + + /** + * @param mixed[] $data + * @param string|int $bucket + * + * @return string + */ + protected function histogramBucketValueKey(array $data, $bucket): string + { + return implode(':', [ + $data['type'], + $data['name'], + $this->encodeLabelValues($data['labelValues']), + $bucket, + ]); + } + + /** + * @param mixed[] $data + * + * @return string + */ + protected function metaKey(array $data): string + { + return implode(':', [ + $data['type'], + $data['name'], + 'meta' + ]); + } + + /** + * @param mixed[] $data + * + * @return string + */ + protected function valueKey(array $data): string + { + return implode(':', [ + $data['type'], + $data['name'], + $this->encodeLabelValues($data['labelValues']), + 'value' + ]); + } + + /** + * @param mixed[] $data + * + * @return mixed[] + */ + protected function metaData(array $data): array + { + $metricsMetaData = $data; + unset($metricsMetaData['value'], $metricsMetaData['command'], $metricsMetaData['labelValues']); + return $metricsMetaData; + } + + /** + * @param mixed[] $samples + */ + protected function sortSamples(array &$samples): void + { + usort($samples, function ($a, $b): int { + return strcmp(implode("", $a['labelValues']), + implode("", $b['labelValues'])); + }); + } + + /** + * @param mixed[] $values + * + * @return string + * @throws RuntimeException + */ + protected function encodeLabelValues(array $values): string + { + $json = json_encode($values); + if (false === $json) { + throw new RuntimeException(json_last_error_msg()); + } + return base64_encode($json); + } + + /** + * @param string $values + * + * @return mixed[] + * @throws RuntimeException + */ + protected function decodeLabelValues(string $values): array + { + $json = base64_decode($values, true); + if (false === $json) { + throw new RuntimeException('Cannot base64 decode label values'); + } + $decodedValues = json_decode($json, true); + if (false === $decodedValues) { + throw new RuntimeException(json_last_error_msg()); + } + return $decodedValues; + } + + /** + * @param string $type + * + * @return array> + * @throws InvalidArgumentException + */ + protected function fetch(string $type): array + { + return $this->cache->get($this->cacheKey($type), []); + } + + /** + * @param string $type + * @param array> $data + * + * @return void + */ + protected function push(string $type, array $data): void + { + $this->cache->put($this->cacheKey($type), $data); + } + + protected function cacheKey(string $type): string + { + return static::CACHE_KEY_PREFIX.$type.static::CACHE_KEY_SUFFIX; + } +} diff --git a/tests/Test/Prometheus/LaravelCache/CollectorRegistryTest.php b/tests/Test/Prometheus/LaravelCache/CollectorRegistryTest.php new file mode 100644 index 00000000..fd0ebcd0 --- /dev/null +++ b/tests/Test/Prometheus/LaravelCache/CollectorRegistryTest.php @@ -0,0 +1,21 @@ +adapter = new LaravelCacheAdapter(new Repository($arrayStore)); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/LaravelCache/CounterTest.php b/tests/Test/Prometheus/LaravelCache/CounterTest.php new file mode 100644 index 00000000..b3941599 --- /dev/null +++ b/tests/Test/Prometheus/LaravelCache/CounterTest.php @@ -0,0 +1,24 @@ +adapter = new LaravelCacheAdapter(new Repository($arrayStore)); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/LaravelCache/GaugeTest.php b/tests/Test/Prometheus/LaravelCache/GaugeTest.php new file mode 100644 index 00000000..b7d16fda --- /dev/null +++ b/tests/Test/Prometheus/LaravelCache/GaugeTest.php @@ -0,0 +1,24 @@ +adapter = new LaravelCacheAdapter(new Repository($arrayStore)); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/LaravelCache/HistogramTest.php b/tests/Test/Prometheus/LaravelCache/HistogramTest.php new file mode 100644 index 00000000..cc088bd1 --- /dev/null +++ b/tests/Test/Prometheus/LaravelCache/HistogramTest.php @@ -0,0 +1,24 @@ +adapter = new LaravelCacheAdapter(new Repository($arrayStore)); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/LaravelCache/SummaryTest.php b/tests/Test/Prometheus/LaravelCache/SummaryTest.php new file mode 100644 index 00000000..2c0421f5 --- /dev/null +++ b/tests/Test/Prometheus/LaravelCache/SummaryTest.php @@ -0,0 +1,24 @@ +adapter = new LaravelCacheAdapter(new Repository($arrayStore)); + $this->adapter->wipeStorage(); + } +} From a2cb96a5e7ae3f7d404662ec2f6deb4e4cbe2542 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 9 Mar 2024 12:21:21 +0100 Subject: [PATCH 2/3] fix type annotations --- .../Storage/LaravelCacheAdapter.php | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/src/Prometheus/Storage/LaravelCacheAdapter.php b/src/Prometheus/Storage/LaravelCacheAdapter.php index 7f30cb99..48977999 100644 --- a/src/Prometheus/Storage/LaravelCacheAdapter.php +++ b/src/Prometheus/Storage/LaravelCacheAdapter.php @@ -118,17 +118,21 @@ protected function collectHistograms(array $histograms): array $data['samples'][] = [ 'name' => $metaData['name'] . '_bucket', 'labelNames' => ['le'], - 'labelValues' => array_merge($decodedLabelValues, - [$bucket]), + 'labelValues' => array_merge( + $decodedLabelValues, + [$bucket] + ), 'value' => $acc, ]; } else { $acc += $histogramBuckets[$labelValues][$bucket]; $data['samples'][] = [ - 'name' => $metaData['name'].'_'.'bucket', + 'name' => $metaData['name'] . '_' . 'bucket', 'labelNames' => ['le'], - 'labelValues' => array_merge($decodedLabelValues, - [$bucket]), + 'labelValues' => array_merge( + $decodedLabelValues, + [$bucket] + ), 'value' => $acc, ]; } @@ -136,7 +140,7 @@ protected function collectHistograms(array $histograms): array // Add the count $data['samples'][] = [ - 'name' => $metaData['name'].'_count', + 'name' => $metaData['name'] . '_count', 'labelNames' => [], 'labelValues' => $decodedLabelValues, 'value' => $acc, @@ -144,7 +148,7 @@ protected function collectHistograms(array $histograms): array // Add the sum $data['samples'][] = [ - 'name' => $metaData['name'].'_sum', + 'name' => $metaData['name'] . '_sum', 'labelNames' => [], 'labelValues' => $decodedLabelValues, 'value' => $histogramBuckets[$labelValues]['sum'], @@ -156,9 +160,11 @@ protected function collectHistograms(array $histograms): array } /** + * @param mixed[] $summary + * * @return MetricFamilySamples[] */ - protected function collectSummaries($summaries): array + protected function collectSummaries(array $summaries): array { $math = new Math(); $output = []; @@ -174,17 +180,19 @@ protected function collectSummaries($summaries): array 'samples' => [], ]; - foreach ($summary['samples'] as $key => &$values) { + foreach ($summary['samples'] as $key => $values) { $parts = explode(':', $key); $labelValues = $parts[2]; $decodedLabelValues = $this->decodeLabelValues($labelValues); // Remove old data - $values = array_filter($values, + $values = array_filter( + $values, function (array $value) use ($data): bool { return time() - $value['time'] <= $data['maxAgeSeconds']; - }); + } + ); if (count($values) === 0) { unset($summary['samples'][$key]); continue; @@ -202,16 +210,20 @@ function (array $value) use ($data): bool { $data['samples'][] = [ 'name' => $metaData['name'], 'labelNames' => ['quantile'], - 'labelValues' => array_merge($decodedLabelValues, - [$quantile]), - 'value' => $math->quantile(array_column($values, - 'value'), $quantile), + 'labelValues' => array_merge( + $decodedLabelValues, + [$quantile] + ), + 'value' => $math->quantile(array_column( + $values, + 'value' + ), $quantile), ]; } // Add the count $data['samples'][] = [ - 'name' => $metaData['name'].'_count', + 'name' => $metaData['name'] . '_count', 'labelNames' => [], 'labelValues' => $decodedLabelValues, 'value' => count($values), @@ -219,7 +231,7 @@ function (array $value) use ($data): bool { // Add the sum $data['samples'][] = [ - 'name' => $metaData['name'].'_sum', + 'name' => $metaData['name'] . '_sum', 'labelNames' => [], 'labelValues' => $decodedLabelValues, 'value' => array_sum(array_column($values, 'value')), @@ -306,7 +318,8 @@ public function updateHistogram(array $data): void } $bucketKey = $this->histogramBucketValueKey($data, $bucketToIncrease); - if (array_key_exists($bucketKey, $histograms[$metaKey]['samples']) + if ( + array_key_exists($bucketKey, $histograms[$metaKey]['samples']) === false ) { $histograms[$metaKey]['samples'][$bucketKey] = 0; @@ -334,7 +347,8 @@ public function updateSummary(array $data): void } $valueKey = $this->valueKey($data); - if (array_key_exists($valueKey, $summaries[$metaKey]['samples']) + if ( + array_key_exists($valueKey, $summaries[$metaKey]['samples']) === false ) { $summaries[$metaKey]['samples'][$valueKey] = []; @@ -363,7 +377,8 @@ public function updateGauge(array $data): void 'samples' => [], ]; } - if (array_key_exists($valueKey, $gauges[$metaKey]['samples']) + if ( + array_key_exists($valueKey, $gauges[$metaKey]['samples']) === false ) { $gauges[$metaKey]['samples'][$valueKey] = 0; @@ -392,7 +407,8 @@ public function updateCounter(array $data): void 'samples' => [], ]; } - if (array_key_exists($valueKey, $counters[$metaKey]['samples']) + if ( + array_key_exists($valueKey, $counters[$metaKey]['samples']) === false ) { $counters[$metaKey]['samples'][$valueKey] = 0; @@ -469,8 +485,10 @@ protected function metaData(array $data): array protected function sortSamples(array &$samples): void { usort($samples, function ($a, $b): int { - return strcmp(implode("", $a['labelValues']), - implode("", $b['labelValues'])); + return strcmp( + implode("", $a['labelValues']), + implode("", $b['labelValues']) + ); }); } @@ -511,7 +529,7 @@ protected function decodeLabelValues(string $values): array /** * @param string $type * - * @return array> + * @return mixed[] * @throws InvalidArgumentException */ protected function fetch(string $type): array @@ -521,7 +539,7 @@ protected function fetch(string $type): array /** * @param string $type - * @param array> $data + * @param mixed[] $data * * @return void */ @@ -532,6 +550,6 @@ protected function push(string $type, array $data): void protected function cacheKey(string $type): string { - return static::CACHE_KEY_PREFIX.$type.static::CACHE_KEY_SUFFIX; + return static::CACHE_KEY_PREFIX . $type . static::CACHE_KEY_SUFFIX; } } From 3ecf44ffe11d699464435b1f08c42ad4b4d29c3e Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 9 Mar 2024 12:29:22 +0100 Subject: [PATCH 3/3] fix typo --- src/Prometheus/Storage/LaravelCacheAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Prometheus/Storage/LaravelCacheAdapter.php b/src/Prometheus/Storage/LaravelCacheAdapter.php index 48977999..2d666bcb 100644 --- a/src/Prometheus/Storage/LaravelCacheAdapter.php +++ b/src/Prometheus/Storage/LaravelCacheAdapter.php @@ -160,7 +160,7 @@ protected function collectHistograms(array $histograms): array } /** - * @param mixed[] $summary + * @param mixed[] $summaries * * @return MetricFamilySamples[] */