From 768a06b225db9f41a01fdd31c5248892b7b27234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20K=C3=A4mmerling?= Date: Sun, 27 Nov 2022 22:36:43 +0100 Subject: [PATCH 1/4] Experiment on Redis-NG Storage without KEYS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Kämmerling --- docker-compose.yml | 53 +- php-fpm/Dockerfile | 6 +- src/Prometheus/Storage/RedisNg.php | 699 ++++++++++++++++++ .../RedisNg/CollectorRegistryTest.php | 20 + tests/Test/Prometheus/RedisNg/CounterTest.php | 21 + .../RedisNg/CounterWithPrefixTest.php | 27 + tests/Test/Prometheus/RedisNg/GaugeTest.php | 22 + .../RedisNg/GaugeWithPrefixTest.php | 27 + .../Test/Prometheus/RedisNg/HistogramTest.php | 22 + .../RedisNg/HistogramWithPrefixTest.php | 27 + tests/Test/Prometheus/RedisNg/SummaryTest.php | 22 + .../RedisNg/SummaryWithPrefixTest.php | 27 + tests/Test/Prometheus/Storage/RedisNgTest.php | 149 ++++ 13 files changed, 1093 insertions(+), 29 deletions(-) create mode 100644 src/Prometheus/Storage/RedisNg.php create mode 100644 tests/Test/Prometheus/RedisNg/CollectorRegistryTest.php create mode 100644 tests/Test/Prometheus/RedisNg/CounterTest.php create mode 100644 tests/Test/Prometheus/RedisNg/CounterWithPrefixTest.php create mode 100644 tests/Test/Prometheus/RedisNg/GaugeTest.php create mode 100644 tests/Test/Prometheus/RedisNg/GaugeWithPrefixTest.php create mode 100644 tests/Test/Prometheus/RedisNg/HistogramTest.php create mode 100644 tests/Test/Prometheus/RedisNg/HistogramWithPrefixTest.php create mode 100644 tests/Test/Prometheus/RedisNg/SummaryTest.php create mode 100644 tests/Test/Prometheus/RedisNg/SummaryWithPrefixTest.php create mode 100644 tests/Test/Prometheus/Storage/RedisNgTest.php diff --git a/docker-compose.yml b/docker-compose.yml index 0245264a..c6a794b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,31 @@ -nginx: - build: nginx/ - links: - - php-fpm - ports: - - 8080:80 +services: + nginx: + build: nginx/ + links: + - php-fpm + ports: + - 8080:80 -php-fpm: - build: php-fpm/ - volumes: - - .:/var/www/html - links: - - redis - environment: - - REDIS_HOST=redis + php-fpm: + build: php-fpm/ + volumes: + - .:/var/www/html + links: + - redis + environment: + - REDIS_HOST=redis -redis: - image: redis - ports: + redis: + image: redis + ports: - 6379:6379 -phpunit: - build: php-fpm/ - volumes: - - .:/var/www/html - links: - - redis - - nginx - environment: - - REDIS_HOST=redis + phpunit: + build: php-fpm/ + volumes: + - .:/var/www/html + links: + - redis + - nginx + environment: + - REDIS_HOST=redis diff --git a/php-fpm/Dockerfile b/php-fpm/Dockerfile index e5328d07..f0d50190 100644 --- a/php-fpm/Dockerfile +++ b/php-fpm/Dockerfile @@ -1,7 +1,7 @@ -FROM php:7.4-fpm +FROM php:8.1-fpm -RUN pecl install redis-5.3.1 && docker-php-ext-enable redis -RUN pecl install apcu-5.1.19 && docker-php-ext-enable apcu +RUN pecl install redis && docker-php-ext-enable redis +RUN pecl install apcu && docker-php-ext-enable apcu COPY www.conf /usr/local/etc/php-fpm.d/ COPY docker-php-ext-apcu-cli.ini /usr/local/etc/php/conf.d/ diff --git a/src/Prometheus/Storage/RedisNg.php b/src/Prometheus/Storage/RedisNg.php new file mode 100644 index 00000000..42cc49f0 --- /dev/null +++ b/src/Prometheus/Storage/RedisNg.php @@ -0,0 +1,699 @@ + '127.0.0.1', + 'port' => 6379, + 'timeout' => 0.1, + 'read_timeout' => '10', + 'persistent_connections' => false, + 'password' => null, + ]; + + /** + * @var string + */ + private static $prefix = 'PROMETHEUS_'; + + /** + * @var mixed[] + */ + private $options = []; + + /** + * @var \Redis + */ + private $redis; + + /** + * @var boolean + */ + private $connectionInitialized = false; + + /** + * Redis constructor. + * @param mixed[] $options + */ + public function __construct(array $options = []) + { + $this->options = array_merge(self::$defaultOptions, $options); + $this->redis = new \Redis(); + } + + /** + * @param \Redis $redis + * @return self + * @throws StorageException + */ + public static function fromExistingConnection(\Redis $redis): self + { + if ($redis->isConnected() === false) { + throw new StorageException('Connection to Redis server not established'); + } + + $self = new self(); + $self->connectionInitialized = true; + $self->redis = $redis; + + return $self; + } + + /** + * @param mixed[] $options + */ + public static function setDefaultOptions(array $options): void + { + self::$defaultOptions = array_merge(self::$defaultOptions, $options); + } + + /** + * @param string $prefix + */ + public static function setPrefix(string $prefix): void + { + self::$prefix = $prefix; + } + + /** + * @throws StorageException + * @deprecated use replacement method wipeStorage from Adapter interface + */ + public function flushRedis(): void + { + $this->wipeStorage(); + } + + /** + * @inheritDoc + */ + public function wipeStorage(): void + { + $this->ensureOpenConnection(); + + $searchPattern = ""; + + $globalPrefix = $this->redis->getOption(\Redis::OPT_PREFIX); + // @phpstan-ignore-next-line false positive, phpstan thinks getOptions returns int + if (is_string($globalPrefix)) { + $searchPattern .= $globalPrefix; + } + + $searchPattern .= self::$prefix; + $searchPattern .= '*'; + + $this->redis->eval( + <<encodeLabelValues($data['labelValues']), + 'value' + ]); + } + + /** + * @return MetricFamilySamples[] + * @throws StorageException + */ + public function collect(): array + { + $this->ensureOpenConnection(); + $metrics = $this->collectHistograms(); + $metrics = array_merge($metrics, $this->collectGauges()); + $metrics = array_merge($metrics, $this->collectCounters()); + $metrics = array_merge($metrics, $this->collectSummaries()); + return array_map( + function (array $metric): MetricFamilySamples { + return new MetricFamilySamples($metric); + }, + $metrics + ); + } + + /** + * @throws StorageException + */ + private function ensureOpenConnection(): void + { + if ($this->connectionInitialized === true) { + return; + } + + $this->connectToServer(); + + if ($this->options['password'] !== null) { + $this->redis->auth($this->options['password']); + } + + if (isset($this->options['database'])) { + $this->redis->select($this->options['database']); + } + + $this->redis->setOption(\Redis::OPT_READ_TIMEOUT, $this->options['read_timeout']); + + $this->connectionInitialized = true; + } + + /** + * @throws StorageException + */ + private function connectToServer(): void + { + try { + $connection_successful = false; + if ($this->options['persistent_connections'] !== false) { + $connection_successful = $this->redis->pconnect( + $this->options['host'], + (int)$this->options['port'], + (float)$this->options['timeout'] + ); + } else { + $connection_successful = $this->redis->connect($this->options['host'], (int)$this->options['port'], (float)$this->options['timeout']); + } + if (!$connection_successful) { + throw new StorageException("Can't connect to Redis server", 0); + } + } catch (\RedisException $e) { + throw new StorageException("Can't connect to Redis server", 0, $e); + } + } + + /** + * @param mixed[] $data + * @throws StorageException + */ + public function updateHistogram(array $data): void + { + $this->ensureOpenConnection(); + $bucketToIncrease = '+Inf'; + foreach ($data['buckets'] as $bucket) { + if ($data['value'] <= $bucket) { + $bucketToIncrease = $bucket; + break; + } + } + $metaData = $data; + unset($metaData['value'], $metaData['labelValues']); + + $this->redis->eval( + <<= tonumber(ARGV[3]) then + redis.call('hSet', KEYS[1], '__meta', ARGV[4]) + redis.call('sAdd', KEYS[2], KEYS[1]) +end +return result +LUA + , + [ + $this->toMetricKey($data), + self::$prefix . Histogram::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX, + json_encode(['b' => 'sum', 'labelValues' => $data['labelValues']]), + json_encode(['b' => $bucketToIncrease, 'labelValues' => $data['labelValues']]), + $data['value'], + json_encode($metaData), + ], + 2 + ); + } + + /** + * @param mixed[] $data + * @throws StorageException + */ + public function updateSummary(array $data): void + { + $this->ensureOpenConnection(); +// store meta + $summaryKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX; + $summaryKeyIndexKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX . ":keys"; + if (!$this->redis->sIsMember($summaryKeyIndexKey, $summaryKey . ':' . $data["name"])) { + $this->redis->sAdd($summaryKeyIndexKey, $summaryKey . ':' . $data["name"]); + } + + $metaKey = $summaryKey . ':' . $this->metaKey($data); + $json = json_encode($this->metaData($data)); + if (false === $json) { + throw new RuntimeException(json_last_error_msg()); + } + $this->redis->setNx($metaKey, $json); + /** @phpstan-ignore-line */ + + // store value key + $valueKey = $summaryKey . ':' . $this->valueKey($data); + $json = json_encode($this->encodeLabelValues($data['labelValues'])); + if (false === $json) { + throw new RuntimeException(json_last_error_msg()); + } + $this->redis->setNx($valueKey, $json); + /** @phpstan-ignore-line */ + + // trick to handle uniqid collision + $done = false; + while (!$done) { + $sampleKey = $valueKey . ':' . uniqid('', true); + $done = $this->redis->set($sampleKey, $data['value'], ['NX', 'EX' => $data['maxAgeSeconds']]); + } + } + + /** + * @param mixed[] $data + * @throws StorageException + */ + public function updateGauge(array $data): void + { + $this->ensureOpenConnection(); + $metaData = $data; + unset($metaData['value'], $metaData['labelValues'], $metaData['command']); + $this->redis->eval( + <<toMetricKey($data), + self::$prefix . Gauge::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX, + $this->getRedisCommand($data['command']), + json_encode($data['labelValues']), + $data['value'], + json_encode($metaData), + ], + 2 + ); + } + + /** + * @param mixed[] $data + * @throws StorageException + */ + public function updateCounter(array $data): void + { + $this->ensureOpenConnection(); + $metaData = $data; + unset($metaData['value'], $metaData['labelValues'], $metaData['command']); + $this->redis->eval( + <<toMetricKey($data), + self::$prefix . Counter::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX, + $this->getRedisCommand($data['command']), + $data['value'], + json_encode($data['labelValues']), + json_encode($metaData), + ], + 2 + ); + } + + + /** + * @param mixed[] $data + * @return mixed[] + */ + private function metaData(array $data): array + { + $metricsMetaData = $data; + unset($metricsMetaData['value'], $metricsMetaData['command'], $metricsMetaData['labelValues']); + return $metricsMetaData; + } + + /** + * @return mixed[] + */ + private function collectHistograms(): array + { + $keys = $this->redis->sMembers(self::$prefix . Histogram::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX); + sort($keys); + $histograms = []; + foreach ($keys as $key) { + $raw = $this->redis->hGetAll(str_replace($this->redis->_prefix(''), '', $key)); + $histogram = json_decode($raw['__meta'], true); + unset($raw['__meta']); + $histogram['samples'] = []; + + // Add the Inf bucket so we can compute it later on + $histogram['buckets'][] = '+Inf'; + + $allLabelValues = []; + foreach (array_keys($raw) as $k) { + $d = json_decode($k, true); + if ($d['b'] == 'sum') { + continue; + } + $allLabelValues[] = $d['labelValues']; + } + + // We need set semantics. + // This is the equivalent of array_unique but for arrays of arrays. + $allLabelValues = array_map("unserialize", array_unique(array_map("serialize", $allLabelValues))); + sort($allLabelValues); + + foreach ($allLabelValues as $labelValues) { + // Fill up all buckets. + // If the bucket doesn't exist fill in values from + // the previous one. + $acc = 0; + foreach ($histogram['buckets'] as $bucket) { + $bucketKey = json_encode(['b' => $bucket, 'labelValues' => $labelValues]); + if (!isset($raw[$bucketKey])) { + $histogram['samples'][] = [ + 'name' => $histogram['name'] . '_bucket', + 'labelNames' => ['le'], + 'labelValues' => array_merge($labelValues, [$bucket]), + 'value' => $acc, + ]; + } else { + $acc += $raw[$bucketKey]; + $histogram['samples'][] = [ + 'name' => $histogram['name'] . '_bucket', + 'labelNames' => ['le'], + 'labelValues' => array_merge($labelValues, [$bucket]), + 'value' => $acc, + ]; + } + } + + // Add the count + $histogram['samples'][] = [ + 'name' => $histogram['name'] . '_count', + 'labelNames' => [], + 'labelValues' => $labelValues, + 'value' => $acc, + ]; + + // Add the sum + $histogram['samples'][] = [ + 'name' => $histogram['name'] . '_sum', + 'labelNames' => [], + 'labelValues' => $labelValues, + 'value' => $raw[json_encode(['b' => 'sum', 'labelValues' => $labelValues])], + ]; + } + $histograms[] = $histogram; + } + return $histograms; + } + + /** + * @param string $key + * + * @return string + */ + private function removePrefixFromKey(string $key): string + { + // @phpstan-ignore-next-line false positive, phpstan thinks getOptions returns int + if ($this->redis->getOption(\Redis::OPT_PREFIX) === null) { + return $key; + } + // @phpstan-ignore-next-line false positive, phpstan thinks getOptions returns int + return substr($key, strlen($this->redis->getOption(\Redis::OPT_PREFIX))); + } + + /** + * @return mixed[] + */ + private function collectSummaries(): array + { + $math = new Math(); + $summaryKeyIndexKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX . ":keys"; + $summaryKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX; + + $keys = $this->redis->sMembers($summaryKeyIndexKey); + //$keys = $this->redis->keys($summaryKey . ':*:meta'); + $summaries = []; + foreach ($keys as $key) { + $metaKey = $this->removePrefixFromKey($key); + $rawSummary = $this->redis->get($metaKey . ':meta'); + var_dump($metaKey . ':meta', $rawSummary); + if ($rawSummary === false) { + continue; + } + $summary = json_decode($rawSummary, true); + $metaData = $summary; + $data = [ + 'name' => $metaData['name'], + 'help' => $metaData['help'], + 'type' => $metaData['type'], + 'labelNames' => $metaData['labelNames'], + 'maxAgeSeconds' => $metaData['maxAgeSeconds'], + 'quantiles' => $metaData['quantiles'], + 'samples' => [], + ]; + var_dump($metaKey . ':value'); + $dd = $this->redis->get($metaKey . ':value'); + var_dump($dd); + $values = $this->redis->keys($metaKey . ':value'); + var_dump($values); + foreach ($values as $valueKeyWithPrefix) { + $valueKey = $this->removePrefixFromKey($valueKeyWithPrefix); + $rawValue = $this->redis->get($valueKey); + var_dump($rawValue); + if ($rawValue === false) { + continue; + } + $value = json_decode($rawValue, true); + $encodedLabelValues = $value; + $decodedLabelValues = $this->decodeLabelValues($encodedLabelValues); + + $samples = []; + var_dump($metaKey . ':' . $encodedLabelValues . ':value:*'); + $sampleValues = $this->redis->keys($metaKey . ':' . $encodedLabelValues . ':value:*'); + foreach ($sampleValues as $sampleValueWithPrefix) { + $sampleValue = $this->removePrefixFromKey($sampleValueWithPrefix); + $samples[] = (float)$this->redis->get($sampleValue); + } + + if (count($samples) === 0) { + $this->redis->del($valueKey); + continue; + } + + // Compute quantiles + sort($samples); + foreach ($data['quantiles'] as $quantile) { + $data['samples'][] = [ + 'name' => $metaData['name'], + 'labelNames' => ['quantile'], + 'labelValues' => array_merge($decodedLabelValues, [$quantile]), + 'value' => $math->quantile($samples, $quantile), + ]; + } + + // Add the count + $data['samples'][] = [ + 'name' => $metaData['name'] . '_count', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => count($samples), + ]; + + // Add the sum + $data['samples'][] = [ + 'name' => $metaData['name'] . '_sum', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => array_sum($samples), + ]; + } + + if (count($data['samples']) > 0) { + $summaries[] = $data; + } else { + $this->redis->del($metaKey); + } + } + return $summaries; + } + + /** + * @return mixed[] + */ + private function collectGauges(): array + { + $keys = $this->redis->sMembers(self::$prefix . Gauge::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX); + sort($keys); + $gauges = []; + foreach ($keys as $key) { + $raw = $this->redis->hGetAll(str_replace($this->redis->_prefix(''), '', $key)); + $gauge = json_decode($raw['__meta'], true); + unset($raw['__meta']); + $gauge['samples'] = []; + foreach ($raw as $k => $value) { + $gauge['samples'][] = [ + 'name' => $gauge['name'], + 'labelNames' => [], + 'labelValues' => json_decode($k, true), + 'value' => $value, + ]; + } + usort($gauge['samples'], function ($a, $b): int { + return strcmp(implode("", $a['labelValues']), implode("", $b['labelValues'])); + }); + $gauges[] = $gauge; + } + return $gauges; + } + + /** + * @return mixed[] + */ + private function collectCounters(): array + { + $keys = $this->redis->sMembers(self::$prefix . Counter::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX); + sort($keys); + $counters = []; + foreach ($keys as $key) { + $raw = $this->redis->hGetAll(str_replace($this->redis->_prefix(''), '', $key)); + $counter = json_decode($raw['__meta'], true); + unset($raw['__meta']); + $counter['samples'] = []; + foreach ($raw as $k => $value) { + $counter['samples'][] = [ + 'name' => $counter['name'], + 'labelNames' => [], + 'labelValues' => json_decode($k, true), + 'value' => $value, + ]; + } + usort($counter['samples'], function ($a, $b): int { + return strcmp(implode("", $a['labelValues']), implode("", $b['labelValues'])); + }); + $counters[] = $counter; + } + return $counters; + } + + /** + * @param int $cmd + * @return string + */ + private function getRedisCommand(int $cmd): string + { + switch ($cmd) { + case Adapter::COMMAND_INCREMENT_INTEGER: + return 'hIncrBy'; + case Adapter::COMMAND_INCREMENT_FLOAT: + return 'hIncrByFloat'; + case Adapter::COMMAND_SET: + return 'hSet'; + default: + throw new InvalidArgumentException("Unknown command"); + } + } + + /** + * @param mixed[] $data + * @return string + */ + private function toMetricKey(array $data): string + { + return implode(':', [self::$prefix, $data['type'], $data['name']]); + } + + /** + * @param mixed[] $values + * @return string + * @throws RuntimeException + */ + private function encodeLabelValues(array $values): string + { + $json = json_encode($values); + if (false === $json) { + throw new RuntimeException(json_last_error_msg()); + } + return base64_encode($json); + } + + /** + * @param string $values + * @return mixed[] + * @throws RuntimeException + */ + private function decodeLabelValues(string $values): array + { + $json = base64_decode($values, true); + if (false === $json) { + throw new RuntimeException('Cannot base64 decode label values'); + } + $decodedValues = json_decode($json, true); + if (false === $decodedValues) { + throw new RuntimeException(json_last_error_msg()); + } + return $decodedValues; + } +} diff --git a/tests/Test/Prometheus/RedisNg/CollectorRegistryTest.php b/tests/Test/Prometheus/RedisNg/CollectorRegistryTest.php new file mode 100644 index 00000000..f164731d --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/CollectorRegistryTest.php @@ -0,0 +1,20 @@ +adapter = new Redis(['host' => REDIS_HOST]); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/CounterTest.php b/tests/Test/Prometheus/RedisNg/CounterTest.php new file mode 100644 index 00000000..b497a8fd --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/CounterTest.php @@ -0,0 +1,21 @@ +adapter = new RedisNg(['host' => REDIS_HOST]); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/CounterWithPrefixTest.php b/tests/Test/Prometheus/RedisNg/CounterWithPrefixTest.php new file mode 100644 index 00000000..a28c1222 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/CounterWithPrefixTest.php @@ -0,0 +1,27 @@ +connect(REDIS_HOST); + + $connection->setOption(\Redis::OPT_PREFIX, 'prefix:'); + + $this->adapter = RedisNg::fromExistingConnection($connection); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/GaugeTest.php b/tests/Test/Prometheus/RedisNg/GaugeTest.php new file mode 100644 index 00000000..cc1f5ca5 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/GaugeTest.php @@ -0,0 +1,22 @@ +adapter = new RedisNg(['host' => REDIS_HOST]); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/GaugeWithPrefixTest.php b/tests/Test/Prometheus/RedisNg/GaugeWithPrefixTest.php new file mode 100644 index 00000000..0b9cf5a0 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/GaugeWithPrefixTest.php @@ -0,0 +1,27 @@ +connect(REDIS_HOST); + + $connection->setOption(\Redis::OPT_PREFIX, 'prefix:'); + + $this->adapter = RedisNg::fromExistingConnection($connection); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/HistogramTest.php b/tests/Test/Prometheus/RedisNg/HistogramTest.php new file mode 100644 index 00000000..1ea1e83a --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/HistogramTest.php @@ -0,0 +1,22 @@ +adapter = new RedisNg(['host' => REDIS_HOST]); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/HistogramWithPrefixTest.php b/tests/Test/Prometheus/RedisNg/HistogramWithPrefixTest.php new file mode 100644 index 00000000..04cae294 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/HistogramWithPrefixTest.php @@ -0,0 +1,27 @@ +connect(REDIS_HOST); + + $connection->setOption(\Redis::OPT_PREFIX, 'prefix:'); + + $this->adapter = RedisNg::fromExistingConnection($connection); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/SummaryTest.php b/tests/Test/Prometheus/RedisNg/SummaryTest.php new file mode 100644 index 00000000..ec5912b4 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/SummaryTest.php @@ -0,0 +1,22 @@ +adapter = new RedisNg(['host' => REDIS_HOST]); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/RedisNg/SummaryWithPrefixTest.php b/tests/Test/Prometheus/RedisNg/SummaryWithPrefixTest.php new file mode 100644 index 00000000..d96e45e5 --- /dev/null +++ b/tests/Test/Prometheus/RedisNg/SummaryWithPrefixTest.php @@ -0,0 +1,27 @@ +connect(REDIS_HOST); + + $connection->setOption(\Redis::OPT_PREFIX, 'prefix:'); + + $this->adapter = RedisNg::fromExistingConnection($connection); + $this->adapter->wipeStorage(); + } +} diff --git a/tests/Test/Prometheus/Storage/RedisNgTest.php b/tests/Test/Prometheus/Storage/RedisNgTest.php new file mode 100644 index 00000000..adbf81eb --- /dev/null +++ b/tests/Test/Prometheus/Storage/RedisNgTest.php @@ -0,0 +1,149 @@ +redisConnection = new \Redis(); + $this->redisConnection->connect(REDIS_HOST); + $this->redisConnection->flushAll(); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnConnectionFailure(): void + { + $redis = new Redis(['host' => '/dev/null']); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to Redis server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldThrowExceptionWhenInjectedRedisIsNotConnected(): void + { + $connection = new \Redis(); + + self::expectException(StorageException::class); + self::expectExceptionMessage('Connection to Redis server not established'); + + Redis::fromExistingConnection($connection); + } + + /** + * @test + */ + public function itShouldNotClearWholeRedisOnFlush(): void + { + $this->redisConnection->set('not a prometheus metric key', 'data'); + + $redis = Redis::fromExistingConnection($this->redisConnection); + $registry = new CollectorRegistry($redis); + + // ensure flush is working correctly on large number of metrics + for ($i = 0; $i < 1000; $i++) { + $registry->getOrRegisterCounter('namespace', "counter_$i", 'counter help')->inc(); + $registry->getOrRegisterGauge('namespace', "gauge_$i", 'gauge help')->inc(); + $registry->getOrRegisterHistogram('namespace', "histogram_$i", 'histogram help')->observe(1); + } + $redis->wipeStorage(); + + $redisKeys = $this->redisConnection->keys("*"); + self::assertThat( + $redisKeys, + self::equalTo([ + 'not a prometheus metric key' + ]) + ); + } + + /** + * @test + */ + public function itShouldOnlyConnectOnceOnSubsequentCalls(): void + { + $clientId = $this->redisConnection->rawCommand('client', 'id'); + $expectedClientId = 'id=' . ($clientId + 1) . ' '; + $notExpectedClientId = 'id=' . ($clientId + 2) . ' '; + + $redis = new Redis(['host' => REDIS_HOST]); + + $redis->collect(); + + self::assertStringContainsString( + $expectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + self::assertStringNotContainsString( + $notExpectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + + $redis->collect(); + + self::assertStringContainsString( + $expectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + self::assertStringNotContainsString( + $notExpectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + } + + /** + * @test + */ + public function itShouldOnlyConnectOnceForInjectedRedisConnectionOnSubsequentCalls(): void + { + $clientId = $this->redisConnection->rawCommand('client', 'id'); + $expectedClientId = 'id=' . $clientId . ' '; + $notExpectedClientId = 'id=' . ($clientId + 1) . ' '; + + $redis = Redis::fromExistingConnection($this->redisConnection); + + $redis->collect(); + + self::assertStringContainsString( + $expectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + self::assertStringNotContainsString( + $notExpectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + + $redis->collect(); + + self::assertStringContainsString( + $expectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + self::assertStringNotContainsString( + $notExpectedClientId, + $this->redisConnection->rawCommand('client', 'list') + ); + } +} From 9f7fcd4186075269a21879f8fb00b4b79bc85c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20K=C3=A4mmerling?= Date: Mon, 28 Nov 2022 18:25:58 +0100 Subject: [PATCH 2/4] Fix tests for RedisNG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Kämmerling --- src/Prometheus/Storage/RedisNg.php | 85 ++++++++++----------- tests/Test/Prometheus/Redis/SummaryTest.php | 5 ++ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/Prometheus/Storage/RedisNg.php b/src/Prometheus/Storage/RedisNg.php index 42cc49f0..ba3b1a14 100644 --- a/src/Prometheus/Storage/RedisNg.php +++ b/src/Prometheus/Storage/RedisNg.php @@ -296,6 +296,7 @@ public function updateSummary(array $data): void // store value key $valueKey = $summaryKey . ':' . $this->valueKey($data); + $json = json_encode($this->encodeLabelValues($data['labelValues'])); if (false === $json) { throw new RuntimeException(json_last_error_msg()); @@ -308,7 +309,9 @@ public function updateSummary(array $data): void while (!$done) { $sampleKey = $valueKey . ':' . uniqid('', true); $done = $this->redis->set($sampleKey, $data['value'], ['NX', 'EX' => $data['maxAgeSeconds']]); + $this->redis->sAdd($summaryKey . ':' . $data["name"] . ":value:keys", $sampleKey); } + } /** @@ -491,7 +494,6 @@ private function collectSummaries(): array { $math = new Math(); $summaryKeyIndexKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX . ":keys"; - $summaryKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX; $keys = $this->redis->sMembers($summaryKeyIndexKey); //$keys = $this->redis->keys($summaryKey . ':*:meta'); @@ -499,7 +501,6 @@ private function collectSummaries(): array foreach ($keys as $key) { $metaKey = $this->removePrefixFromKey($key); $rawSummary = $this->redis->get($metaKey . ':meta'); - var_dump($metaKey . ':meta', $rawSummary); if ($rawSummary === false) { continue; } @@ -514,63 +515,57 @@ private function collectSummaries(): array 'quantiles' => $metaData['quantiles'], 'samples' => [], ]; - var_dump($metaKey . ':value'); - $dd = $this->redis->get($metaKey . ':value'); - var_dump($dd); - $values = $this->redis->keys($metaKey . ':value'); - var_dump($values); + $values = $this->redis->sMembers($metaKey . ':value:keys'); + $samples = []; foreach ($values as $valueKeyWithPrefix) { $valueKey = $this->removePrefixFromKey($valueKeyWithPrefix); - $rawValue = $this->redis->get($valueKey); - var_dump($rawValue); + $rawValue = explode(":", $valueKey); if ($rawValue === false) { continue; } - $value = json_decode($rawValue, true); - $encodedLabelValues = $value; + $encodedLabelValues = $rawValue[2]; $decodedLabelValues = $this->decodeLabelValues($encodedLabelValues); - $samples = []; - var_dump($metaKey . ':' . $encodedLabelValues . ':value:*'); - $sampleValues = $this->redis->keys($metaKey . ':' . $encodedLabelValues . ':value:*'); - foreach ($sampleValues as $sampleValueWithPrefix) { - $sampleValue = $this->removePrefixFromKey($sampleValueWithPrefix); - $samples[] = (float)$this->redis->get($sampleValue); - } - if (count($samples) === 0) { - $this->redis->del($valueKey); - continue; + $sampleValue = $this->removePrefixFromKey($valueKeyWithPrefix); + $return = $this->redis->get($sampleValue); + if ($return !== false) { + $samples[] = (float)$return; } + } + if (count($samples) === 0) { + $this->redis->del($valueKey); + continue; + } - // Compute quantiles - sort($samples); - foreach ($data['quantiles'] as $quantile) { - $data['samples'][] = [ - 'name' => $metaData['name'], - 'labelNames' => ['quantile'], - 'labelValues' => array_merge($decodedLabelValues, [$quantile]), - 'value' => $math->quantile($samples, $quantile), - ]; - } - - // Add the count - $data['samples'][] = [ - 'name' => $metaData['name'] . '_count', - 'labelNames' => [], - 'labelValues' => $decodedLabelValues, - 'value' => count($samples), - ]; - - // Add the sum + // Compute quantiles + sort($samples); + foreach ($data['quantiles'] as $quantile) { $data['samples'][] = [ - 'name' => $metaData['name'] . '_sum', - 'labelNames' => [], - 'labelValues' => $decodedLabelValues, - 'value' => array_sum($samples), + 'name' => $metaData['name'], + 'labelNames' => ['quantile'], + 'labelValues' => array_merge($decodedLabelValues, [$quantile]), + 'value' => $math->quantile($samples, $quantile), ]; } + // Add the count + $data['samples'][] = [ + 'name' => $metaData['name'] . '_count', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => count($samples), + ]; + + // Add the sum + $data['samples'][] = [ + 'name' => $metaData['name'] . '_sum', + 'labelNames' => [], + 'labelValues' => $decodedLabelValues, + 'value' => array_sum($samples), + ]; + + if (count($data['samples']) > 0) { $summaries[] = $data; } else { diff --git a/tests/Test/Prometheus/Redis/SummaryTest.php b/tests/Test/Prometheus/Redis/SummaryTest.php index 1446f65a..d6b0c596 100644 --- a/tests/Test/Prometheus/Redis/SummaryTest.php +++ b/tests/Test/Prometheus/Redis/SummaryTest.php @@ -18,4 +18,9 @@ public function configureAdapter(): void $this->adapter = new Redis(['host' => REDIS_HOST]); $this->adapter->wipeStorage(); } + /** @test */ + public function itShouldObserveWithLabels(): void + { + parent::itShouldObserveWithLabels(); // TODO: Change the autogenerated stub + } } From fba180ba1c3b55e352ce5df59e09e6227821d967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20K=C3=A4mmerling?= Date: Mon, 28 Nov 2022 18:28:35 +0100 Subject: [PATCH 3/4] Fix tests for RedisNG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Kämmerling --- src/Prometheus/Storage/RedisNg.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Prometheus/Storage/RedisNg.php b/src/Prometheus/Storage/RedisNg.php index ba3b1a14..28d28cf6 100644 --- a/src/Prometheus/Storage/RedisNg.php +++ b/src/Prometheus/Storage/RedisNg.php @@ -311,7 +311,6 @@ public function updateSummary(array $data): void $done = $this->redis->set($sampleKey, $data['value'], ['NX', 'EX' => $data['maxAgeSeconds']]); $this->redis->sAdd($summaryKey . ':' . $data["name"] . ":value:keys", $sampleKey); } - } /** From 0ecf10842c1d76f4f92009aeab6b9d3cd25e5f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20K=C3=A4mmerling?= Date: Mon, 28 Nov 2022 18:51:44 +0100 Subject: [PATCH 4/4] Fix RedisNG with Prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Kämmerling --- src/Prometheus/Storage/RedisNg.php | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Prometheus/Storage/RedisNg.php b/src/Prometheus/Storage/RedisNg.php index 28d28cf6..990d0cf8 100644 --- a/src/Prometheus/Storage/RedisNg.php +++ b/src/Prometheus/Storage/RedisNg.php @@ -495,10 +495,8 @@ private function collectSummaries(): array $summaryKeyIndexKey = self::$prefix . Summary::TYPE . self::PROMETHEUS_METRIC_KEYS_SUFFIX . ":keys"; $keys = $this->redis->sMembers($summaryKeyIndexKey); - //$keys = $this->redis->keys($summaryKey . ':*:meta'); $summaries = []; - foreach ($keys as $key) { - $metaKey = $this->removePrefixFromKey($key); + foreach ($keys as $metaKey) { $rawSummary = $this->redis->get($metaKey . ':meta'); if ($rawSummary === false) { continue; @@ -516,8 +514,7 @@ private function collectSummaries(): array ]; $values = $this->redis->sMembers($metaKey . ':value:keys'); $samples = []; - foreach ($values as $valueKeyWithPrefix) { - $valueKey = $this->removePrefixFromKey($valueKeyWithPrefix); + foreach ($values as $valueKey) { $rawValue = explode(":", $valueKey); if ($rawValue === false) { continue; @@ -525,9 +522,7 @@ private function collectSummaries(): array $encodedLabelValues = $rawValue[2]; $decodedLabelValues = $this->decodeLabelValues($encodedLabelValues); - - $sampleValue = $this->removePrefixFromKey($valueKeyWithPrefix); - $return = $this->redis->get($sampleValue); + $return = $this->redis->get($valueKey); if ($return !== false) { $samples[] = (float)$return; }