From 96c52a6dfd7d72858565ff5ebdb9888302b99374 Mon Sep 17 00:00:00 2001 From: Gabriel Levcovitz Date: Wed, 19 Mar 2025 18:26:11 -0300 Subject: [PATCH] feat: add tracked object counts metric review changes --- hathor/conf/settings.py | 5 ++++- hathor/prometheus.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hathor/conf/settings.py b/hathor/conf/settings.py index a2cc56af9..fadcb3f52 100644 --- a/hathor/conf/settings.py +++ b/hathor/conf/settings.py @@ -409,7 +409,10 @@ def GENESIS_TX2_TIMESTAMP(self) -> int: WS_SEND_METRICS_INTERVAL: int = 1 # Interval (in seconds) to write data to prometheus - PROMETHEUS_WRITE_INTERVAL: int = 5 + PROMETHEUS_WRITE_INTERVAL: int = 15 + + # Interval (in seconds) to update GC data for prometheus + PROMETHEUS_UPDATE_GC_INTERVAL: int = 60 # Interval (in seconds) to collect metrics data METRICS_COLLECT_DATA_INTERVAL: int = 5 diff --git a/hathor/prometheus.py b/hathor/prometheus.py index 5418c44d4..699869976 100644 --- a/hathor/prometheus.py +++ b/hathor/prometheus.py @@ -63,7 +63,8 @@ } GC_METRICS = { - 'objects': 'Number of objects tracked by the garbage collector', + 'tracked_objects': 'Number of objects tracked by the garbage collector', + 'collection_counts': 'Counts used by GC to control the moment each generation should be collected', 'collections': 'Number of collections done by the garbage collector', 'collected': 'Number of objects collected by the garbage collector', 'uncollectable': 'Number of objects that could not be collected by the garbage collector', @@ -109,10 +110,17 @@ def __init__(self, metrics: 'Metrics', path: str, filename: str = 'hathor.prom', # Interval in which the write data method will be called (in seconds) self.call_interval: int = self._settings.PROMETHEUS_WRITE_INTERVAL + # Interval in which the update GC data method will be called (in seconds) + self.gc_update_interval: int = self._settings.PROMETHEUS_UPDATE_GC_INTERVAL + # A timer to periodically write data to prometheus self._lc_write_data = LoopingCall(self._write_data) self._lc_write_data.clock = get_global_reactor() + # A timer to periodically update GC data + self._lc_update_gc_data = LoopingCall(self._set_garbage_collection_metrics) + self._lc_update_gc_data.clock = get_global_reactor() + def _initial_setup(self) -> None: """ Start a collector registry to send data to node exporter and create one object to hold each metric data @@ -178,6 +186,7 @@ def start(self) -> None: """ self.running = True self._lc_write_data.start(self.call_interval, now=False) + self._lc_update_gc_data.start(self.gc_update_interval, now=False) def set_new_metrics(self) -> None: """ Update metric_gauges dict with new data from metrics @@ -187,7 +196,6 @@ def set_new_metrics(self) -> None: self._set_rocksdb_tx_storage_metrics() self._set_new_peer_connection_metrics() - self._set_garbage_collection_metrics() write_to_textfile(self.filepath, self.registry) @@ -212,7 +220,8 @@ def _set_garbage_collection_metrics(self) -> None: threshold = gc.get_threshold() for i in range(3): - self.gc_metrics['objects'].labels(generation=i).set(counts[i]) + self.gc_metrics['tracked_objects'].labels(generation=i).set(len(gc.get_objects(i))) + self.gc_metrics['collection_counts'].labels(generation=i).set(counts[i]) self.gc_metrics['collections'].labels(generation=i).set(stats[i]['collections']) self.gc_metrics['collected'].labels(generation=i).set(stats[i]['collected']) self.gc_metrics['uncollectable'].labels(generation=i).set(stats[i]['uncollectable'])