diff --git a/yarn-project/end-to-end/scripts/run_test.sh b/yarn-project/end-to-end/scripts/run_test.sh index c428921f9a69..043b26f70c61 100755 --- a/yarn-project/end-to-end/scripts/run_test.sh +++ b/yarn-project/end-to-end/scripts/run_test.sh @@ -38,6 +38,7 @@ case "$type" in -e BENCH_OUTPUT \ -e CAPTURE_IVC_FOLDER \ -e LOG_LEVEL \ + -e COLLECT_METRICS \ --workdir "$repo_dir/yarn-project/end-to-end" \ aztecprotocol/build:3.0 ./scripts/test_simple.sh $TEST ;; diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.test.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.test.ts index 126e9c0cf4a1..4ee932ad66f5 100644 --- a/yarn-project/telemetry-client/src/prom_otel_adapter.test.ts +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.test.ts @@ -1,9 +1,10 @@ import type { Logger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { jest } from '@jest/globals'; -import { OtelGauge } from './prom_otel_adapter.js'; -import type { Meter, ObservableGauge } from './telemetry.js'; +import { OtelAvgMinMax, OtelGauge, OtelHistogram } from './prom_otel_adapter.js'; +import type { Histogram, Meter, ObservableGauge } from './telemetry.js'; describe('OtelGauge', () => { let mockLogger: Logger; @@ -90,7 +91,7 @@ describe('OtelGauge', () => { ['region', 'instance'], ); - expect(() => otelGaugeWithLabels.inc(invalidLabelConfig as any, 1)).toThrowError('Invalid label key: invalid'); + expect(() => otelGaugeWithLabels.inc(invalidLabelConfig as any, 1)).toThrow('Invalid label key: invalid'); expect(mockLogger.error).not.toHaveBeenCalled(); }); @@ -115,3 +116,176 @@ describe('OtelGauge', () => { expect(mockResult.observe).toHaveBeenCalledWith(5, labelConfig); }); }); + +describe('OtelHistogram', () => { + let mockLogger: Logger; + let mockMeter: Meter; + let mockHistogram: Histogram; + let otelHistogram: OtelHistogram>; + + beforeEach(() => { + mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn(), silent: jest.fn() } as unknown as Logger; + mockHistogram = { record: jest.fn() } as unknown as Histogram; + mockMeter = { createHistogram: jest.fn(() => mockHistogram) } as unknown as Meter; + + otelHistogram = new OtelHistogram(mockLogger, mockMeter, 'test_histogram', 'Test histogram'); + + jest.spyOn(Timer.prototype, 's').mockImplementation(function (this: Timer) { + return 100; + }); + }); + + test('observes value without labels', () => { + otelHistogram.observe(5); + expect(mockHistogram.record).toHaveBeenCalledWith(5); + }); + + test('observes value with labels', () => { + const labelConfig = { topic: 'tx' }; + const otelHistogramWithLabels = new OtelHistogram( + mockLogger, + mockMeter, + 'test_histogram_with_labels', + 'Test histogram with labels', + [], + ['topic'], + ); + + otelHistogramWithLabels.observe(labelConfig, 10); + expect(mockHistogram.record).toHaveBeenCalledWith(10, labelConfig); + }); + + test('throws error for invalid labels', () => { + const invalidLabelConfig = { invalid: 'tx' }; + const otelHistogramWithLabels = new OtelHistogram( + mockLogger, + mockMeter, + 'test_histogram_with_labels', + 'Test histogram with labels', + [], + ['topic'], + ); + + expect(() => otelHistogramWithLabels.observe(invalidLabelConfig as any, 5)).toThrow('Invalid label key: invalid'); + }); + + test('starts timer and records duration without labels', () => { + const timer = otelHistogram.startTimer(); + timer(); + + expect(mockHistogram.record).toHaveBeenCalledWith(100); + }); + + test('starts timer and records duration with labels', () => { + const labelConfig1 = { topic: 'tx' }; + const labelConfig2 = { topic: 'block_proposal' }; + const otelHistogramWithLabels = new OtelHistogram( + mockLogger, + mockMeter, + 'test_histogram_with_labels', + 'Test histogram with labels', + [], + ['topic'], + ); + + const timer1 = otelHistogramWithLabels.startTimer(labelConfig1); + timer1(); + + const timer2 = otelHistogramWithLabels.startTimer(labelConfig2); + timer2(); + + expect(mockHistogram.record).toHaveBeenCalledWith(100, labelConfig1); + expect(mockHistogram.record).toHaveBeenCalledWith(100, labelConfig2); + }); +}); + +describe('OtelAvgMinMax', () => { + let mockLogger: Logger; + let mockMeter: Meter; + let mockAvgGauge: ObservableGauge; + let mockMinGauge: ObservableGauge; + let mockMaxGauge: ObservableGauge; + let otelAvgMinMax: OtelAvgMinMax>; + + beforeEach(() => { + mockLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() } as unknown as Logger; + mockAvgGauge = { addCallback: jest.fn() } as unknown as ObservableGauge; + mockMinGauge = { addCallback: jest.fn() } as unknown as ObservableGauge; + mockMaxGauge = { addCallback: jest.fn() } as unknown as ObservableGauge; + mockMeter = { + createObservableGauge: jest + .fn() + .mockReturnValueOnce(mockAvgGauge) + .mockReturnValueOnce(mockMinGauge) + .mockReturnValueOnce(mockMaxGauge), + } as unknown as Meter; + }); + + test('creates three gauges with correct names and descriptions', () => { + otelAvgMinMax = new OtelAvgMinMax(mockLogger, mockMeter, 'test_avgminmax', 'Test AvgMinMax'); + expect(mockMeter.createObservableGauge).toHaveBeenCalledWith('test_avgminmax_avg', { + description: 'Test AvgMinMax (average)', + }); + expect(mockMeter.createObservableGauge).toHaveBeenCalledWith('test_avgminmax_min', { + description: 'Test AvgMinMax (minimum)', + }); + expect(mockMeter.createObservableGauge).toHaveBeenCalledWith('test_avgminmax_max', { + description: 'Test AvgMinMax (maximum)', + }); + }); + + test('handles empty values array', () => { + otelAvgMinMax = new OtelAvgMinMax(mockLogger, mockMeter, 'test_avgminmax', 'Test AvgMinMax'); + otelAvgMinMax.set([]); + expect(otelAvgMinMax['currentValues']).toEqual([]); + + const mockResult = { observe: jest.fn() }; + otelAvgMinMax['observeAvg'](mockResult); + expect(mockResult.observe).not.toHaveBeenCalled(); + }); + + test('calculates aggregations for unlabeled values', () => { + otelAvgMinMax = new OtelAvgMinMax(mockLogger, mockMeter, 'test_avgminmax', 'Test AvgMinMax'); + const values = [6, 1, 2]; + otelAvgMinMax.set(values); + + const mockResult = { observe: jest.fn() }; + + otelAvgMinMax['observeAvg'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(3); + + mockResult.observe.mockClear(); + otelAvgMinMax['observeMin'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(1); + + mockResult.observe.mockClear(); + otelAvgMinMax['observeMax'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(6); + }); + + test('calculates aggregations for labeled values', () => { + const otelAvgMinMaxWithLabels = new OtelAvgMinMax( + mockLogger, + mockMeter, + 'test_avgminmax_with_labels', + 'Test AvgMinMax with labels', + ['topic'], + ); + const labelConfig = { topic: 'tx' }; + + otelAvgMinMaxWithLabels.set(labelConfig, [6, 1, 2]); + + const mockResult = { observe: jest.fn() }; + + otelAvgMinMaxWithLabels['observeAvg'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(3, labelConfig); + + mockResult.observe.mockClear(); + otelAvgMinMaxWithLabels['observeMin'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(1, labelConfig); + + mockResult.observe.mockClear(); + otelAvgMinMaxWithLabels['observeMax'](mockResult); + expect(mockResult.observe).toHaveBeenCalledWith(6, labelConfig); + }); +}); diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts index 4261de3be577..0964e51e234b 100644 --- a/yarn-project/telemetry-client/src/prom_otel_adapter.ts +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -1,8 +1,9 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { Registry } from 'prom-client'; -import type { Meter, MetricsType, ObservableGauge, TelemetryClient } from './telemetry.js'; +import type { Histogram, Meter, MetricsType, ObservableGauge, TelemetryClient } from './telemetry.js'; /** * Types matching the gossipsub and libp2p services @@ -125,7 +126,7 @@ export class OtelGauge implements IGaug } for (const [labelStr, value] of this.labeledValues.entries()) { - const labels = this.parseLabelsSafely(labelStr); + const labels = parseLabelsSafely(labelStr, this.logger); if (labels) { result.observe(value, labels); } @@ -146,7 +147,7 @@ export class OtelGauge implements IGaug } if (labelsOrValue) { - this.validateLabels(labelsOrValue); + validateLabels(labelsOrValue, this.labelNames, 'Gauge'); const labelKey = JSON.stringify(labelsOrValue); const currentValue = this.labeledValues.get(labelKey) ?? 0; this.labeledValues.set(labelKey, currentValue + (value ?? 1)); @@ -169,7 +170,7 @@ export class OtelGauge implements IGaug return; } - this.validateLabels(labelsOrValue); + validateLabels(labelsOrValue, this.labelNames, 'Gauge'); const labelKey = JSON.stringify(labelsOrValue); this.labeledValues.set(labelKey, value!); } @@ -180,7 +181,7 @@ export class OtelGauge implements IGaug */ dec(labels?: Labels): void { if (labels) { - this.validateLabels(labels); + validateLabels(labels, this.labelNames, 'Gauge'); const labelKey = JSON.stringify(labels); const currentValue = this.labeledValues.get(labelKey) ?? 0; this.labeledValues.set(labelKey, currentValue - 1); @@ -197,84 +198,215 @@ export class OtelGauge implements IGaug this.currentValue = 0; this.labeledValues.clear(); } +} + +/** + * Implementation of a Histogram collector + */ +export class OtelHistogram implements IHistogram { + private histogram: Histogram; + + constructor( + private logger: Logger, + meter: Meter, + name: string, + help: string, + buckets: number[] = [], + private labelNames: Array = [], + ) { + this.histogram = meter.createHistogram(name as MetricsType, { + description: help, + advice: buckets.length ? { explicitBucketBoundaries: buckets } : undefined, + }); + } /** - * Validates that provided labels match the expected schema - * @param labels - Labels object to validate - * @throws Error if invalid labels are provided + * Starts a timer and returns a function that when called will record the time elapsed + * @param labels - Optional labels for the observation */ - private validateLabels(labels: Labels): void { - if (this.labelNames.length === 0) { - throw new Error('Gauge was initialized without labels support'); + startTimer(labels?: Labels): () => void { + if (labels) { + validateLabels(labels, this.labelNames, 'Histogram'); } - for (const key of Object.keys(labels)) { - if (!this.labelNames.includes(key as keyof Labels)) { - throw new Error(`Invalid label key: ${key}`); + const timer = new Timer(); + return () => { + // Use timer.s() here to get the duration in seconds since this is only currently used by gossipsub_heartbeat_duration_seconds + const duration = timer.s(); + if (labels) { + this.observe(labels, duration); + } else { + this.observe(duration); } - } + }; } /** - * Safely parses label string back to object - * @param labelStr - Stringified labels object - * @returns Labels object or null if parsing fails + * Observes a value + * @param value - Value to observe + */ + observe(value: number): void; + /** + * Observes a value with labels + * @param labels - Labels object + * @param value - Value to observe */ - private parseLabelsSafely(labelStr: string): Labels | null { - try { - return JSON.parse(labelStr) as Labels; - } catch { - this.logger.error(`Failed to parse label string: ${labelStr}`); - return null; + observe(labels: Labels, value: number): void; + observe(labelsOrValue: Labels | number, value?: number): void { + if (typeof labelsOrValue === 'number') { + this.histogram.record(labelsOrValue); + } else { + validateLabels(labelsOrValue, this.labelNames, 'Histogram'); + this.histogram.record(value!, labelsOrValue); } } + + reset(): void { + // OpenTelemetry histograms cannot be reset, but we implement the interface + this.logger.silent('OpenTelemetry histograms cannot be fully reset'); + } } /** - * Noop implementation of a Historgram collec + * Implementation of an AvgMinMax collector */ -class NoopOtelHistogram implements IHistogram { +export class OtelAvgMinMax implements IAvgMinMax { + private gauges: { + avg: ObservableGauge; + min: ObservableGauge; + max: ObservableGauge; + }; + + private currentValues: number[] = []; + private labeledValues: Map = new Map(); + constructor( private logger: Logger, - _meter: Meter, - _name: string, // MetricsType must be registered in the aztec labels registry - _help: string, - _buckets: number[] = [], - _labelNames: Array = [], - ) {} - - // Overload signatures - observe(_value: number): void; - observe(_labels: Labels, _value: number): void; - observe(_valueOrLabels: number | Labels, _value?: number): void {} - - startTimer(_labels?: Labels): (_labels?: Labels) => number { - return () => 0; + meter: Meter, + name: string, + help: string, + private labelNames: Array = [], + ) { + // Create three separate gauges for avg, min, and max + this.gauges = { + avg: meter.createObservableGauge(`${name}_avg` as MetricsType, { + description: `${help} (average)`, + }), + min: meter.createObservableGauge(`${name}_min` as MetricsType, { + description: `${help} (minimum)`, + }), + max: meter.createObservableGauge(`${name}_max` as MetricsType, { + description: `${help} (maximum)`, + }), + }; + + // Register callbacks for each gauge + this.gauges.avg.addCallback(this.observeAvg.bind(this)); + this.gauges.min.addCallback(this.observeMin.bind(this)); + this.gauges.max.addCallback(this.observeMax.bind(this)); } + /** + * Sets the values for calculating avg, min, max + * @param values - Array of values + */ + set(values: number[]): void; + /** + * Sets the values for calculating avg, min, max with labels + * @param labels - Labels object + * @param values - Array of values + */ + set(labels: Labels, values: number[]): void; + set(labelsOrValues: number[] | Labels, values?: number[]): void { + if (Array.isArray(labelsOrValues)) { + this.currentValues = labelsOrValues; + return; + } else { + validateLabels(labelsOrValues, this.labelNames, 'AvgMinMax'); + const labelKey = JSON.stringify(labelsOrValues); + this.labeledValues.set(labelKey, values || []); + } + } + + /** + * Resets all stored values + */ reset(): void { - // OpenTelemetry histograms cannot be reset, but we implement the interface - this.logger.silent('OpenTelemetry histograms cannot be reset'); + this.currentValues = []; + this.labeledValues.clear(); + } + + /** + * General function to observe an aggregation + * @param result - Observer result + * @param aggregateFn - Function that calculates the aggregation + */ + private observeAggregation(result: any, aggregateFn: (arr: number[]) => number): void { + // Observe unlabeled values + if (this.currentValues.length > 0) { + result.observe(aggregateFn(this.currentValues)); + } + + // Observe labeled values + for (const [labelStr, values] of this.labeledValues.entries()) { + if (values.length > 0) { + const labels = parseLabelsSafely(labelStr, this.logger); + if (labels) { + result.observe(aggregateFn(values), labels); + } + } + } + } + + private observeAvg(result: any): void { + this.observeAggregation(result, arr => arr.reduce((sum, val) => sum + val, 0) / arr.length); + } + + private observeMin(result: any): void { + this.observeAggregation(result, arr => Math.min.apply(null, arr)); + } + + private observeMax(result: any): void { + this.observeAggregation(result, arr => Math.max.apply(null, arr)); } } /** - * Noop implementation of an AvgMinMax collector + * Validates that provided labels match the expected schema + * @param labels - Labels object to validate + * @param labelNames - Array of allowed label names + * @param metricType - Type of metric for error message ('Gauge', 'Histogram', 'AvgMinMax') + * @throws Error if invalid labels are provided */ -class NoopOtelAvgMinMax implements IAvgMinMax { - constructor( - private _logger: Logger, - _meter: Meter, - _name: string, // MetricsType must be registered in the aztec labels registry - _help: string, - _labelNames: Array = [], - ) {} - - set(_values: number[]): void; - set(_labels: Labels, _values: number[]): void; - set(_valueOrLabels: number[] | Labels, _values?: number[]): void {} - - reset(): void {} +function validateLabels( + labels: Labels, + labelNames: Array, + metricType: string, +): void { + if (labelNames.length === 0) { + throw new Error(`${metricType} was initialized without labels support`); + } + + for (const key of Object.keys(labels)) { + if (!labelNames.includes(key as keyof Labels)) { + throw new Error(`Invalid label key: ${key}`); + } + } +} + +/** + * Safely parses label string back to object + * @param labelStr - Stringified labels object + * @param logger - Logger instance for error reporting + * @returns Labels object or null if parsing fails + */ +function parseLabelsSafely(labelStr: string, logger: Logger): Labels | null { + try { + return JSON.parse(labelStr) as Labels; + } catch { + logger.error(`Failed to parse label string: ${labelStr}`); + return null; + } } /** @@ -304,7 +436,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { } histogram(configuration: HistogramConfig): IHistogram { - return new NoopOtelHistogram( + return new OtelHistogram( this.logger, this.meter, configuration.name as MetricsType, @@ -315,7 +447,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { } avgMinMax(configuration: AvgMinMaxConfig): IAvgMinMax { - return new NoopOtelAvgMinMax( + return new OtelAvgMinMax( this.logger, this.meter, configuration.name as MetricsType,