From 8b6aaca703bf89ac4eda641a85419394243fd762 Mon Sep 17 00:00:00 2001 From: Mitch Date: Tue, 26 Nov 2024 20:59:41 -0500 Subject: [PATCH 1/4] plumb gas per second metric --- .../e2e_token_contract/token_contract_test.ts | 6 +++-- .../transfer_in_public.test.ts | 2 ++ .../src/fixtures/snapshot_manager.ts | 2 +- .../simulator/src/public/executor_metrics.ts | 24 ++++++++++++++++++- .../simulator/src/public/public_processor.ts | 3 +++ .../src/public/public_tx_simulator.ts | 23 ++++++++++++++---- .../telemetry-client/src/attributes.ts | 4 ++++ yarn-project/telemetry-client/src/metrics.ts | 1 + yarn-project/telemetry-client/src/otel.ts | 11 ++++++++- 9 files changed, 67 insertions(+), 9 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts index 711e3fa92d77..0eb7afb8b835 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts @@ -14,7 +14,7 @@ import { import { mintTokensToPrivate } from '../fixtures/token_utils.js'; import { TokenSimulator } from '../simulators/token_simulator.js'; -const { E2E_DATA_PATH: dataPath } = process.env; +const { E2E_DATA_PATH: dataPath, METRICS_PORT: metricsPort } = process.env; export class TokenContractTest { static TOKEN_NAME = 'USDC'; @@ -30,7 +30,9 @@ export class TokenContractTest { constructor(testName: string) { this.logger = createDebugLogger(`aztec:e2e_token_contract:${testName}`); - this.snapshotManager = createSnapshotManager(`e2e_token_contract/${testName}`, dataPath); + this.snapshotManager = createSnapshotManager(`e2e_token_contract/${testName}`, dataPath, { + metricsPort: metricsPort ? parseInt(metricsPort) : undefined, + }); } /** diff --git a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts index 0c92a459ad0b..bba93c18da87 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts @@ -3,6 +3,8 @@ import { Fr } from '@aztec/aztec.js'; import { U128_UNDERFLOW_ERROR } from '../fixtures/fixtures.js'; import { TokenContractTest } from './token_contract_test.js'; +// rate(aztec_public_executor_simulation_gas_per_second_per_second_sum[5m]) / rate(aztec_public_executor_simulation_gas_per_second_per_second_count[5m]) + describe('e2e_token_contract transfer public', () => { const t = new TokenContractTest('transfer_in_public'); let { asset, accounts, tokenSim, wallets, badAccount } = t; diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 6f58a48dda6f..488e7291bda4 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -345,7 +345,7 @@ async function setupFromFresh( aztecNodeConfig.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } - const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ 'basenode'); + const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ statePath); logger.verbose('Creating and synching an aztec node...'); const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, { telemetry }); diff --git a/yarn-project/simulator/src/public/executor_metrics.ts b/yarn-project/simulator/src/public/executor_metrics.ts index 4b648dfe4efa..762f5e366a49 100644 --- a/yarn-project/simulator/src/public/executor_metrics.ts +++ b/yarn-project/simulator/src/public/executor_metrics.ts @@ -1,17 +1,23 @@ import { Attributes, + type Gauge, type Histogram, Metrics, type TelemetryClient, + type Tracer, type UpDownCounter, ValueType, + linearBuckets, } from '@aztec/telemetry-client'; export class ExecutorMetrics { + public readonly tracer: Tracer; private fnCount: UpDownCounter; private fnDuration: Histogram; + private gasPerSecond: Gauge; constructor(client: TelemetryClient, name = 'PublicExecutor') { + this.tracer = client.getTracer(name); const meter = client.getMeter(name); this.fnCount = meter.createUpDownCounter(Metrics.PUBLIC_EXECUTOR_SIMULATION_COUNT, { @@ -23,13 +29,29 @@ export class ExecutorMetrics { unit: 'ms', valueType: ValueType.INT, }); + + this.gasPerSecond = meter.createHistogram(Metrics.PUBLIC_EXECUTOR_SIMULATION_GAS_PER_SECOND, { + description: 'Gas used per second', + unit: 'gas/s', + valueType: ValueType.INT, + advice: { + explicitBucketBoundaries: linearBuckets(0, 10_000_000, 10), + }, + }); } - recordFunctionSimulation(durationMs: number) { + recordFunctionSimulation(durationMs: number, gasUsed: number, fnName: string) { this.fnCount.add(1, { [Attributes.OK]: true, + [Attributes.APP_CIRCUIT_NAME]: fnName, }); this.fnDuration.record(Math.ceil(durationMs)); + if (durationMs > 0 && gasUsed > 0) { + const gasPerSecond = Math.round((gasUsed * 1000) / durationMs); + this.gasPerSecond.record(gasPerSecond, { + [Attributes.APP_CIRCUIT_NAME]: fnName, + }); + } } recordFunctionSimulationFailure() { diff --git a/yarn-project/simulator/src/public/public_processor.ts b/yarn-project/simulator/src/public/public_processor.ts index f19c5d069736..23e33b9ee2fd 100644 --- a/yarn-project/simulator/src/public/public_processor.ts +++ b/yarn-project/simulator/src/public/public_processor.ts @@ -229,6 +229,9 @@ export class PublicProcessor { return new PublicDataWrite(leafSlot, updatedBalance); } + @trackSpan('PublicProcessor.processPrivateOnlyTx', (tx: Tx) => ({ + [Attributes.TX_HASH]: tx.getTxHash().toString(), + })) private async processPrivateOnlyTx(tx: Tx): Promise<[ProcessedTx]> { const gasFees = this.globalVariables.gasFees; const transactionFee = tx.data.gasUsed.computeFee(gasFees); diff --git a/yarn-project/simulator/src/public/public_tx_simulator.ts b/yarn-project/simulator/src/public/public_tx_simulator.ts index 44801eff13f8..37bf5245c426 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator.ts @@ -20,7 +20,7 @@ import { } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; -import { type TelemetryClient } from '@aztec/telemetry-client'; +import { Attributes, type TelemetryClient, type Tracer, trackSpan } from '@aztec/telemetry-client'; import { type AvmFinalizedCallResult } from '../avm/avm_contract_call_result.js'; import { type AvmPersistableStateManager, AvmSimulator } from '../avm/index.js'; @@ -55,15 +55,18 @@ export class PublicTxSimulator { constructor( private db: MerkleTreeReadOperations, private worldStateDB: WorldStateDB, - client: TelemetryClient, + telemetryClient: TelemetryClient, private globalVariables: GlobalVariables, private realAvmProvingRequests: boolean = true, private doMerkleOperations: boolean = false, ) { this.log = createDebugLogger(`aztec:public_tx_simulator`); - this.metrics = new ExecutorMetrics(client, 'PublicTxSimulator'); + this.metrics = new ExecutorMetrics(telemetryClient, 'PublicTxSimulator'); } + get tracer(): Tracer { + return this.metrics.tracer; + } /** * Simulate a transaction's public portion including all of its phases. * @param tx - The transaction to simulate. @@ -244,6 +247,12 @@ export class PublicTxSimulator { * @param executionRequest - The execution request (includes args) * @returns The result of execution. */ + @trackSpan('PublicTxSimulator.simulateEnqueuedCall', (phase, context, _callRequest, executionRequest) => ({ + [Attributes.TX_HASH]: context.getTxHash().toString(), + [Attributes.TARGET_ADDRESS]: executionRequest.callContext.contractAddress.toString(), + [Attributes.SENDER_ADDRESS]: executionRequest.callContext.msgSender.toString(), + [Attributes.SIMULATOR_PHASE]: TxExecutionPhase[phase].toString(), + })) private async simulateEnqueuedCall( phase: TxExecutionPhase, context: PublicTxContext, @@ -312,6 +321,12 @@ export class PublicTxSimulator { * @param fnName - The name of the function * @returns The result of execution. */ + @trackSpan( + 'PublicTxSimulator.simulateEnqueuedCallInternal', + (_stateManager, _executionRequest, _allocatedGas, _transactionFee, fnName) => ({ + [Attributes.APP_CIRCUIT_NAME]: fnName, + }), + ) private async simulateEnqueuedCallInternal( stateManager: AvmPersistableStateManager, executionRequest: PublicExecutionRequest, @@ -356,7 +371,7 @@ export class PublicTxSimulator { if (result.reverted) { this.metrics.recordFunctionSimulationFailure(); } else { - this.metrics.recordFunctionSimulation(timer.ms()); + this.metrics.recordFunctionSimulation(timer.ms(), allocatedGas.sub(result.gasLeft).l2Gas, fnName); } return result; diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 32ce263917a0..02402802a257 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -83,3 +83,7 @@ export const P2P_ID = 'aztec.p2p.id'; export const POOL_NAME = 'aztec.pool.name'; export const SEQUENCER_STATE = 'aztec.sequencer.state'; + +export const SIMULATOR_PHASE = 'aztec.simulator.phase'; +export const TARGET_ADDRESS = 'aztec.address.target'; +export const SENDER_ADDRESS = 'aztec.address.sender'; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index f61e5be54ea2..214d85f57f41 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -71,6 +71,7 @@ export const PUBLIC_PROCESSOR_DEPLOY_BYTECODE_SIZE = 'aztec.public_processor.dep export const PUBLIC_EXECUTOR_SIMULATION_COUNT = 'aztec.public_executor.simulation_count'; export const PUBLIC_EXECUTOR_SIMULATION_DURATION = 'aztec.public_executor.simulation_duration'; +export const PUBLIC_EXECUTOR_SIMULATION_GAS_PER_SECOND = 'aztec.public_executor.simulation_gas_per_second'; export const PUBLIC_EXECUTION_SIMULATION_BYTECODE_SIZE = 'aztec.public_executor.simulation_bytecode_size'; export const PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION = diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index 446c50f0e89e..f94d054cb314 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -75,7 +75,16 @@ export class OpenTelemetryClient implements TelemetryClient { } public async stop() { - await Promise.all([this.meterProvider.shutdown(), this.loggerProvider.shutdown()]); + const flushAndShutdown = async (provider: { forceFlush: () => Promise; shutdown: () => Promise }) => { + await provider.forceFlush(); + await provider.shutdown(); + }; + + await Promise.all([ + flushAndShutdown(this.meterProvider), + flushAndShutdown(this.loggerProvider), + this.traceProvider instanceof NodeTracerProvider ? flushAndShutdown(this.traceProvider) : Promise.resolve(), + ]); } public static async createAndStart(config: TelemetryClientConfig, log: DebugLogger): Promise { From 9cef6e44779a147009685d0e759d4ed8aa24bb37 Mon Sep 17 00:00:00 2001 From: Mitch Date: Tue, 26 Nov 2024 21:10:57 -0500 Subject: [PATCH 2/4] enable the metric test --- .../end-to-end/scripts/e2e_test_config.yml | 3 ++- .../transfer_in_public.test.ts | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/scripts/e2e_test_config.yml b/yarn-project/end-to-end/scripts/e2e_test_config.yml index 4faa61cceb77..a09f5f5a568f 100644 --- a/yarn-project/end-to-end/scripts/e2e_test_config.yml +++ b/yarn-project/end-to-end/scripts/e2e_test_config.yml @@ -84,7 +84,8 @@ tests: e2e_state_vars: {} e2e_static_calls: {} e2e_synching: {} - e2e_token_contract: {} + e2e_token_contract: + with_alerts: true e2e_p2p_gossip: test_path: 'e2e_p2p/gossip_network.test.ts' with_alerts: true diff --git a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts index bba93c18da87..5f1198eadab3 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts @@ -1,9 +1,21 @@ import { Fr } from '@aztec/aztec.js'; import { U128_UNDERFLOW_ERROR } from '../fixtures/fixtures.js'; +import { AlertChecker, type AlertConfig } from '../quality_of_service/alert_checker.js'; import { TokenContractTest } from './token_contract_test.js'; -// rate(aztec_public_executor_simulation_gas_per_second_per_second_sum[5m]) / rate(aztec_public_executor_simulation_gas_per_second_per_second_count[5m]) +const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; + +const qosAlerts: AlertConfig[] = [ + { + // 2.5M gas/s + alert: 'public_executor_gas_per_second', + expr: 'rate(aztec_public_executor_simulation_gas_per_second_per_second_sum[5m]) / rate(aztec_public_executor_simulation_gas_per_second_per_second_count[5m]) < 2500000', + for: '5m', + annotations: {}, + labels: {}, + }, +]; describe('e2e_token_contract transfer public', () => { const t = new TokenContractTest('transfer_in_public'); @@ -19,6 +31,10 @@ describe('e2e_token_contract transfer public', () => { afterAll(async () => { await t.teardown(); + if (CHECK_ALERTS) { + const alertChecker = new AlertChecker(t.logger); + await alertChecker.runAlertCheck(qosAlerts); + } }); afterEach(async () => { From fe8a3c3bc07096df2873415d86f50da8528c0636 Mon Sep 17 00:00:00 2001 From: Mitch Date: Tue, 26 Nov 2024 21:12:50 -0500 Subject: [PATCH 3/4] use correct type --- yarn-project/simulator/src/public/executor_metrics.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn-project/simulator/src/public/executor_metrics.ts b/yarn-project/simulator/src/public/executor_metrics.ts index 762f5e366a49..5b17d68354cf 100644 --- a/yarn-project/simulator/src/public/executor_metrics.ts +++ b/yarn-project/simulator/src/public/executor_metrics.ts @@ -1,6 +1,5 @@ import { Attributes, - type Gauge, type Histogram, Metrics, type TelemetryClient, @@ -14,7 +13,7 @@ export class ExecutorMetrics { public readonly tracer: Tracer; private fnCount: UpDownCounter; private fnDuration: Histogram; - private gasPerSecond: Gauge; + private gasPerSecond: Histogram; constructor(client: TelemetryClient, name = 'PublicExecutor') { this.tracer = client.getTracer(name); From 37f4ce36dad4f09d86cf78b5c4083ff8df6b1915 Mon Sep 17 00:00:00 2001 From: Mitch Date: Wed, 27 Nov 2024 08:58:48 -0500 Subject: [PATCH 4/4] change nomenclature to mana/second drop target to just a dummy 10 value --- .../transfer_in_public.test.ts | 7 ++++--- .../simulator/src/public/executor_metrics.ts | 17 +++++++++-------- yarn-project/telemetry-client/src/attributes.ts | 1 + yarn-project/telemetry-client/src/metrics.ts | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts index 5f1198eadab3..b0440880ecb1 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/transfer_in_public.test.ts @@ -8,9 +8,10 @@ const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; const qosAlerts: AlertConfig[] = [ { - // 2.5M gas/s - alert: 'public_executor_gas_per_second', - expr: 'rate(aztec_public_executor_simulation_gas_per_second_per_second_sum[5m]) / rate(aztec_public_executor_simulation_gas_per_second_per_second_count[5m]) < 2500000', + // Dummy alert to check that the metric is being emitted. + // Separate benchmark tests will use dedicated machines with the published system requirements. + alert: 'publishing_mana_per_second', + expr: 'rate(aztec_public_executor_simulation_mana_per_second_per_second_sum[5m]) / rate(aztec_public_executor_simulation_mana_per_second_per_second_count[5m]) < 10', for: '5m', annotations: {}, labels: {}, diff --git a/yarn-project/simulator/src/public/executor_metrics.ts b/yarn-project/simulator/src/public/executor_metrics.ts index 5b17d68354cf..17267f58b900 100644 --- a/yarn-project/simulator/src/public/executor_metrics.ts +++ b/yarn-project/simulator/src/public/executor_metrics.ts @@ -13,7 +13,7 @@ export class ExecutorMetrics { public readonly tracer: Tracer; private fnCount: UpDownCounter; private fnDuration: Histogram; - private gasPerSecond: Histogram; + private manaPerSecond: Histogram; constructor(client: TelemetryClient, name = 'PublicExecutor') { this.tracer = client.getTracer(name); @@ -29,9 +29,9 @@ export class ExecutorMetrics { valueType: ValueType.INT, }); - this.gasPerSecond = meter.createHistogram(Metrics.PUBLIC_EXECUTOR_SIMULATION_GAS_PER_SECOND, { - description: 'Gas used per second', - unit: 'gas/s', + this.manaPerSecond = meter.createHistogram(Metrics.PUBLIC_EXECUTOR_SIMULATION_MANA_PER_SECOND, { + description: 'Mana used per second', + unit: 'mana/s', valueType: ValueType.INT, advice: { explicitBucketBoundaries: linearBuckets(0, 10_000_000, 10), @@ -39,15 +39,16 @@ export class ExecutorMetrics { }); } - recordFunctionSimulation(durationMs: number, gasUsed: number, fnName: string) { + recordFunctionSimulation(durationMs: number, manaUsed: number, fnName: string) { this.fnCount.add(1, { [Attributes.OK]: true, [Attributes.APP_CIRCUIT_NAME]: fnName, + [Attributes.MANA_USED]: manaUsed, }); this.fnDuration.record(Math.ceil(durationMs)); - if (durationMs > 0 && gasUsed > 0) { - const gasPerSecond = Math.round((gasUsed * 1000) / durationMs); - this.gasPerSecond.record(gasPerSecond, { + if (durationMs > 0 && manaUsed > 0) { + const manaPerSecond = Math.round((manaUsed * 1000) / durationMs); + this.manaPerSecond.record(manaPerSecond, { [Attributes.APP_CIRCUIT_NAME]: fnName, }); } diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 02402802a257..87c4be24ce0b 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -87,3 +87,4 @@ export const SEQUENCER_STATE = 'aztec.sequencer.state'; export const SIMULATOR_PHASE = 'aztec.simulator.phase'; export const TARGET_ADDRESS = 'aztec.address.target'; export const SENDER_ADDRESS = 'aztec.address.sender'; +export const MANA_USED = 'aztec.mana.used'; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 214d85f57f41..01f3e4834973 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -71,7 +71,7 @@ export const PUBLIC_PROCESSOR_DEPLOY_BYTECODE_SIZE = 'aztec.public_processor.dep export const PUBLIC_EXECUTOR_SIMULATION_COUNT = 'aztec.public_executor.simulation_count'; export const PUBLIC_EXECUTOR_SIMULATION_DURATION = 'aztec.public_executor.simulation_duration'; -export const PUBLIC_EXECUTOR_SIMULATION_GAS_PER_SECOND = 'aztec.public_executor.simulation_gas_per_second'; +export const PUBLIC_EXECUTOR_SIMULATION_MANA_PER_SECOND = 'aztec.public_executor.simulation_mana_per_second'; export const PUBLIC_EXECUTION_SIMULATION_BYTECODE_SIZE = 'aztec.public_executor.simulation_bytecode_size'; export const PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION =