diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml index 518cf3d81d43..89166794d256 100644 --- a/.github/workflows/devnet-deploys.yml +++ b/.github/workflows/devnet-deploys.yml @@ -66,6 +66,8 @@ env: TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }} TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }} TF_VAR_FORK_ADMIN_API_KEY: ${{ secrets.DEVNET_API_KEY }} + TF_VAR_MAINNET_FORK_CPU_UNITS: 2048 + TF_VAR_MAINNET_FORK_MEMORY_UNITS: 4096 # Faucet TF_VAR_FAUCET_ACCOUNT_INDEX: 9 @@ -123,6 +125,12 @@ jobs: min_txs_per_block: ${{ steps.set_network_vars.outputs.min_txs_per_block }} bot_flush_setup_txs: ${{ steps.set_network_vars.outputs.bot_flush_setup_txs }} bot_max_pending_txs: ${{ steps.set_network_vars.outputs.bot_max_pending_txs }} + mainnet_fork_cpu_units: ${{ steps.set_network_vars.outputs.mainnet_fork_cpu_units }} + mainnet_fork_memory_units: ${{ steps.set_network_vars.outputs.mainnet_fork_memory_units }} + bot_skip_simulation: ${{ steps.set_network_vars.outputs.bot_skip_simulation }} + bot_l2_gas_limit: ${{ steps.set_network_vars.outputs.bot_l2_gas_limit }} + bot_da_gas_limit: ${{ steps.set_network_vars.outputs.bot_da_gas_limit }} + bot_count: ${{ steps.set_network_vars.outputs.bot_count }} steps: - name: Set network vars shell: bash @@ -135,7 +143,7 @@ jobs: echo "branch_name=devnet" >> $GITHUB_OUTPUT echo "network_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT echo "network_fork_admin_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT - echo "agents_per_prover=4" >> $GITHUB_OUTPUT + echo "agents_per_prover=2" >> $GITHUB_OUTPUT echo "bot_interval=180" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT @@ -147,9 +155,15 @@ jobs: echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT echo "min_txs_per_block=1" >> $GITHUB_OUTPUT echo "max_txs_per_block=64" >> $GITHUB_OUTPUT - echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT + echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT + echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT + echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT + echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT + echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT + echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT + echo "bot_count=1" >> $GITHUB_OUTPUT elif [ "$BRANCH_NAME" = "provernet" ] then echo "deploy_tag=provernet" >> $GITHUB_OUTPUT @@ -171,6 +185,12 @@ jobs: echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT echo "bot_flush_setup_txs=true" >> $GITHUB_OUTPUT echo "bot_max_pending_txs=32" >> $GITHUB_OUTPUT + echo "mainnet_fork_cpu_units=8192" >> $GITHUB_OUTPUT + echo "mainnet_fork_memory_units=32768" >> $GITHUB_OUTPUT + echo "bot_skip_simulation=true" >> $GITHUB_OUTPUT + echo "bot_l2_gas_limit=1000000000" >> $GITHUB_OUTPUT + echo "bot_da_gas_limit=1000000000" >> $GITHUB_OUTPUT + echo "bot_count=1" >> $GITHUB_OUTPUT elif [ "$BRANCH_NAME" = "alphanet" ] then echo "deploy_tag=alphanet" >> $GITHUB_OUTPUT @@ -178,7 +198,7 @@ jobs: echo "network_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT echo "network_fork_admin_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT echo "agents_per_prover=1" >> $GITHUB_OUTPUT - echo "bot_interval=30" >> $GITHUB_OUTPUT + echo "bot_interval=10" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT @@ -192,6 +212,12 @@ jobs: echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT + echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT + echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT + echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT + echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT + echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT + echo "bot_count=1" >> $GITHUB_OUTPUT else echo "Unrecognized Branch!!" exit 1 @@ -462,6 +488,12 @@ jobs: TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }} TF_VAR_SEQ_MIN_TX_PER_BLOCK: 1 TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }} + TF_VAR_MAINNET_FORK_CPU_UNITS: ${{ needs.set-network.outputs.mainnet_fork_cpu_units }} + TF_VAR_MAINNET_FORK_MEMORY_UNITS: ${{ needs.set-network.outputs.mainnet_fork_memory_units }} + TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }} + TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }} + TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }} + TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }} steps: - uses: actions/checkout@v4 with: @@ -679,6 +711,10 @@ jobs: TF_VAR_BOT_FOLLOW_CHAIN: ${{ needs.set-network.outputs.bot_follow_chain }} TF_VAR_PROVING_ENABLED: true TF_VAR_BOT_NO_START: false + TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }} + TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }} + TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }} + TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }} steps: - uses: actions/checkout@v4 with: diff --git a/iac/mainnet-fork/terraform/main.tf b/iac/mainnet-fork/terraform/main.tf index 49c407db09c5..737be2e298f9 100644 --- a/iac/mainnet-fork/terraform/main.tf +++ b/iac/mainnet-fork/terraform/main.tf @@ -102,8 +102,8 @@ resource "aws_ecs_task_definition" "aztec_mainnet_fork" { family = "${var.DEPLOY_TAG}-mainnet-fork" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = "2048" - memory = "4096" + cpu = var.MAINNET_FORK_CPU_UNITS + memory = var.MAINNET_FORK_MEMORY_UNITS execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn volume { diff --git a/iac/mainnet-fork/terraform/variables.tf b/iac/mainnet-fork/terraform/variables.tf index 1ba3012169b8..c64895845c9b 100644 --- a/iac/mainnet-fork/terraform/variables.tf +++ b/iac/mainnet-fork/terraform/variables.tf @@ -25,3 +25,13 @@ variable "DEPLOY_TAG" { variable "L1_CHAIN_ID" { type = string } + +variable "MAINNET_FORK_CPU_UNITS" { + type = string + default = "2048" +} + +variable "MAINNET_FORK_MEMORY_UNITS" { + type = string + default = "4096" +} diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index 9514bf2b7e45..c1571c1429c4 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -2,7 +2,10 @@ "name": "@aztec/archiver", "version": "0.1.0", "type": "module", - "exports": "./dest/index.js", + "exports": { + ".": "./dest/index.js", + "./data-retrieval": "./dest/archiver/data_retrieval.js" + }, "typedocOptions": { "entryPoints": [ "./src/index.ts" diff --git a/yarn-project/archiver/src/archiver/data_retrieval.ts b/yarn-project/archiver/src/archiver/data_retrieval.ts index 4ae92665b3d7..81aca60a7837 100644 --- a/yarn-project/archiver/src/archiver/data_retrieval.ts +++ b/yarn-project/archiver/src/archiver/data_retrieval.ts @@ -1,5 +1,5 @@ import { type Body, type InboxLeaf } from '@aztec/circuit-types'; -import { type AppendOnlyTreeSnapshot, Fr, type Header } from '@aztec/circuits.js'; +import { type AppendOnlyTreeSnapshot, Fr, type Header, type Proof } from '@aztec/circuits.js'; import { type EthAddress } from '@aztec/foundation/eth-address'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RollupAbi } from '@aztec/l1-artifacts'; @@ -7,6 +7,7 @@ import { RollupAbi } from '@aztec/l1-artifacts'; import { type Hex, type PublicClient, getAbiItem } from 'viem'; import { + getBlockProofFromSubmitProofTx, getL2BlockProposedLogs, getMessageSentLogs, getTxsPublishedLogs, @@ -163,3 +164,24 @@ export async function retrieveL2ProofVerifiedEvents( txHash: log.transactionHash, })); } + +/** Retrieve submitted proofs from the rollup contract */ +export async function retrieveL2ProofsFromRollup( + publicClient: PublicClient, + rollupAddress: EthAddress, + searchStartBlock: bigint, + searchEndBlock?: bigint, +): Promise> { + const logs = await retrieveL2ProofVerifiedEvents(publicClient, rollupAddress, searchStartBlock, searchEndBlock); + const retrievedData: { proof: Proof; proverId: Fr; l2BlockNumber: bigint; txHash: `0x${string}` }[] = []; + const lastProcessedL1BlockNumber = logs.length > 0 ? logs.at(-1)!.l1BlockNumber : searchStartBlock - 1n; + + for (const { txHash, proverId, l2BlockNumber } of logs) { + const proofData = await getBlockProofFromSubmitProofTx(publicClient, txHash, l2BlockNumber, proverId); + retrievedData.push({ proof: proofData.proof, proverId: proofData.proverId, l2BlockNumber, txHash }); + } + return { + retrievedData, + lastProcessedL1BlockNumber, + }; +} diff --git a/yarn-project/archiver/src/archiver/eth_log_handlers.ts b/yarn-project/archiver/src/archiver/eth_log_handlers.ts index 04d8a8b53391..990cccde1aed 100644 --- a/yarn-project/archiver/src/archiver/eth_log_handlers.ts +++ b/yarn-project/archiver/src/archiver/eth_log_handlers.ts @@ -1,5 +1,5 @@ import { Body, InboxLeaf } from '@aztec/circuit-types'; -import { AppendOnlyTreeSnapshot, Header } from '@aztec/circuits.js'; +import { AppendOnlyTreeSnapshot, Header, Proof } from '@aztec/circuits.js'; import { type EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { numToUInt32BE } from '@aztec/foundation/serialize'; @@ -257,3 +257,57 @@ export function getMessageSentLogs( toBlock: toBlock + 1n, // the toBlock argument in getLogs is exclusive }); } + +export type SubmitBlockProof = { + header: Header; + archiveRoot: Fr; + proverId: Fr; + aggregationObject: Buffer; + proof: Proof; +}; + +/** + * Gets block metadata (header and archive snapshot) from the calldata of an L1 transaction. + * Assumes that the block was published from an EOA. + * TODO: Add retries and error management. + * @param publicClient - The viem public client to use for transaction retrieval. + * @param txHash - Hash of the tx that published it. + * @param l2BlockNum - L2 block number. + * @returns L2 block metadata (header and archive) from the calldata, deserialized + */ +export async function getBlockProofFromSubmitProofTx( + publicClient: PublicClient, + txHash: `0x${string}`, + l2BlockNum: bigint, + expectedProverId: Fr, +): Promise { + const { input: data } = await publicClient.getTransaction({ hash: txHash }); + const { functionName, args } = decodeFunctionData({ + abi: RollupAbi, + data, + }); + + if (!(functionName === 'submitBlockRootProof')) { + throw new Error(`Unexpected method called ${functionName}`); + } + const [headerHex, archiveHex, proverIdHex, aggregationObjectHex, proofHex] = args!; + + const header = Header.fromBuffer(Buffer.from(hexToBytes(headerHex))); + const proverId = Fr.fromString(proverIdHex); + + const blockNumberFromHeader = header.globalVariables.blockNumber.toBigInt(); + if (blockNumberFromHeader !== l2BlockNum) { + throw new Error(`Block number mismatch: expected ${l2BlockNum} but got ${blockNumberFromHeader}`); + } + if (!proverId.equals(expectedProverId)) { + throw new Error(`Prover ID mismatch: expected ${expectedProverId} but got ${proverId}`); + } + + return { + header, + proverId, + aggregationObject: Buffer.from(hexToBytes(aggregationObjectHex)), + archiveRoot: Fr.fromString(archiveHex), + proof: Proof.fromBuffer(Buffer.from(hexToBytes(proofHex))), + }; +} diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index fe0c37048ee4..b986f0d167bd 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -48,6 +48,7 @@ "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/p2p": "workspace:^", "@aztec/p2p-bootstrap": "workspace:^", + "@aztec/proof-verifier": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/prover-client": "workspace:^", "@aztec/prover-node": "workspace:^", diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts index 1a405bb5475e..953f07adcb34 100644 --- a/yarn-project/aztec/src/cli/aztec_start_options.ts +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -9,6 +9,7 @@ import { isBooleanConfigValue, } from '@aztec/foundation/config'; import { bootnodeConfigMappings, p2pConfigMappings } from '@aztec/p2p'; +import { proofVerifierConfigMappings } from '@aztec/proof-verifier'; import { proverClientConfigMappings } from '@aztec/prover-client'; import { proverNodeConfigMappings } from '@aztec/prover-node'; import { allPxeConfigMappings } from '@aztec/pxe'; @@ -303,6 +304,15 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { }, ...getOptions('bot', botConfigMappings), ], + 'PROOF VERIFIER': [ + { + flag: '--proof-verifier', + description: 'Starts Aztec Proof Verifier with options', + defaultValue: undefined, + envVar: undefined, + }, + ...getOptions('proofVerifier', proofVerifierConfigMappings), + ], TXE: [ { flag: '--txe', diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index d05cc439a67f..fc8a0d226af0 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -76,6 +76,9 @@ export function injectAztecCommands(program: Command, userLog: LogFn, debugLogge if (options.node) { const { startNode } = await import('./cmds/start_node.js'); services = await startNode(options, signalHandlers, userLog); + } else if (options.proofVerifier) { + const { startProofVerifier } = await import('./cmds/start_proof_verifier.js'); + services = await startProofVerifier(options, signalHandlers, userLog); } else if (options.bot) { const { startBot } = await import('./cmds/start_bot.js'); services = await startBot(options, signalHandlers, userLog); @@ -101,7 +104,7 @@ export function injectAztecCommands(program: Command, userLog: LogFn, debugLogge userLog(`Cannot run a standalone sequencer without a node`); process.exit(1); } else { - userLog(`No module specified to start ${JSON.stringify(options, null, 2)}`); + userLog(`No module specified to start`); process.exit(1); } } diff --git a/yarn-project/aztec/src/cli/cmds/start_proof_verifier.ts b/yarn-project/aztec/src/cli/cmds/start_proof_verifier.ts new file mode 100644 index 000000000000..89477caaffb8 --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_proof_verifier.ts @@ -0,0 +1,26 @@ +import { type ServerList } from '@aztec/foundation/json-rpc/server'; +import { type LogFn } from '@aztec/foundation/log'; +import { ProofVerifier, proofVerifierConfigMappings } from '@aztec/proof-verifier'; +import { createAndStartTelemetryClient, telemetryClientConfigMappings } from '@aztec/telemetry-client/start'; + +import { extractRelevantOptions } from '../util.js'; + +export async function startProofVerifier( + options: any, + signalHandlers: (() => Promise)[], + userLog: LogFn, +): Promise { + const services: ServerList = []; + + const config = extractRelevantOptions(options, proofVerifierConfigMappings, 'proofVerifier'); + + const telemetryConfig = extractRelevantOptions(options, telemetryClientConfigMappings, 'tel'); + const telemetry = await createAndStartTelemetryClient(telemetryConfig); + const proofVerifier = await ProofVerifier.new(config, telemetry); + + userLog('Starting proof verifier'); + proofVerifier.start(); + + signalHandlers.push(() => proofVerifier.stop()); + return services; +} diff --git a/yarn-project/aztec/terraform/bot/main.tf b/yarn-project/aztec/terraform/bot/main.tf index 31f9d1cd9bae..4dfe0641bf4a 100644 --- a/yarn-project/aztec/terraform/bot/main.tf +++ b/yarn-project/aztec/terraform/bot/main.tf @@ -173,6 +173,9 @@ resource "aws_ecs_task_definition" "aztec-bot" { { name = "NETWORK", value = var.DEPLOY_TAG }, { name = "BOT_FLUSH_SETUP_TRANSACTIONS", value = tostring(var.BOT_FLUSH_SETUP_TRANSACTIONS) }, { name = "BOT_MAX_PENDING_TXS", value = tostring(var.BOT_MAX_PENDING_TXS) }, + { name = "BOT_SKIP_PUBLIC_SIMULATION", value = tostring(var.BOT_SKIP_PUBLIC_SIMULATION) }, + { name = "BOT_L2_GAS_LIMIT", value = var.BOT_L2_GAS_LIMIT }, + { name = "BOT_DA_GAS_LIMIT", value = var.BOT_DA_GAS_LIMIT }, { name = "LOG_JSON", value = "1" } ] logConfiguration = { diff --git a/yarn-project/aztec/terraform/bot/variables.tf b/yarn-project/aztec/terraform/bot/variables.tf index 4d3d78100f4b..679dd2187ab7 100644 --- a/yarn-project/aztec/terraform/bot/variables.tf +++ b/yarn-project/aztec/terraform/bot/variables.tf @@ -66,3 +66,18 @@ variable "BOT_MAX_PENDING_TXS" { type = number default = 1 } + +variable "BOT_SKIP_PUBLIC_SIMULATION" { + type = bool + default = false +} + +variable "BOT_L2_GAS_LIMIT" { + type = string +} + +variable "BOT_DA_GAS_LIMIT" { + type = string +} + + diff --git a/yarn-project/aztec/terraform/proof-verifier/main.tf b/yarn-project/aztec/terraform/proof-verifier/main.tf new file mode 100644 index 000000000000..970bcc2f9fee --- /dev/null +++ b/yarn-project/aztec/terraform/proof-verifier/main.tf @@ -0,0 +1,221 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + region = "eu-west-2" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.74.2" + } + } +} + +# Define provider and region +provider "aws" { + region = "eu-west-2" +} + +data "terraform_remote_state" "aztec2_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec2/iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "setup_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "setup/setup-iac" + region = "eu-west-2" + } +} + +resource "aws_cloudwatch_log_group" "aztec-proof-verifier-log-group" { + name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proof-verifier" + retention_in_days = 14 +} + +resource "aws_service_discovery_service" "aztec-proof-verifier" { + name = "${var.DEPLOY_TAG}-aztec-proof-verifier" + + health_check_custom_config { + failure_threshold = 1 + } + + dns_config { + namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id + + dns_records { + ttl = 60 + type = "A" + } + + dns_records { + ttl = 60 + type = "SRV" + } + + routing_policy = "MULTIVALUE" + } + + # Terraform just fails if this resource changes and you have registered instances. + provisioner "local-exec" { + when = destroy + command = "${path.module}/../servicediscovery-drain.sh ${self.id}" + } +} + +# Create a fleet. +data "template_file" "user_data" { + template = <> /etc/ecs/ecs.config +echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-proof-verifier"}' >> /etc/ecs/ecs.config +EOF +} + +resource "aws_launch_template" "proof_verifier_launch_template" { + name = "${var.DEPLOY_TAG}-pf-launch-template" + image_id = "ami-0cd4858f2b923aa6b" + instance_type = "m4.2xlarge" // 8 cores, 32 GB + vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + + iam_instance_profile { + name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name + } + + key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name + + user_data = base64encode(data.template_file.user_data.rendered) + + tag_specifications { + resource_type = "instance" + tags = { + Name = "${var.DEPLOY_TAG}-proof-verifier" + prometheus = "" + } + } +} + +resource "aws_ec2_fleet" "proof_verifier_fleet" { + launch_template_config { + launch_template_specification { + launch_template_id = aws_launch_template.proof_verifier_launch_template.id + version = aws_launch_template.proof_verifier_launch_template.latest_version + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id + availability_zone = "eu-west-2a" + max_price = "0.15" + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + availability_zone = "eu-west-2b" + max_price = "0.15" + } + } + + target_capacity_specification { + default_target_capacity_type = "on-demand" + total_target_capacity = 1 + spot_target_capacity = 0 + on_demand_target_capacity = 1 + } + + terminate_instances = true + terminate_instances_with_expiration = true +} + +resource "aws_ecs_task_definition" "aztec-proof-verifier" { + family = "${var.DEPLOY_TAG}-aztec-proof-verifier" + network_mode = "awsvpc" + requires_compatibilities = ["EC2"] + execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn + task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn + + container_definitions = jsonencode([ + { + name = "${var.DEPLOY_TAG}-aztec-proof-verifier" + image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}" + command = ["start", "--proof-verifier"] + essential = true + cpu = 8192 + memoryReservation = 30720 + portMappings = [ + { + containerPort = 80 + } + ] + environment = [ + { name = "PROOF_VERIFIER_L1_START_BLOCK", value = "15918000" }, + { name = "PROOF_VERIFIER_POLL_INTERVAL_MS", value = tostring(var.PROOF_VERIFIER_POLL_INTERVAL_MS) }, + { name = "ETHEREUM_HOST", value = var.ETHEREUM_HOST }, + { name = "L1_CHAIN_ID", value = tostring(var.L1_CHAIN_ID) }, + { name = "ROLLUP_CONTRACT_ADDRESS", value = var.ROLLUP_CONTRACT_ADDRESS }, + { + name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" + value = "http://aztec-otel.local:4318/v1/metrics" + }, + { + name = "OTEL_SERVICE_NAME" + value = "${var.DEPLOY_TAG}-aztec-proof-verifier" + }, + { name = "LOG_LEVEL", value = var.LOG_LEVEL }, + { name = "NETWORK", value = var.DEPLOY_TAG }, + { name = "LOG_JSON", value = "1" } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.aztec-proof-verifier-log-group.name + "awslogs-region" = "eu-west-2" + "awslogs-stream-prefix" = "ecs" + } + } + } + ]) +} + +resource "aws_ecs_service" "aztec-proof-verifier" { + name = "${var.DEPLOY_TAG}-aztec-proof-verifier" + cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id + launch_type = "EC2" + desired_count = 1 + deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 0 + force_new_deployment = true + enable_execute_command = true + + network_configuration { + subnets = [ + data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, + data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + ] + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + } + + # load_balancer { + # target_group_arn = aws_alb_target_group.bot_http.arn + # container_name = "${var.DEPLOY_TAG}-aztec-proof-verifier" + # container_port = 80 + # } + + service_registries { + registry_arn = aws_service_discovery_service.aztec-proof-verifier.arn + container_name = "${var.DEPLOY_TAG}-aztec-proof-verifier" + container_port = 80 + } + + placement_constraints { + type = "memberOf" + expression = "attribute:group == ${var.DEPLOY_TAG}-proof-verifier" + } + + task_definition = aws_ecs_task_definition.aztec-proof-verifier.family +} diff --git a/yarn-project/aztec/terraform/proof-verifier/variables.tf b/yarn-project/aztec/terraform/proof-verifier/variables.tf new file mode 100644 index 000000000000..69aac96df1e1 --- /dev/null +++ b/yarn-project/aztec/terraform/proof-verifier/variables.tf @@ -0,0 +1,29 @@ +variable "DEPLOY_TAG" { + type = string +} + +variable "DOCKERHUB_ACCOUNT" { + type = string +} + +variable "LOG_LEVEL" { + type = string + default = "verbose" +} + +variable "ETHEREUM_HOST" { + type = string +} + +variable "L1_CHAIN_ID" { + type = number +} + +variable "ROLLUP_CONTRACT_ADDRESS" { + type = string +} + +variable "PROOF_VERIFIER_POLL_INTERVAL_MS" { + type = number + default = 60000 +} diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf index 924aae3e6eac..537bc7e73f05 100644 --- a/yarn-project/aztec/terraform/prover/main.tf +++ b/yarn-project/aztec/terraform/prover/main.tf @@ -101,12 +101,12 @@ EOF } # Launch template for our prover agents -# 32 cores and 128 GB memory +# 16 cores and 128 GB memory resource "aws_launch_template" "proving-agent-launch-template" { count = local.node_count name = "${var.DEPLOY_TAG}-proving-agent-launch-template-${count.index + 1}" image_id = "ami-0cd4858f2b923aa6b" - instance_type = "m5.8xlarge" + instance_type = "r5a.4xlarge" vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] iam_instance_profile { @@ -237,7 +237,7 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" { "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}", "command": ["start", "--prover"], "essential": true, - "cpu": 32768, + "cpu": 16384, "memoryReservation": 122880, "portMappings": [ { diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index 72f82e9d8050..570da13289f6 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -66,6 +66,9 @@ { "path": "../p2p-bootstrap" }, + { + "path": "../proof-verifier" + }, { "path": "../protocol-contracts" }, diff --git a/yarn-project/end-to-end/package.local.json b/yarn-project/end-to-end/package.local.json index a5214893419b..62f136fa45da 100644 --- a/yarn-project/end-to-end/package.local.json +++ b/yarn-project/end-to-end/package.local.json @@ -5,4 +5,4 @@ "test": "LOG_LEVEL=${LOG_LEVEL:-verbose} DEBUG_COLORS=1 NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --testTimeout=300000 --forceExit", "test:unit": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest src/fixtures" } -} +} \ No newline at end of file diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 43072be644c8..43f908c5d725 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -111,5 +111,7 @@ export type EnvVar = | 'VALIDATOR_DISABLED' | 'PROVER_NODE_DISABLE_AUTOMATIC_PROVING' | 'PROVER_NODE_MAX_PENDING_JOBS' + | 'PROOF_VERIFIER_POLL_INTERVAL_MS' + | 'PROOF_VERIFIER_L1_START_BLOCK' | 'LOG_LEVEL' | 'DEBUG'; diff --git a/yarn-project/package.json b/yarn-project/package.json index 5707c688028b..316150e46fdc 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -49,6 +49,7 @@ "noir-protocol-circuits-types", "p2p", "p2p-bootstrap", + "proof-verifier", "protocol-contracts", "prover-client", "prover-node", diff --git a/yarn-project/proof-verifier/.eslintrc.cjs b/yarn-project/proof-verifier/.eslintrc.cjs new file mode 100644 index 000000000000..e659927475c0 --- /dev/null +++ b/yarn-project/proof-verifier/.eslintrc.cjs @@ -0,0 +1 @@ +module.exports = require('@aztec/foundation/eslint'); diff --git a/yarn-project/proof-verifier/package.json b/yarn-project/proof-verifier/package.json new file mode 100644 index 000000000000..1a6c5f42afd7 --- /dev/null +++ b/yarn-project/proof-verifier/package.json @@ -0,0 +1,73 @@ +{ + "name": "@aztec/proof-verifier", + "type": "module", + "exports": { + ".": "./dest/index.js" + }, + "scripts": { + "build": "yarn clean && tsc -b", + "build:dev": "tsc -b --watch", + "clean": "rm -rf ./dest .tsbuildinfo", + "formatting": "run -T prettier --check ./src && run -T eslint ./src", + "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + }, + "engines": { + "node": ">=18" + }, + "files": [ + "dest", + "src", + "!*.test.*" + ], + "dependencies": { + "@aztec/archiver": "workspace:^", + "@aztec/bb-prover": "workspace:^", + "@aztec/circuit-types": "workspace:^", + "@aztec/circuits.js": "workspace:^", + "@aztec/ethereum": "workspace:^", + "@aztec/foundation": "workspace:^", + "@aztec/noir-protocol-circuits-types": "workspace:^", + "@aztec/telemetry-client": "workspace:^", + "viem": "^2.7.15" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@types/jest": "^29.5.0", + "@types/node": "^18.7.23", + "jest": "^29.5.0", + "ts-node": "^10.9.1", + "typescript": "^5.0.4" + }, + "jest": { + "extensionsToTreatAsEsm": [ + ".ts" + ], + "transform": { + "^.+\\.tsx?$": [ + "@swc/jest", + { + "jsc": { + "parser": { + "syntax": "typescript", + "decorators": true + } + } + } + ] + }, + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" + }, + "reporters": [ + [ + "default", + { + "summaryThreshold": 9999 + } + ] + ], + "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", + "rootDir": "./src" + } +} diff --git a/yarn-project/proof-verifier/src/config.ts b/yarn-project/proof-verifier/src/config.ts new file mode 100644 index 000000000000..fcedde43e4c2 --- /dev/null +++ b/yarn-project/proof-verifier/src/config.ts @@ -0,0 +1,73 @@ +import { EthAddress } from '@aztec/circuits.js'; +import { + type ConfigMappingsType, + booleanConfigHelper, + getConfigFromMappings, + numberConfigHelper, +} from '@aztec/foundation/config'; +import { type TelemetryClientConfig, telemetryClientConfigMappings } from '@aztec/telemetry-client/start'; + +export type ProofVerifierConfig = { + /** The URL to an L1 node */ + l1Url: string; + /** The L1 chain ID */ + l1ChainId: number; + /** Start block number */ + l1StartBlock: number; + /** The address of the Rollup contract */ + rollupAddress: EthAddress; + /** How often to poll L1 for proof submission */ + pollIntervalMs: number; + /** The path to the bb binary */ + bbBinaryPath: string; + /** Where bb stores temporary files */ + bbWorkingDirectory: string; + /** Whether to skip cleanup of bb temporary files */ + bbSkipCleanup: boolean; +} & TelemetryClientConfig; + +export const proofVerifierConfigMappings: ConfigMappingsType = { + ...telemetryClientConfigMappings, + l1Url: { + env: 'ETHEREUM_HOST', + description: 'The URL to an L1 node', + }, + l1ChainId: { + env: 'L1_CHAIN_ID', + parseEnv: (val: string) => +val, + defaultValue: 31337, + description: 'The chain ID of the ethereum host.', + }, + l1StartBlock: { + env: 'PROOF_VERIFIER_L1_START_BLOCK', + description: 'Start block number', + ...numberConfigHelper(1), + }, + rollupAddress: { + env: 'ROLLUP_CONTRACT_ADDRESS', + description: 'The address of the Rollup contract', + parseEnv: EthAddress.fromString, + }, + pollIntervalMs: { + env: 'PROOF_VERIFIER_POLL_INTERVAL_MS', + description: 'How often to poll L1 for proof submission', + ...numberConfigHelper(60_000), + }, + bbBinaryPath: { + env: 'BB_BINARY_PATH', + description: 'The path to the bb binary', + }, + bbWorkingDirectory: { + env: 'BB_WORKING_DIRECTORY', + description: 'Where bb stores temporary files', + }, + bbSkipCleanup: { + env: 'BB_SKIP_CLEANUP', + description: 'Whether to skip cleanup of bb temporary files', + ...booleanConfigHelper(false), + }, +}; + +export function getProofVerifierConfigFromEnv(): ProofVerifierConfig { + return getConfigFromMappings(proofVerifierConfigMappings); +} diff --git a/yarn-project/proof-verifier/src/index.ts b/yarn-project/proof-verifier/src/index.ts new file mode 100644 index 000000000000..4b3ddaa6d8f4 --- /dev/null +++ b/yarn-project/proof-verifier/src/index.ts @@ -0,0 +1,2 @@ +export * from './config.js'; +export * from './proof_verifier.js'; diff --git a/yarn-project/proof-verifier/src/proof_verifier.ts b/yarn-project/proof-verifier/src/proof_verifier.ts new file mode 100644 index 000000000000..265a7e8aac52 --- /dev/null +++ b/yarn-project/proof-verifier/src/proof_verifier.ts @@ -0,0 +1,112 @@ +import { retrieveL2ProofsFromRollup } from '@aztec/archiver/data-retrieval'; +import { BBCircuitVerifier } from '@aztec/bb-prover'; +import { createEthereumChain } from '@aztec/ethereum'; +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { RunningPromise } from '@aztec/foundation/running-promise'; +import { Attributes, Metrics, type TelemetryClient, type UpDownCounter, ValueType } from '@aztec/telemetry-client'; + +import { type PublicClient, createPublicClient, http } from 'viem'; + +import { type ProofVerifierConfig } from './config.js'; + +const EXPECTED_PROOF_SIZE = 13988; + +export class ProofVerifier { + private runningPromise: RunningPromise; + private synchedToL1Block = 0n; + + private proofVerified: UpDownCounter; + + constructor( + private config: ProofVerifierConfig, + private client: PublicClient, + private verifier: BBCircuitVerifier, + telemetryClient: TelemetryClient, + private logger: DebugLogger, + ) { + this.runningPromise = new RunningPromise(this.work.bind(this), config.pollIntervalMs); + this.proofVerified = telemetryClient.getMeter('ProofVerifier').createUpDownCounter(Metrics.PROOF_VERIFIER_COUNT, { + valueType: ValueType.INT, + description: 'The number of proofs verified by the block verifier bot', + }); + this.synchedToL1Block = BigInt(config.l1StartBlock - 1); + } + + static async new(config: ProofVerifierConfig, telemetryClient: TelemetryClient): Promise { + const logger = createDebugLogger('aztec:block-verifier-bot'); + const verifier = await BBCircuitVerifier.new(config, [], logger); + const client = createPublicClient({ + chain: createEthereumChain(config.l1Url, config.l1ChainId).chainInfo, + transport: http(config.l1Url), + }); + + return new ProofVerifier(config, client, verifier, telemetryClient, logger); + } + + start() { + this.logger.info(`Starting proof verifier monitoring rollup=${this.config.rollupAddress}`); + this.runningPromise.start(); + } + + async stop() { + await this.runningPromise.stop(); + } + + private async work() { + const startBlock = this.synchedToL1Block + 1n; + this.logger.debug(`Fetching proofs from L1 block ${startBlock}`); + const { lastProcessedL1BlockNumber, retrievedData } = await retrieveL2ProofsFromRollup( + this.client, + this.config.rollupAddress, + startBlock, + ); + + if (retrievedData.length === 0) { + this.logger.debug(`No proofs found since L1 block ${startBlock}`); + return; + } else { + this.logger.debug(`Fetched ${retrievedData.length} proofs since L1 block ${startBlock}`); + } + + for (const { l2BlockNumber, txHash, proof, proverId } of retrievedData) { + this.logger.debug( + `Proof size ${proof.buffer.length} for L2 block proverId=${proverId} l2Block=${l2BlockNumber} l1Tx=${txHash}`, + ); + + const invalidProofFormat = proof.buffer.length < EXPECTED_PROOF_SIZE; + if (invalidProofFormat) { + this.logger.warn( + `Invalid proof format detected: proof length=${proof.buffer.length}bytes proverId=${proverId} l2Block=${l2BlockNumber} l1Tx=${txHash}`, + ); + } + + try { + await this.verifier.verifyProofForCircuit('RootRollupArtifact', proof); + this.logger.info(`Verified proof for L2 block proverId=${proverId} l2Block=${l2BlockNumber} l1Tx=${txHash}`); + + this.proofVerified.add(1, { + [Attributes.ROLLUP_PROVER_ID]: proverId.toString(), + [Attributes.STATUS]: 'valid', + }); + } catch (err) { + this.logger.warn( + `Failed to verify proof for L2 block proverId=${proverId} l2Block=${l2BlockNumber} l1Tx=${txHash}`, + ); + + if (invalidProofFormat) { + this.proofVerified.add(1, { + [Attributes.ROLLUP_PROVER_ID]: proverId.toString(), + [Attributes.STATUS]: 'invalid_proof_format', + }); + } else { + this.proofVerified.add(1, { + [Attributes.ROLLUP_PROVER_ID]: proverId.toString(), + [Attributes.STATUS]: 'invalid', + }); + } + } + } + + this.synchedToL1Block = lastProcessedL1BlockNumber; + } +} diff --git a/yarn-project/proof-verifier/tsconfig.json b/yarn-project/proof-verifier/tsconfig.json new file mode 100644 index 000000000000..283f554f5701 --- /dev/null +++ b/yarn-project/proof-verifier/tsconfig.json @@ -0,0 +1,35 @@ +{ + "extends": "..", + "compilerOptions": { + "outDir": "dest", + "rootDir": "src", + "tsBuildInfoFile": ".tsbuildinfo" + }, + "references": [ + { + "path": "../archiver" + }, + { + "path": "../bb-prover" + }, + { + "path": "../circuit-types" + }, + { + "path": "../circuits.js" + }, + { + "path": "../ethereum" + }, + { + "path": "../foundation" + }, + { + "path": "../noir-protocol-circuits-types" + }, + { + "path": "../telemetry-client" + } + ], + "include": ["src"] +} diff --git a/yarn-project/sequencer-client/src/publisher/l1-publisher.ts b/yarn-project/sequencer-client/src/publisher/l1-publisher.ts index 5c617ab36240..ad5e091ab34e 100644 --- a/yarn-project/sequencer-client/src/publisher/l1-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/l1-publisher.ts @@ -332,7 +332,7 @@ export class L1Publisher { archive: archiveRoot.toBuffer(), proverId: proverId.toBuffer(), aggregationObject: serializeToBuffer(aggregationObject), - proof: proof.withoutPublicInputs(), + proof: proof.toBuffer(), }; // Process block diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 48d8deda753b..094044f416da 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -71,3 +71,5 @@ export const WORLD_STATE_FORK_DURATION = 'aztec.world_state.fork.duration'; export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration'; export const WORLD_STATE_MERKLE_TREE_SIZE = 'aztec.world_state.merkle_tree_size'; export const WORLD_STATE_DB_SIZE = 'aztec.world_state.db_size'; + +export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; diff --git a/yarn-project/tsconfig.json b/yarn-project/tsconfig.json index 52f60af50fb9..45e18ab81290 100644 --- a/yarn-project/tsconfig.json +++ b/yarn-project/tsconfig.json @@ -53,7 +53,8 @@ { "path": "scripts/tsconfig.json" }, { "path": "entrypoints/tsconfig.json" }, { "path": "cli/tsconfig.json" }, - { "path": "cli-wallet/tsconfig.json" } + { "path": "cli-wallet/tsconfig.json" }, + { "path": "proof-verifier/tsconfig.json" } ], "files": ["./@types/jest/index.d.ts"], "exclude": ["node_modules", "**/node_modules", "**/.*/"] diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 3c960d96f536..cc8bf108dd1f 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -251,6 +251,7 @@ __metadata: "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/p2p": "workspace:^" "@aztec/p2p-bootstrap": "workspace:^" + "@aztec/proof-verifier": "workspace:^" "@aztec/protocol-contracts": "workspace:^" "@aztec/prover-client": "workspace:^" "@aztec/prover-node": "workspace:^" @@ -877,6 +878,28 @@ __metadata: languageName: unknown linkType: soft +"@aztec/proof-verifier@workspace:^, @aztec/proof-verifier@workspace:proof-verifier": + version: 0.0.0-use.local + resolution: "@aztec/proof-verifier@workspace:proof-verifier" + dependencies: + "@aztec/archiver": "workspace:^" + "@aztec/bb-prover": "workspace:^" + "@aztec/circuit-types": "workspace:^" + "@aztec/circuits.js": "workspace:^" + "@aztec/ethereum": "workspace:^" + "@aztec/foundation": "workspace:^" + "@aztec/noir-protocol-circuits-types": "workspace:^" + "@aztec/telemetry-client": "workspace:^" + "@jest/globals": ^29.5.0 + "@types/jest": ^29.5.0 + "@types/node": ^18.7.23 + jest: ^29.5.0 + ts-node: ^10.9.1 + typescript: ^5.0.4 + viem: ^2.7.15 + languageName: unknown + linkType: soft + "@aztec/protocol-contracts@workspace:^, @aztec/protocol-contracts@workspace:protocol-contracts": version: 0.0.0-use.local resolution: "@aztec/protocol-contracts@workspace:protocol-contracts"