diff --git a/.claude/skills/updating-changelog/SKILL.md b/.claude/skills/updating-changelog/SKILL.md index 018859f8833e..1af52b765da4 100644 --- a/.claude/skills/updating-changelog/SKILL.md +++ b/.claude/skills/updating-changelog/SKILL.md @@ -14,7 +14,7 @@ Read `.release-please-manifest.json` to get the version (e.g., `{"." : "4.0.0"}` **Target files:** - Aztec contract developers: `docs/docs-developers/docs/resources/migration_notes.md` -- Node operators and Ethereum contract developers: `docs/docs-network/reference/changelog/v{major}.md` +- Node operators and Ethereum contract developers: `docs/docs-operate/operators/reference/changelog/v{major}.md` ### 2. Analyze Branch Changes @@ -60,7 +60,7 @@ Explanation of what changed. ## Node Operator Changelog Format -**File:** `docs/docs-network/reference/changelog/v{major}.md` +**File:** `docs/docs-operate/operators/reference/changelog/v{major}.md` **Breaking changes:** ````markdown diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index d2b6d6be837c..ec0a4c81b234 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -94,6 +94,7 @@ jobs: PR_COMMITS: ${{ github.event.pull_request.commits }} PR_NUMBER: ${{ github.event.pull_request.number }} GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_ACTOR: ${{ github.actor }} # NOTE: $CI_MODE is set in the Determine CI Mode step. run: ./.github/ci3.sh $CI_MODE diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml index f342710184e5..f9f561f6d88e 100644 --- a/.github/workflows/weekly-proving-bench.yml +++ b/.github/workflows/weekly-proving-bench.yml @@ -47,7 +47,7 @@ jobs: fi - name: Run real proving benchmarks - timeout-minutes: 150 + timeout-minutes: 180 env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -57,7 +57,7 @@ jobs: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 150 + AWS_SHUTDOWN_TIME: 180 NO_SPOT: 1 run: | ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" diff --git a/.test_patterns.yml b/.test_patterns.yml index 18b68fcee5bb..32666e514a5a 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -309,6 +309,11 @@ tests: owners: - *palla + - regex: "src/e2e_block_building.test.ts" + error_regex: "✕ processes txs until hitting timetable" + owners: + - *palla + # http://ci.aztec-labs.com/e8228a36afda93b8 # Test passed but there was an error on stopping - regex: "playground/scripts/run_test.sh" diff --git a/bootstrap.sh b/bootstrap.sh index cb0ca10bdeae..edc7bb5775fa 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -233,11 +233,14 @@ function start_txes { export TOKIO_WORKER_THREADS=1 # Starting txe servers with incrementing port numbers. + # Base port is below the Linux ephemeral range (32768-60999) to avoid conflicts. + local txe_base_port=14730 for i in $(seq 0 $((NUM_TXES-1))); do - port=$((45730 + i)) + port=$((txe_base_port + i)) existing_pid=$(lsof -ti :$port || true) if [ -n "$existing_pid" ]; then echo "Killing existing process $existing_pid on port: $port" + check_port $port kill -9 $existing_pid &>/dev/null || true while kill -0 $existing_pid &>/dev/null; do sleep 0.1; done fi @@ -248,8 +251,12 @@ function start_txes { echo "Waiting for TXE's to start..." for i in $(seq 0 $((NUM_TXES-1))); do local j=0 - while ! nc -z 127.0.0.1 $((45730 + i)) &>/dev/null; do - [ $j == 60 ] && echo_stderr "TXE $i took too long to start. Exiting." && exit 1 + while ! nc -z 127.0.0.1 $((txe_base_port + i)) &>/dev/null; do + if [ $j == 60 ]; then + echo_stderr "TXE $i failed to start on port $((txe_base_port + i)) after 60s." + check_port $((txe_base_port + i)) + exit 1 + fi sleep 1 j=$((j+1)) done diff --git a/ci3/aws_request_instance_type b/ci3/aws_request_instance_type index df835a7cb77e..bf9afbf4c36c 100755 --- a/ci3/aws_request_instance_type +++ b/ci3/aws_request_instance_type @@ -88,6 +88,21 @@ fi echo "Instance id: $iid" +tags="Key=Name,Value=$name Key=Group,Value=build-instance" +[ -n "${GITHUB_ACTOR:-}" ] && tags+=" Key=GithubActor,Value=$GITHUB_ACTOR" +[ -n "${CI_MODE:-}" ] && tags+=" Key=CICommand,Value=$CI_MODE" +[ -n "${CI_DASHBOARD:-}" ] && tags+=" Key=Dashboard,Value=$CI_DASHBOARD" +if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then + echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done." + tags+=" Key=Keep-Alive,Value=true" +fi +aws ec2 create-tags --resources $iid --tags $tags + +# Record the instance type so callers can pass it downstream (e.g. into Docker). +echo $instance_type > $state_dir/instance_type +# Record whether this is spot or on-demand. +[ -f "$sir_path" ] && echo spot > $state_dir/spot || echo ondemand > $state_dir/spot + while [ -z "${ip:-}" ]; do sleep 1 ip=$(aws ec2 describe-instances \ diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index b68f18c014d3..28a0b1c70401 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -89,6 +89,8 @@ if [[ -f "$state_dir/sir" ]]; then sir=$(cat $state_dir/sir) fi iid=$(cat $state_dir/iid) +export EC2_INSTANCE_TYPE=$(cat $state_dir/instance_type 2>/dev/null || echo "unknown") +export EC2_SPOT=$(cat $state_dir/spot 2>/dev/null || echo "unknown") # If AWS credentials are not set, try to load them from ~/.aws/build_instance_credentials. if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then @@ -191,16 +193,6 @@ container_script=$( log_ci_run FAILED \$ci_log_id merge_train_failure_slack_notify \$ci_log_id release_canary_slack_notify \$ci_log_id - ci_failed_data=\$(jq -n \\ - --arg status "failed" \\ - --arg log_id "\$ci_log_id" \\ - --arg ref_name "\${TARGET_BRANCH:-\$REF_NAME}" \\ - --arg commit_hash "\$COMMIT_HASH" \\ - --arg commit_author "\$COMMIT_AUTHOR" \\ - --arg commit_msg "\$COMMIT_MSG" \\ - --argjson exit_code "\$code" \\ - '{status: \$status, log_id: \$log_id, ref_name: \$ref_name, commit_hash: \$commit_hash, commit_author: \$commit_author, commit_msg: \$commit_msg, exit_code: \$exit_code, timestamp: now | todate}') - redis_publish "ci:run:failed" "\$ci_failed_data" ;; esac exit \$code @@ -330,6 +322,9 @@ function run { -e AWS_TOKEN=\$aws_token \ -e NAMESPACE=${NAMESPACE:-} \ -e NETWORK=${NETWORK:-} \ + -e GITHUB_ACTOR=${GITHUB_ACTOR:-} \ + -e EC2_INSTANCE_TYPE=${EC2_INSTANCE_TYPE:-unknown} \ + -e EC2_SPOT=${EC2_SPOT:-unknown} \ --pids-limit=65536 \ --shm-size=2g \ aztecprotocol/devbox:3.0 bash -c $(printf '%q' "$container_script") diff --git a/ci3/check_port b/ci3/check_port new file mode 100755 index 000000000000..a637a8e78405 --- /dev/null +++ b/ci3/check_port @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Check if a port is free. If taken, print the process tree of the holder. +# Usage: check_port +# Exit code: 0 if free, 1 if taken. +set -eu + +port="${1:?Usage: check_port }" + +pid=$(lsof -ti :"$port" 2>/dev/null | head -1 || true) +if [ -z "$pid" ]; then + exit 0 +fi + +echo "Port $port is taken by PID $pid:" >&2 +# Show the command line of the process. +ps -p "$pid" -o pid,ppid,user,args --no-headers >&2 || true +# Show the process tree rooted at this PID. +pstree -apls "$pid" >&2 2>/dev/null || pstree -p "$pid" >&2 2>/dev/null || true +exit 1 diff --git a/ci3/ci-metrics/Dockerfile b/ci3/ci-metrics/Dockerfile new file mode 100644 index 000000000000..4013545da66d --- /dev/null +++ b/ci3/ci-metrics/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12 + +RUN apt update && apt install -y jq redis-tools && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt gunicorn +RUN git config --global --add safe.directory /aztec-packages +COPY . . +EXPOSE 8081 +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "app:app"] diff --git a/ci3/ci-metrics/app.py b/ci3/ci-metrics/app.py new file mode 100644 index 000000000000..c62875e7d19a --- /dev/null +++ b/ci3/ci-metrics/app.py @@ -0,0 +1,848 @@ +from flask import Flask, request, Response, redirect +from flask_compress import Compress +from flask_httpauth import HTTPBasicAuth +from datetime import datetime, timedelta +import json +import os +import re +import redis +import threading +from pathlib import Path + +import db +import metrics +import github_data +import billing.aws as billing_aws +from billing import ( + get_billing_files_in_range, + aggregate_billing_weekly, aggregate_billing_monthly, + serve_billing_dashboard, +) + +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) +LOGS_DISK_PATH = os.getenv('LOGS_DISK_PATH', '/logs-disk') +DASHBOARD_PASSWORD = os.getenv('DASHBOARD_PASSWORD', 'password') + +r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=False) + +app = Flask(__name__) +Compress(app) +auth = HTTPBasicAuth() + + +@auth.verify_password +def verify_password(username, password): + return password == DASHBOARD_PASSWORD + + +def _init(): + """Initialize SQLite and start background threads.""" + try: + db.get_db() + metrics.start_test_listener(r) + metrics.start_ci_run_sync(r) + print("[ci-metrics] Background threads started") + except Exception as e: + print(f"[ci-metrics] Warning: startup failed: {e}") + +threading.Thread(target=_init, daemon=True, name='metrics-init').start() + + +# ---- Helpers ---- + +def _aggregate_dates(by_date_list, granularity, sum_fields, avg_fields=None): + """Aggregate a list of {date, ...} dicts by weekly/monthly granularity.""" + if granularity == 'daily' or not by_date_list: + return by_date_list + + buckets = {} + for entry in by_date_list: + d = datetime.strptime(entry['date'], '%Y-%m-%d') + if granularity == 'weekly': + key = (d - timedelta(days=d.weekday())).strftime('%Y-%m-%d') + else: # monthly + key = d.strftime('%Y-%m') + '-01' + + if key not in buckets: + buckets[key] = {'date': key} + for f in sum_fields: + buckets[key][f] = 0 + if avg_fields: + for f in avg_fields: + buckets[key][f'_avg_sum_{f}'] = 0 + buckets[key][f'_avg_cnt_{f}'] = 0 + + for f in sum_fields: + buckets[key][f] += entry.get(f) or 0 + if avg_fields: + for f in avg_fields: + val = entry.get(f) + if val is not None: + buckets[key][f'_avg_sum_{f}'] += val + buckets[key][f'_avg_cnt_{f}'] += 1 + + result = [] + for key in sorted(buckets): + b = buckets[key] + out = {'date': b['date']} + for f in sum_fields: + out[f] = round(b[f], 2) if isinstance(b[f], float) else b[f] + if avg_fields: + for f in avg_fields: + cnt = b[f'_avg_cnt_{f}'] + out[f] = round(b[f'_avg_sum_{f}'] / cnt, 1) if cnt else None + result.append(out) + + return result + + +def _json(data): + return Response(json.dumps(data), mimetype='application/json') + + +# ---- Namespace billing ---- + +@app.route('/namespace-billing') +@auth.login_required +def namespace_billing(): + html = serve_billing_dashboard() + if html: + return html + return "Billing dashboard not found", 404 + + +@app.route('/api/billing/data') +@auth.login_required +def billing_data(): + date_from_str = request.args.get('from') + date_to_str = request.args.get('to') + granularity = request.args.get('granularity', 'daily') + + if not date_from_str or not date_to_str: + return _json({'error': 'from and to date params required (YYYY-MM-DD)'}), 400 + try: + date_from = datetime.strptime(date_from_str, '%Y-%m-%d') + date_to = datetime.strptime(date_to_str, '%Y-%m-%d') + except ValueError: + return _json({'error': 'Invalid date format, use YYYY-MM-DD'}), 400 + + daily_data = get_billing_files_in_range(date_from, date_to) + + # Filter out namespaces costing less than $1 total across the range + ns_totals = {} + for entry in daily_data: + for ns, ns_data in entry.get('namespaces', {}).items(): + ns_totals[ns] = ns_totals.get(ns, 0) + ns_data.get('total', 0) + cheap_ns = {ns for ns, total in ns_totals.items() if total < 1.0} + if cheap_ns: + for entry in daily_data: + entry['namespaces'] = {ns: d for ns, d in entry.get('namespaces', {}).items() + if ns not in cheap_ns} + + if granularity == 'weekly': + result = aggregate_billing_weekly(daily_data) + elif granularity == 'monthly': + result = aggregate_billing_monthly(daily_data) + else: + result = daily_data + + return _json(result) + + +# ---- CI runs ---- + +@app.route('/api/ci/runs') +@auth.login_required +def api_ci_runs(): + date_from = request.args.get('from', '') + date_to = request.args.get('to', '') + status_filter = request.args.get('status', '') + author = request.args.get('author', '') + dashboard = request.args.get('dashboard', '') + limit = min(int(request.args.get('limit', 100)), 1000) + offset = int(request.args.get('offset', 0)) + + ts_from = int(datetime.strptime(date_from, '%Y-%m-%d').timestamp() * 1000) if date_from else None + ts_to = int((datetime.strptime(date_to, '%Y-%m-%d') + timedelta(days=1)).timestamp() * 1000) if date_to else None + + runs = metrics.get_ci_runs(r, ts_from, ts_to) + + if status_filter: + runs = [run for run in runs if run.get('status') == status_filter] + if author: + runs = [run for run in runs if run.get('author') == author] + if dashboard: + runs = [run for run in runs if run.get('dashboard') == dashboard] + + runs.sort(key=lambda x: x.get('timestamp', 0), reverse=True) + runs = runs[offset:offset + limit] + + return _json(runs) + + +@app.route('/api/ci/stats') +@auth.login_required +def api_ci_stats(): + ts_from = int((datetime.now() - timedelta(days=7)).timestamp() * 1000) + runs = metrics.get_ci_runs(r, ts_from) + + total = len(runs) + passed = sum(1 for run in runs if run.get('status') == 'PASSED') + failed = sum(1 for run in runs if run.get('status') == 'FAILED') + costs = [run['cost_usd'] for run in runs if run.get('cost_usd') is not None] + durations = [] + for run in runs: + complete = run.get('complete') + ts = run.get('timestamp') + if complete and ts: + durations.append((complete - ts) / 60000.0) + + return _json({ + 'total_runs': total, + 'passed': passed, + 'failed': failed, + 'total_cost': round(sum(costs), 2) if costs else None, + 'avg_duration_mins': round(sum(durations) / len(durations), 1) if durations else None, + }) + + +# ---- Cost endpoints ---- + +@app.route('/api/costs/overview') +@auth.login_required +def api_costs_overview(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + granularity = request.args.get('granularity', 'daily') + result = billing_aws.get_costs_overview(date_from, date_to) + if granularity != 'daily' and result.get('by_date'): + buckets = {} + for entry in result['by_date']: + d = datetime.strptime(entry['date'], '%Y-%m-%d') + if granularity == 'weekly': + key = (d - timedelta(days=d.weekday())).strftime('%Y-%m-%d') + else: + key = d.strftime('%Y-%m') + '-01' + if key not in buckets: + buckets[key] = {'date': key, 'aws': {}, 'gcp': {}, 'aws_total': 0, 'gcp_total': 0} + for cat, amt in entry.get('aws', {}).items(): + buckets[key]['aws'][cat] = buckets[key]['aws'].get(cat, 0) + amt + for cat, amt in entry.get('gcp', {}).items(): + buckets[key]['gcp'][cat] = buckets[key]['gcp'].get(cat, 0) + amt + buckets[key]['aws_total'] += entry.get('aws_total', 0) + buckets[key]['gcp_total'] += entry.get('gcp_total', 0) + result['by_date'] = sorted(buckets.values(), key=lambda x: x['date']) + return _json(result) + + +@app.route('/api/costs/details') +@auth.login_required +def api_costs_details(): + """Per-resource (USAGE_TYPE) cost breakdown.""" + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + + rows = billing_aws.get_aws_cost_details(date_from, date_to) + + usage_map = {} + for row in rows: + ut = row['usage_type'] + if ut not in usage_map: + usage_map[ut] = { + 'usage_type': ut, + 'service': row['service'], + 'category': row['category'], + 'total': 0, + 'by_date': {}, + 'is_ri': 'HeavyUsage' in ut, + } + usage_map[ut]['total'] += row['amount_usd'] + d = row['date'] + usage_map[ut]['by_date'][d] = usage_map[ut]['by_date'].get(d, 0) + row['amount_usd'] + + items = sorted(usage_map.values(), key=lambda x: -x['total']) + for item in items: + item['total'] = round(item['total'], 2) + item['by_date'] = {d: round(v, 4) for d, v in sorted(item['by_date'].items())} + + all_dates = sorted({row['date'] for row in rows}) + ri_items = [i for i in items if i['is_ri']] + ri_total = round(sum(i['total'] for i in ri_items), 2) + + return _json({ + 'items': items, + 'dates': all_dates, + 'ri_total': ri_total, + 'grand_total': round(sum(i['total'] for i in items), 2), + }) + + +@app.route('/api/costs/attribution') +@auth.login_required +def api_costs_attribution(): + """CI cost attribution by user, branch, instance.""" + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + ts_from = int(datetime.strptime(date_from, '%Y-%m-%d').timestamp() * 1000) + ts_to = int((datetime.strptime(date_to, '%Y-%m-%d') + timedelta(days=1)).timestamp() * 1000) + + runs = metrics.get_ci_runs(r, ts_from, ts_to) + runs_with_cost = [run for run in runs if run.get('cost_usd') is not None] + + # Enrich merge queue runs with PR author from GitHub + pr_numbers = {run.get('pr_number') for run in runs_with_cost if run.get('pr_number')} + pr_authors = github_data.batch_get_pr_authors(pr_numbers) + + granularity = request.args.get('granularity', 'daily') + + instances = [] + by_user = {} + by_branch = {} + by_type = {} + by_date_type = {} + + for run in runs_with_cost: + info = billing_aws.decode_branch_info(run) + cost = run['cost_usd'] + date = metrics._ts_to_date(run.get('timestamp', 0)) + + author = info['author'] + prn = info['pr_number'] + if prn and int(prn) in pr_authors: + author = pr_authors[int(prn)]['author'] + + inst_type = run.get('instance_type', 'unknown') + vcpus = run.get('instance_vcpus') + if inst_type == 'unknown' and vcpus: + inst_type = f'{vcpus}vcpu' + + instances.append({ + 'instance_name': info['instance_name'], + 'date': date, + 'cost_usd': cost, + 'author': author, + 'branch': info['branch'], + 'pr_number': prn, + 'type': info['type'], + 'instance_type': inst_type, + 'spot': run.get('spot', False), + 'job_id': run.get('job_id', ''), + 'duration_mins': round((run.get('complete', 0) - run.get('timestamp', 0)) / 60000, 1) if run.get('complete') else None, + }) + + if author not in by_user: + by_user[author] = {'aws_cost': 0, 'gcp_cost': 0, 'runs': 0, 'by_date': {}} + by_user[author]['aws_cost'] += cost + by_user[author]['runs'] += 1 + by_user[author]['by_date'][date] = by_user[author]['by_date'].get(date, 0) + cost + + branch_key = info['branch'] or info['type'] + if branch_key not in by_branch: + by_branch[branch_key] = {'cost': 0, 'runs': 0, 'type': info['type'], 'author': author} + by_branch[branch_key]['cost'] += cost + by_branch[branch_key]['runs'] += 1 + + rt = info['type'] + if rt not in by_type: + by_type[rt] = {'cost': 0, 'runs': 0} + by_type[rt]['cost'] += cost + by_type[rt]['runs'] += 1 + + if date not in by_date_type: + by_date_type[date] = {} + by_date_type[date][rt] = by_date_type[date].get(rt, 0) + cost + + # GCP costs — reported as total, no namespace→user heuristic + gcp_total = 0 + try: + from billing.gcp import get_billing_files_in_range as get_gcp_billing + gcp_data = get_gcp_billing( + datetime.strptime(date_from, '%Y-%m-%d'), + datetime.strptime(date_to, '%Y-%m-%d'), + ) + for entry in gcp_data: + for ns, ns_data in entry.get('namespaces', {}).items(): + gcp_total += ns_data.get('total', 0) + except Exception as e: + print(f"[attribution] GKE billing error: {e}") + + # Sort and format + user_list = [{'author': a, 'aws_cost': round(v['aws_cost'], 2), 'gcp_cost': round(v['gcp_cost'], 2), + 'total_cost': round(v['aws_cost'] + v['gcp_cost'], 2), 'runs': v['runs'], + 'by_date': {d: round(c, 2) for d, c in sorted(v['by_date'].items())}} + for a, v in sorted(by_user.items(), key=lambda x: -(x[1]['aws_cost'] + x[1]['gcp_cost']))] + + branch_list = [{'branch': b, 'cost': round(v['cost'], 2), 'runs': v['runs'], + 'type': v['type'], 'author': v['author']} + for b, v in sorted(by_branch.items(), key=lambda x: -x[1]['cost'])[:100]] + + type_list = [{'type': t, 'cost': round(v['cost'], 2), 'runs': v['runs']} + for t, v in sorted(by_type.items(), key=lambda x: -x[1]['cost'])] + + instances.sort(key=lambda x: -(x['cost_usd'] or 0)) + + all_types = sorted(by_type.keys()) + by_date_list = [] + for date in sorted(by_date_type): + entry = {'date': date, 'total': 0, 'runs': 0} + for rt in all_types: + entry[rt] = round(by_date_type[date].get(rt, 0), 2) + entry['total'] += by_date_type[date].get(rt, 0) + entry['total'] = round(entry['total'], 2) + entry['runs'] = sum(1 for inst in instances if inst['date'] == date) + by_date_list.append(entry) + + by_date_list = _aggregate_dates(by_date_list, granularity, + sum_fields=['total', 'runs'] + all_types) + + total_aws = sum(u['aws_cost'] for u in user_list) + + return _json({ + 'by_user': user_list, + 'by_branch': branch_list, + 'by_type': type_list, + 'by_date': by_date_list, + 'run_types': all_types, + 'instances': instances[:500], + 'totals': {'aws': round(total_aws, 2), 'gcp': round(gcp_total, 2), + 'gcp_unattributed': round(gcp_total, 2), + 'combined': round(total_aws + gcp_total, 2)}, + }) + + +@app.route('/api/costs/runners') +@auth.login_required +def api_costs_runners(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + granularity = request.args.get('granularity', 'daily') + dashboard = request.args.get('dashboard', '') + ts_from = int(datetime.strptime(date_from, '%Y-%m-%d').timestamp() * 1000) + ts_to = int((datetime.strptime(date_to, '%Y-%m-%d') + timedelta(days=1)).timestamp() * 1000) + + runs = metrics.get_ci_runs(r, ts_from, ts_to) + runs_with_cost = [run for run in runs if run.get('cost_usd') is not None] + if dashboard: + runs_with_cost = [run for run in runs_with_cost if run.get('dashboard') == dashboard] + + by_date_map = {} + for run in runs_with_cost: + date = metrics._ts_to_date(run.get('timestamp', 0)) + if date not in by_date_map: + by_date_map[date] = {'spot_cost': 0, 'ondemand_cost': 0, 'total': 0} + cost = run['cost_usd'] + if run.get('spot'): + by_date_map[date]['spot_cost'] += cost + else: + by_date_map[date]['ondemand_cost'] += cost + by_date_map[date]['total'] += cost + + by_date = [{'date': date, 'spot_cost': round(d['spot_cost'], 2), + 'ondemand_cost': round(d['ondemand_cost'], 2), 'total': round(d['total'], 2), + 'spot_pct': round(100.0 * d['spot_cost'] / max(d['total'], 0.01), 1)} + for date, d in sorted(by_date_map.items())] + + by_date = _aggregate_dates(by_date, granularity, + sum_fields=['spot_cost', 'ondemand_cost', 'total']) + for d in by_date: + d['spot_pct'] = round(100.0 * d['spot_cost'] / max(d['total'], 0.01), 1) + + by_instance_map = {} + for run in runs_with_cost: + inst = run.get('instance_type', 'unknown') + if inst not in by_instance_map: + by_instance_map[inst] = {'cost': 0, 'runs': 0} + by_instance_map[inst]['cost'] += run['cost_usd'] + by_instance_map[inst]['runs'] += 1 + by_instance = [{'instance_type': k, 'cost': round(v['cost'], 2), 'runs': v['runs']} + for k, v in sorted(by_instance_map.items(), key=lambda x: -x[1]['cost'])] + + by_dash_map = {} + for run in runs_with_cost: + dash = run.get('dashboard', 'unknown') + if dash not in by_dash_map: + by_dash_map[dash] = {'cost': 0, 'runs': 0} + by_dash_map[dash]['cost'] += run['cost_usd'] + by_dash_map[dash]['runs'] += 1 + by_dashboard = [{'dashboard': k, 'cost': round(v['cost'], 2), 'runs': v['runs']} + for k, v in sorted(by_dash_map.items(), key=lambda x: -x[1]['cost'])] + + total_cost = sum(run['cost_usd'] for run in runs_with_cost) + spot_cost = sum(run['cost_usd'] for run in runs_with_cost if run.get('spot')) + + return _json({ + 'by_date': by_date, + 'by_instance_type': by_instance, + 'by_dashboard': by_dashboard, + 'summary': { + 'total_cost': round(total_cost, 2), + 'spot_pct': round(100.0 * spot_cost / max(total_cost, 0.01), 1), + 'avg_cost_per_run': round(total_cost / max(len(runs_with_cost), 1), 2), + 'total_runs': len(runs_with_cost), + }, + }) + + +# ---- CI Performance ---- + +@app.route('/api/ci/performance') +@auth.login_required +def api_ci_performance(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + dashboard = request.args.get('dashboard', '') + granularity = request.args.get('granularity', 'daily') + ts_from = int(datetime.strptime(date_from, '%Y-%m-%d').timestamp() * 1000) + ts_to = int((datetime.strptime(date_to, '%Y-%m-%d') + timedelta(days=1)).timestamp() * 1000) + + runs = metrics.get_ci_runs(r, ts_from, ts_to) + runs = [run for run in runs if run.get('status') in ('PASSED', 'FAILED')] + if dashboard: + runs = [run for run in runs if run.get('dashboard') == dashboard] + + by_date_map = {} + for run in runs: + date = metrics._ts_to_date(run.get('timestamp', 0)) + if date not in by_date_map: + by_date_map[date] = {'total': 0, 'passed': 0, 'failed': 0, 'durations': []} + by_date_map[date]['total'] += 1 + if run.get('status') == 'PASSED': + by_date_map[date]['passed'] += 1 + else: + by_date_map[date]['failed'] += 1 + complete = run.get('complete') + ts = run.get('timestamp') + if complete and ts: + by_date_map[date]['durations'].append((complete - ts) / 60000.0) + + by_date = [] + for date in sorted(by_date_map): + d = by_date_map[date] + by_date.append({ + 'date': date, + 'total': d['total'], + 'passed': d['passed'], + 'failed': d['failed'], + 'pass_rate': round(100.0 * d['passed'] / max(d['total'], 1), 1), + 'failure_rate': round(100.0 * d['failed'] / max(d['total'], 1), 1), + 'avg_duration_mins': round(sum(d['durations']) / len(d['durations']), 1) if d['durations'] else None, + }) + + by_date = _aggregate_dates(by_date, granularity, + sum_fields=['total', 'passed', 'failed'], + avg_fields=['avg_duration_mins']) + for d in by_date: + d['pass_rate'] = round(100.0 * d['passed'] / max(d['total'], 1), 1) + d['failure_rate'] = round(100.0 * d['failed'] / max(d['total'], 1), 1) + + # Daily flake/failure counts from test_events + if dashboard: + flake_daily = db.query(''' + SELECT substr(timestamp, 1, 10) as date, COUNT(*) as count + FROM test_events WHERE status = 'flaked' AND dashboard = ? + AND timestamp >= ? AND timestamp < ? + GROUP BY substr(timestamp, 1, 10) + ''', (dashboard, date_from, date_to + 'T23:59:59')) + fail_test_daily = db.query(''' + SELECT substr(timestamp, 1, 10) as date, COUNT(*) as count + FROM test_events WHERE status = 'failed' AND dashboard = ? + AND timestamp >= ? AND timestamp < ? + GROUP BY substr(timestamp, 1, 10) + ''', (dashboard, date_from, date_to + 'T23:59:59')) + else: + flake_daily = db.query(''' + SELECT substr(timestamp, 1, 10) as date, COUNT(*) as count + FROM test_events WHERE status = 'flaked' + AND timestamp >= ? AND timestamp < ? + GROUP BY substr(timestamp, 1, 10) + ''', (date_from, date_to + 'T23:59:59')) + fail_test_daily = db.query(''' + SELECT substr(timestamp, 1, 10) as date, COUNT(*) as count + FROM test_events WHERE status = 'failed' + AND timestamp >= ? AND timestamp < ? + GROUP BY substr(timestamp, 1, 10) + ''', (date_from, date_to + 'T23:59:59')) + flake_daily_map = {r['date']: r['count'] for r in flake_daily} + fail_test_daily_map = {r['date']: r['count'] for r in fail_test_daily} + for d in by_date: + d['flake_count'] = flake_daily_map.get(d['date'], 0) + d['test_failure_count'] = fail_test_daily_map.get(d['date'], 0) + + # Top flakes/failures + if dashboard: + top_flakes = db.query(''' + SELECT test_cmd, COUNT(*) as count, ref_name + FROM test_events WHERE status='flaked' AND dashboard = ? + AND timestamp >= ? AND timestamp <= ? + GROUP BY test_cmd ORDER BY count DESC LIMIT 15 + ''', (dashboard, date_from, date_to + 'T23:59:59')) + top_failures = db.query(''' + SELECT test_cmd, COUNT(*) as count + FROM test_events WHERE status='failed' AND dashboard = ? + AND timestamp >= ? AND timestamp <= ? + GROUP BY test_cmd ORDER BY count DESC LIMIT 15 + ''', (dashboard, date_from, date_to + 'T23:59:59')) + else: + top_flakes = db.query(''' + SELECT test_cmd, COUNT(*) as count, ref_name + FROM test_events WHERE status='flaked' AND timestamp >= ? AND timestamp <= ? + GROUP BY test_cmd ORDER BY count DESC LIMIT 15 + ''', (date_from, date_to + 'T23:59:59')) + top_failures = db.query(''' + SELECT test_cmd, COUNT(*) as count + FROM test_events WHERE status='failed' AND timestamp >= ? AND timestamp <= ? + GROUP BY test_cmd ORDER BY count DESC LIMIT 15 + ''', (date_from, date_to + 'T23:59:59')) + + # Summary + total = len(runs) + passed = sum(1 for run in runs if run.get('status') == 'PASSED') + failed = total - passed + durations = [] + for run in runs: + complete = run.get('complete') + ts = run.get('timestamp') + if complete and ts: + durations.append((complete - ts) / 60000.0) + + if dashboard: + flake_count = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status='flaked' AND dashboard = ? + AND timestamp >= ? AND timestamp <= ? + ''', (dashboard, date_from, date_to + 'T23:59:59')) + total_tests = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status IN ('failed','flaked') AND dashboard = ? + AND timestamp >= ? AND timestamp <= ? + ''', (dashboard, date_from, date_to + 'T23:59:59')) + total_failures_count = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status='failed' AND dashboard = ? + AND timestamp >= ? AND timestamp <= ? + ''', (dashboard, date_from, date_to + 'T23:59:59')) + else: + flake_count = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status='flaked' AND timestamp >= ? AND timestamp <= ? + ''', (date_from, date_to + 'T23:59:59')) + total_tests = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status IN ('failed','flaked') AND timestamp >= ? AND timestamp <= ? + ''', (date_from, date_to + 'T23:59:59')) + total_failures_count = db.query(''' + SELECT COUNT(*) as c FROM test_events WHERE status='failed' AND timestamp >= ? AND timestamp <= ? + ''', (date_from, date_to + 'T23:59:59')) + + fc = flake_count[0]['c'] if flake_count else 0 + tc = total_tests[0]['c'] if total_tests else 0 + tfc = total_failures_count[0]['c'] if total_failures_count else 0 + + return _json({ + 'by_date': by_date, + 'top_flakes': top_flakes, + 'top_failures': top_failures, + 'summary': { + 'total_runs': total, + 'pass_rate': round(100.0 * passed / max(total, 1), 1), + 'failure_rate': round(100.0 * failed / max(total, 1), 1), + 'avg_duration_mins': round(sum(durations) / len(durations), 1) if durations else None, + 'flake_rate': round(100.0 * fc / max(tc, 1), 1) if tc else 0, + 'total_flakes': fc, + 'total_test_failures': tfc, + }, + }) + + +# ---- GitHub integration ---- + +@app.route('/api/deployments/speed') +@auth.login_required +def api_deploy_speed(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + workflow = request.args.get('workflow', '') + granularity = request.args.get('granularity', 'daily') + result = github_data.get_deployment_speed(date_from, date_to, workflow) + if granularity != 'daily' and result.get('by_date'): + result['by_date'] = _aggregate_dates( + result['by_date'], granularity, + sum_fields=['count', 'success', 'failure'], + avg_fields=['median_mins', 'p95_mins']) + return _json(result) + + +@app.route('/api/branches/lag') +@auth.login_required +def api_branch_lag(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + return _json(github_data.get_branch_lag(date_from, date_to)) + + +@app.route('/api/prs/metrics') +@auth.login_required +def api_pr_metrics(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + author = request.args.get('author', '') + ts_from = int(datetime.strptime(date_from, '%Y-%m-%d').timestamp() * 1000) + ts_to = int((datetime.strptime(date_to, '%Y-%m-%d') + timedelta(days=1)).timestamp() * 1000) + ci_runs = metrics.get_ci_runs(r, ts_from, ts_to) + return _json(github_data.get_pr_metrics(date_from, date_to, author, ci_runs)) + + +@app.route('/api/merge-queue/stats') +@auth.login_required +def api_merge_queue_stats(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + return _json(github_data.get_merge_queue_stats(date_from, date_to)) + + +@app.route('/api/ci/flakes-by-command') +@auth.login_required +def api_flakes_by_command(): + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + dashboard = request.args.get('dashboard', '') + metrics.sync_failed_tests_to_sqlite(r) + return _json(metrics.get_flakes_by_command(date_from, date_to, dashboard)) + + +# ---- Test timings ---- + +@app.route('/api/tests/timings') +@auth.login_required +def api_test_timings(): + """Test timing statistics: duration by test command, with trends.""" + date_from = request.args.get('from', (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')) + date_to = request.args.get('to', datetime.now().strftime('%Y-%m-%d')) + dashboard = request.args.get('dashboard', '') + status = request.args.get('status', '') # filter to specific status + test_cmd = request.args.get('test_cmd', '') # filter to specific test + + conditions = ['duration_secs IS NOT NULL', 'duration_secs > 0', + 'timestamp >= ?', "timestamp < ? || 'T23:59:59'"] + params = [date_from, date_to] + + if dashboard: + conditions.append('dashboard = ?') + params.append(dashboard) + if status: + conditions.append('status = ?') + params.append(status) + if test_cmd: + conditions.append('test_cmd = ?') + params.append(test_cmd) + + where = 'WHERE ' + ' AND '.join(conditions) + + # Per-test stats + by_test = db.query(f''' + SELECT test_cmd, + COUNT(*) as count, + ROUND(AVG(duration_secs), 1) as avg_secs, + ROUND(MIN(duration_secs), 1) as min_secs, + ROUND(MAX(duration_secs), 1) as max_secs, + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) as passed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed, + SUM(CASE WHEN status = 'flaked' THEN 1 ELSE 0 END) as flaked, + dashboard + FROM test_events {where} + GROUP BY test_cmd + ORDER BY count DESC + LIMIT 200 + ''', params) + + # Add pass rate + for row in by_test: + total = row['passed'] + row['failed'] + row['flaked'] + row['pass_rate'] = round(100.0 * row['passed'] / max(total, 1), 1) + row['total_time_secs'] = round(row['avg_secs'] * row['count'], 0) + + # Daily time series (aggregate across all tests or filtered test) + by_date = db.query(f''' + SELECT substr(timestamp, 1, 10) as date, + COUNT(*) as count, + ROUND(AVG(duration_secs), 1) as avg_secs, + ROUND(MAX(duration_secs), 1) as max_secs, + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) as passed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed, + SUM(CASE WHEN status = 'flaked' THEN 1 ELSE 0 END) as flaked + FROM test_events {where} + GROUP BY substr(timestamp, 1, 10) + ORDER BY date + ''', params) + + # Summary + summary_rows = db.query(f''' + SELECT COUNT(*) as count, + ROUND(AVG(duration_secs), 1) as avg_secs, + ROUND(MAX(duration_secs), 1) as max_secs, + SUM(duration_secs) as total_secs, + SUM(CASE WHEN status = 'passed' THEN 1 ELSE 0 END) as passed, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed, + SUM(CASE WHEN status = 'flaked' THEN 1 ELSE 0 END) as flaked + FROM test_events {where} + ''', params) + s = summary_rows[0] if summary_rows else {} + + # Slowest individual test runs + slowest = db.query(f''' + SELECT test_cmd, status, duration_secs, dashboard, + substr(timestamp, 1, 10) as date, commit_author, log_url + FROM test_events {where} + ORDER BY duration_secs DESC + LIMIT 50 + ''', params) + + return _json({ + 'by_test': by_test, + 'by_date': by_date, + 'slowest': slowest, + 'summary': { + 'total_runs': s.get('count', 0), + 'avg_duration_secs': s.get('avg_secs'), + 'max_duration_secs': s.get('max_secs'), + 'total_compute_secs': round(s.get('total_secs', 0) or 0, 0), + 'passed': s.get('passed', 0), + 'failed': s.get('failed', 0), + 'flaked': s.get('flaked', 0), + }, + }) + + +# ---- Dashboard views ---- + +@app.route('/ci-health') +@auth.login_required +def ci_health(): + return redirect('/ci-insights') + + +@app.route('/ci-insights') +@auth.login_required +def ci_insights(): + path = Path(__file__).parent / 'views' / 'ci-insights.html' + if path.exists(): + return path.read_text() + return "Dashboard not found", 404 + + +@app.route('/cost-overview') +@auth.login_required +def cost_overview(): + path = Path(__file__).parent / 'views' / 'cost-overview.html' + if path.exists(): + return path.read_text() + return "Dashboard not found", 404 + + +@app.route('/test-timings') +@auth.login_required +def test_timings(): + path = Path(__file__).parent / 'views' / 'test-timings.html' + if path.exists(): + return path.read_text() + return "Dashboard not found", 404 + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8081) diff --git a/ci3/ci-metrics/billing/__init__.py b/ci3/ci-metrics/billing/__init__.py new file mode 100644 index 000000000000..e097751047c2 --- /dev/null +++ b/ci3/ci-metrics/billing/__init__.py @@ -0,0 +1,14 @@ +"""Billing package: GKE namespace billing and AWS cost data.""" + +from billing.gcp import ( + get_billing_files_in_range, + aggregate_billing_weekly, + aggregate_billing_monthly, + serve_billing_dashboard, +) +from billing.aws import ( + get_costs_overview, + get_aws_cost_details, + decode_branch_info, + decode_instance_name, +) diff --git a/ci3/ci-metrics/billing/aws.py b/ci3/ci-metrics/billing/aws.py new file mode 100644 index 000000000000..481393d74ec3 --- /dev/null +++ b/ci3/ci-metrics/billing/aws.py @@ -0,0 +1,347 @@ +"""AWS Cost Explorer fetch with in-memory cache. + +Fetches on first request, caches for 6 hours. No SQLite, no background threads. +""" +import threading +import time +from datetime import datetime, timedelta, timezone + +SERVICE_CATEGORY_MAP = { + # Compute + 'Amazon Elastic Compute Cloud - Compute': 'ec2', + 'EC2 - Other': 'ec2', # EBS volumes, snapshots, NAT gateways, data transfer + 'Amazon Elastic Container Service': 'ecs', + 'Amazon Elastic Kubernetes Service': 'eks', + 'Amazon EC2 Container Registry (ECR)': 'ecr', + 'AWS Lambda': 'lambda', + 'Amazon Lightsail': 'lightsail', + # Storage + 'Amazon Simple Storage Service': 's3', + 'Amazon Elastic File System': 'efs', + 'Amazon Elastic Block Store': 'ebs', + 'Amazon ElastiCache': 'elasticache', + 'Amazon Relational Database Service': 'rds', + 'Amazon DynamoDB': 'dynamodb', + 'AWS Backup': 'backup', + # Networking + 'Amazon CloudFront': 'cloudfront', + 'CloudFront Flat-Rate Plans': 'cloudfront', + 'Amazon Virtual Private Cloud': 'vpc', + 'Elastic Load Balancing': 'elb', + 'Amazon Elastic Load Balancing': 'elb', + 'Amazon Route 53': 'route53', + 'Amazon API Gateway': 'apigateway', + 'AWS Data Transfer': 'data_transfer', + 'AWS Global Accelerator': 'global_accelerator', + # Monitoring & Security + 'AmazonCloudWatch': 'cloudwatch', + 'AWS CloudTrail': 'cloudtrail', + 'AWS Secrets Manager': 'secrets', + 'AWS Key Management Service': 'kms', + 'AWS WAF': 'waf', + 'AWS Config': 'config', + 'AWS Certificate Manager': 'acm', + # CI/CD & Dev Tools + 'AWS CodeBuild': 'codebuild', + 'AWS CodePipeline': 'codepipeline', + 'AWS CloudFormation': 'cloudformation', + 'AWS Amplify': 'amplify', + # Data & Analytics + 'AWS Glue': 'glue', + # IoT + 'AWS IoT': 'iot', + 'Amazon Location Service': 'location', + # Messaging + 'Amazon Simple Notification Service': 'sns', + 'Amazon Simple Queue Service': 'sqs', + # Other + 'Tax': 'tax', + 'AWS Support (Business)': 'support', + 'AWS Support (Enterprise)': 'support', + 'AWS Cost Explorer': 'cost_explorer', +} + +import re + +_cache = {'rows': [], 'ts': 0} +_cache_lock = threading.Lock() +_detail_cache = {'rows': [], 'ts': 0} +_detail_cache_lock = threading.Lock() +_CACHE_TTL = 6 * 3600 + +# Known job postfixes from ci.sh (these become INSTANCE_POSTFIX) +_JOB_POSTFIXES = re.compile( + r'_(x[0-9]+-(?:full|fast)|a[0-9]+-(?:full|fast)|n-deploy-[0-9]+|grind-test-[a-f0-9]+)$' +) +_ARCH_SUFFIXES = ('_amd64', '_arm64', '_x86_64', '_aarch64') + + +def decode_instance_name(run: dict) -> str: + """Reconstruct the EC2 instance name from CI run metadata. + + bootstrap_ec2 naming: + merge queue: pr-{number}_{arch}[_{postfix}] + branch: {sanitized_branch}_{arch}[_{postfix}] + """ + name = run.get('name', '') + pr = run.get('pr_number') + arch = run.get('arch', 'amd64') + # Normalize arch names + if arch in ('x86_64', 'amd64'): + arch = 'amd64' + elif arch in ('aarch64', 'arm64'): + arch = 'arm64' + job = run.get('job_id', '') + + if '(queue)' in name and pr: + base = f'pr-{pr}_{arch}' + elif pr: + base = f'pr-{pr}_{arch}' + else: + # Replicate: echo -n "$REF_NAME" | head -c 50 | tr -c 'a-zA-Z0-9-' '_' + sanitized = re.sub(r'[^a-zA-Z0-9-]', '_', name[:50]) + base = f'{sanitized}_{arch}' + if job: + return f'{base}_{job}' + return base + + +def decode_branch_info(run: dict) -> dict: + """Extract branch/PR/user context from a CI run.""" + name = run.get('name', '') + dashboard = run.get('dashboard', '') + pr = run.get('pr_number') + author = run.get('author', 'unknown') + + if '(queue)' in name or dashboard == 'next': + run_type = 'merge-queue' + branch = name.replace(' (queue)', '') + elif dashboard == 'prs': + run_type = 'pr' + branch = name + elif dashboard in ('nightly', 'releases', 'network', 'deflake'): + run_type = dashboard + branch = name + else: + run_type = 'other' + branch = name + + return { + 'type': run_type, + 'branch': branch, + 'pr_number': pr, + 'author': author, + 'instance_name': decode_instance_name(run), + } + + +def _fetch_aws_costs(date_from: str, date_to: str) -> list[dict]: + try: + import boto3 + except ImportError: + print("[rk_aws_costs] boto3 not installed, skipping") + return [] + + try: + client = boto3.client('ce', region_name='us-east-2') + rows = [] + next_token = None + + while True: + kwargs = dict( + TimePeriod={'Start': date_from, 'End': date_to}, + Granularity='DAILY', + Metrics=['UnblendedCost'], + GroupBy=[{'Type': 'DIMENSION', 'Key': 'SERVICE'}], + ) + if next_token: + kwargs['NextPageToken'] = next_token + + response = client.get_cost_and_usage(**kwargs) + + for result in response['ResultsByTime']: + date = result['TimePeriod']['Start'] + for group in result['Groups']: + service = group['Keys'][0] + amount = float(group['Metrics']['UnblendedCost']['Amount']) + if amount == 0: + continue + category = SERVICE_CATEGORY_MAP.get(service, 'other') + if category == 'other': + print(f"[rk_aws_costs] unmapped service: {service!r} (${amount:.2f})") + rows.append({ + 'date': date, + 'service': service, + 'category': category, + 'amount_usd': round(amount, 4), + }) + + next_token = response.get('NextPageToken') + if not next_token: + break + + return rows + except Exception as e: + print(f"[rk_aws_costs] Error: {e}") + return [] + + +def _ensure_cached(): + now = time.time() + if _cache['rows'] and now - _cache['ts'] < _CACHE_TTL: + return + if not _cache_lock.acquire(blocking=False): + return + try: + today = datetime.now(timezone.utc).date() + rows = _fetch_aws_costs( + (today - timedelta(days=365)).isoformat(), + today.isoformat(), + ) + if rows: + _cache['rows'] = rows + _cache['ts'] = now + finally: + _cache_lock.release() + + +def get_aws_costs(date_from: str, date_to: str) -> list[dict]: + """Get AWS costs for date range. Blocks on first fetch, async refresh after.""" + if not _cache['rows']: + _ensure_cached() # block on first load so dashboard isn't empty + else: + threading.Thread(target=_ensure_cached, daemon=True).start() + return [r for r in _cache['rows'] if date_from <= r['date'] <= date_to] + + +def _fetch_aws_cost_details(date_from: str, date_to: str) -> list[dict]: + """Fetch per-resource (USAGE_TYPE) cost breakdown from AWS Cost Explorer.""" + try: + import boto3 + except ImportError: + return [] + + try: + client = boto3.client('ce', region_name='us-east-2') + rows = [] + next_token = None + + while True: + kwargs = dict( + TimePeriod={'Start': date_from, 'End': date_to}, + Granularity='DAILY', + Metrics=['UnblendedCost'], + GroupBy=[ + {'Type': 'DIMENSION', 'Key': 'SERVICE'}, + {'Type': 'DIMENSION', 'Key': 'USAGE_TYPE'}, + ], + ) + if next_token: + kwargs['NextPageToken'] = next_token + + response = client.get_cost_and_usage(**kwargs) + + for result in response['ResultsByTime']: + date = result['TimePeriod']['Start'] + for group in result['Groups']: + service = group['Keys'][0] + usage_type = group['Keys'][1] + amount = float(group['Metrics']['UnblendedCost']['Amount']) + if amount == 0: + continue + category = SERVICE_CATEGORY_MAP.get(service, 'other') + rows.append({ + 'date': date, + 'service': service, + 'usage_type': usage_type, + 'category': category, + 'amount_usd': round(amount, 4), + }) + + next_token = response.get('NextPageToken') + if not next_token: + break + + return rows + except Exception as e: + print(f"[rk_aws_costs] Detail fetch error: {e}") + return [] + + +def _ensure_detail_cached(): + now = time.time() + if _detail_cache['rows'] and now - _detail_cache['ts'] < _CACHE_TTL: + return + if not _detail_cache_lock.acquire(blocking=False): + return + try: + today = datetime.now(timezone.utc).date() + rows = _fetch_aws_cost_details( + (today - timedelta(days=365)).isoformat(), + today.isoformat(), + ) + if rows: + _detail_cache['rows'] = rows + _detail_cache['ts'] = now + finally: + _detail_cache_lock.release() + + +def get_aws_cost_details(date_from: str, date_to: str) -> list[dict]: + """Get per-resource AWS cost details. Blocks on first fetch, async refresh after.""" + if not _detail_cache['rows']: + _ensure_detail_cached() + else: + threading.Thread(target=_ensure_detail_cached, daemon=True).start() + return [r for r in _detail_cache['rows'] if date_from <= r['date'] <= date_to] + + +def get_costs_overview(date_from: str, date_to: str) -> dict: + """Combined AWS + GCP cost overview. GCP data comes from billing JSON files.""" + aws_rows = get_aws_costs(date_from, date_to) + + # GCP data from billing files (already on disk, no SQLite needed) + gcp_by_date = {} + try: + from billing.gcp import get_billing_files_in_range + billing_data = get_billing_files_in_range( + datetime.strptime(date_from, '%Y-%m-%d'), + datetime.strptime(date_to, '%Y-%m-%d'), + ) + for entry in billing_data: + d = entry['date'] + if d not in gcp_by_date: + gcp_by_date[d] = {} + for ns_data in entry.get('namespaces', {}).values(): + for cat, amt in ns_data.get('breakdown', {}).items(): + gcp_by_date[d][cat] = gcp_by_date[d].get(cat, 0) + amt + except Exception as e: + print(f"[rk_aws_costs] GCP billing read failed: {e}") + + by_date = {} + for r in aws_rows: + d = r['date'] + if d not in by_date: + by_date[d] = {'date': d, 'aws': {}, 'gcp': {}, 'aws_total': 0, 'gcp_total': 0} + cat = r['category'] + by_date[d]['aws'][cat] = by_date[d]['aws'].get(cat, 0) + r['amount_usd'] + by_date[d]['aws_total'] += r['amount_usd'] + + for d, cats in gcp_by_date.items(): + if d not in by_date: + by_date[d] = {'date': d, 'aws': {}, 'gcp': {}, 'aws_total': 0, 'gcp_total': 0} + by_date[d]['gcp'] = cats + by_date[d]['gcp_total'] = sum(cats.values()) + + sorted_dates = sorted(by_date.values(), key=lambda x: x['date']) + aws_total = sum(d['aws_total'] for d in sorted_dates) + gcp_total = sum(d['gcp_total'] for d in sorted_dates) + + return { + 'by_date': sorted_dates, + 'totals': { + 'aws': round(aws_total, 2), + 'gcp': round(gcp_total, 2), + 'combined': round(aws_total + gcp_total, 2), + } + } diff --git a/ci3/ci-metrics/billing/billing-dashboard.html b/ci3/ci-metrics/billing/billing-dashboard.html new file mode 100644 index 000000000000..87193ffae207 --- /dev/null +++ b/ci3/ci-metrics/billing/billing-dashboard.html @@ -0,0 +1,415 @@ + + + + + ACI - Namespace Billing + + + + +

namespace billing

+ +
+ + + + | + + + | + + + + | + + + + + + | + + + +
+ +
+ +
+ +
+
+

cost over time

+
+
+
+

cost by namespace

+
+
+
+

cost by category

+
+
+
+ + + + +
+ + + + + diff --git a/ci3/ci-metrics/billing/explore.py b/ci3/ci-metrics/billing/explore.py new file mode 100644 index 000000000000..c591d8c847ef --- /dev/null +++ b/ci3/ci-metrics/billing/explore.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 +"""CLI tool to explore GCP billing data from the Cloud Billing BigQuery export. + +Queries the actual billing export tables (not usage metering) to get real +invoice-level costs. Caches results in SQLite for fast re-queries. + +Usage: + python billing_explore.py discover # find billing export tables + python billing_explore.py fetch [--months N] # fetch & cache billing data + python billing_explore.py monthly # show monthly totals + python billing_explore.py monthly --by service # monthly by service + python billing_explore.py monthly --by sku # monthly by SKU + python billing_explore.py monthly --by project # monthly by project + python billing_explore.py daily [--month 2024-12] # daily for a month + python billing_explore.py top [--month 2024-12] # top costs for a month + python billing_explore.py compare # compare billing export vs usage metering +""" +import argparse +import os +import sqlite3 +import sys +from datetime import datetime, timedelta, timezone + +DB_PATH = os.path.join(os.getenv('LOGS_DISK_PATH', '/tmp'), 'billing_explore.db') + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS gcp_billing ( + date TEXT NOT NULL, + project_id TEXT NOT NULL DEFAULT '', + service TEXT NOT NULL DEFAULT '', + sku TEXT NOT NULL DEFAULT '', + cost REAL NOT NULL DEFAULT 0, + credits REAL NOT NULL DEFAULT 0, + usage_amount REAL NOT NULL DEFAULT 0, + usage_unit TEXT NOT NULL DEFAULT '', + currency TEXT NOT NULL DEFAULT 'USD', + fetched_at TEXT NOT NULL, + PRIMARY KEY (date, project_id, service, sku) +); +CREATE INDEX IF NOT EXISTS idx_gcp_billing_date ON gcp_billing(date); +CREATE INDEX IF NOT EXISTS idx_gcp_billing_service ON gcp_billing(service); + +CREATE TABLE IF NOT EXISTS gcp_billing_meta ( + key TEXT PRIMARY KEY, + value TEXT +); +""" + + +def get_db(): + os.makedirs(os.path.dirname(DB_PATH) or '.', exist_ok=True) + conn = sqlite3.connect(DB_PATH) + conn.execute('PRAGMA busy_timeout = 5000') + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + return conn + + +def fmt_usd(v): + if v >= 1000: + return f'${v:,.0f}' + if v >= 1: + return f'${v:,.2f}' + return f'${v:,.4f}' + + +# ---- BigQuery Discovery ---- + +def cmd_discover(args): + """Find billing export tables in the project.""" + from google.cloud import bigquery + project = args.project + client = bigquery.Client(project=project) + + print(f'Listing datasets in project: {project}') + datasets = list(client.list_datasets()) + if not datasets: + print(' No datasets found.') + return + + for ds in datasets: + ds_id = ds.dataset_id + tables = list(client.list_tables(ds.reference)) + billing_tables = [t for t in tables if 'billing' in t.table_id.lower() or 'cost' in t.table_id.lower()] + if billing_tables: + print(f'\n Dataset: {ds_id}') + for t in billing_tables: + full = f'{project}.{ds_id}.{t.table_id}' + print(f' {full}') + # Show schema for first billing table + tbl = client.get_table(t.reference) + print(f' rows: {tbl.num_rows}, size: {tbl.num_bytes / 1e6:.1f} MB') + print(f' columns: {", ".join(f.name for f in tbl.schema[:15])}') + else: + # Check for usage metering tables too + usage_tables = [t for t in tables if 'gke_cluster' in t.table_id.lower()] + if usage_tables: + print(f'\n Dataset: {ds_id} (usage metering)') + for t in usage_tables: + print(f' {project}.{ds_id}.{t.table_id}') + + # Also try common billing export naming patterns + print(f'\n Trying common billing export table patterns...') + for ds in datasets: + for t in client.list_tables(ds.reference): + if t.table_id.startswith('gcp_billing_export'): + full = f'{project}.{ds.dataset_id}.{t.table_id}' + print(f' FOUND: {full}') + + +# ---- BigQuery Fetch ---- + +def cmd_fetch(args): + """Fetch billing data from BigQuery and cache in SQLite.""" + from google.cloud import bigquery + + table = args.table + project = args.project + months = args.months + + if not table: + print('ERROR: --table is required. Run "discover" first to find the billing export table.') + print(' e.g. --table project.dataset.gcp_billing_export_resource_v1_XXXXXX') + sys.exit(1) + + client = bigquery.Client(project=project) + end_date = datetime.now(timezone.utc).date() + start_date = end_date - timedelta(days=months * 31) + + print(f'Fetching billing data from {start_date} to {end_date}') + print(f'Table: {table}') + + # Query the billing export table + # The standard billing export has: billing_account_id, service.description, + # sku.description, usage_start_time, project.id, cost, credits, usage.amount, usage.unit + query = f""" + SELECT + DATE(usage_start_time) AS date, + COALESCE(project.id, '') AS project_id, + COALESCE(service.description, '') AS service, + COALESCE(sku.description, '') AS sku, + SUM(cost) AS cost, + SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) AS credits, + SUM(usage.amount) AS usage_amount, + MAX(usage.unit) AS usage_unit + FROM `{table}` + WHERE DATE(usage_start_time) BETWEEN @start_date AND @end_date + GROUP BY date, project_id, service, sku + HAVING ABS(cost) > 0.0001 OR ABS(credits) > 0.0001 + ORDER BY date, service, sku + """ + + job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter('start_date', 'DATE', start_date.isoformat()), + bigquery.ScalarQueryParameter('end_date', 'DATE', end_date.isoformat()), + ] + ) + + print('Running query...') + result = list(client.query(query, job_config=job_config).result()) + print(f'Got {len(result)} rows') + + if not result: + print('No data returned. Check table name and date range.') + return + + # Store in SQLite + db = get_db() + now = datetime.now(timezone.utc).isoformat() + + db.execute('DELETE FROM gcp_billing WHERE date >= ? AND date <= ?', + (start_date.isoformat(), end_date.isoformat())) + + for row in result: + db.execute(''' + INSERT OR REPLACE INTO gcp_billing + (date, project_id, service, sku, cost, credits, usage_amount, usage_unit, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + row.date.isoformat() if hasattr(row.date, 'isoformat') else str(row.date), + row.project_id or '', + row.service or '', + row.sku or '', + float(row.cost or 0), + float(row.credits or 0), + float(row.usage_amount or 0), + row.usage_unit or '', + now, + )) + + db.commit() + db.execute("INSERT OR REPLACE INTO gcp_billing_meta VALUES ('last_fetch', ?)", (now,)) + db.execute("INSERT OR REPLACE INTO gcp_billing_meta VALUES ('table', ?)", (table,)) + db.commit() + + print(f'Cached {len(result)} rows in {DB_PATH}') + + # Show quick summary + rows = db.execute(''' + SELECT substr(date, 1, 7) as month, SUM(cost) as cost, SUM(credits) as credits + FROM gcp_billing GROUP BY month ORDER BY month + ''').fetchall() + print(f'\n{"Month":<10} {"Gross":>12} {"Credits":>12} {"Net":>12}') + print('-' * 48) + for r in rows: + net = r['cost'] + r['credits'] + print(f'{r["month"]:<10} {fmt_usd(r["cost"]):>12} {fmt_usd(r["credits"]):>12} {fmt_usd(net):>12}') + + +# ---- Reports ---- + +def cmd_monthly(args): + """Show monthly totals.""" + db = get_db() + group_by = args.by + + if group_by == 'service': + rows = db.execute(''' + SELECT substr(date, 1, 7) as month, service, + SUM(cost) as cost, SUM(credits) as credits + FROM gcp_billing GROUP BY month, service ORDER BY month, cost DESC + ''').fetchall() + + current_month = None + for r in rows: + if r['month'] != current_month: + current_month = r['month'] + month_total = sum(row['cost'] + row['credits'] for row in rows if row['month'] == current_month) + print(f'\n {current_month} (net: {fmt_usd(month_total)})') + print(f' {"Service":<45} {"Gross":>10} {"Credits":>10} {"Net":>10}') + print(' ' + '-' * 77) + net = r['cost'] + r['credits'] + if abs(net) >= 0.01: + print(f' {r["service"]:<45} {fmt_usd(r["cost"]):>10} {fmt_usd(r["credits"]):>10} {fmt_usd(net):>10}') + + elif group_by == 'sku': + month_filter = args.month + if not month_filter: + # Use most recent month + row = db.execute('SELECT MAX(substr(date, 1, 7)) as m FROM gcp_billing').fetchone() + month_filter = row['m'] if row else None + + if not month_filter: + print('No data.') + return + + rows = db.execute(''' + SELECT service, sku, SUM(cost) as cost, SUM(credits) as credits, + SUM(usage_amount) as usage_amount, MAX(usage_unit) as usage_unit + FROM gcp_billing WHERE substr(date, 1, 7) = ? + GROUP BY service, sku ORDER BY cost DESC + ''', (month_filter,)).fetchall() + + total = sum(r['cost'] + r['credits'] for r in rows) + print(f'\n {month_filter} (net: {fmt_usd(total)})') + print(f' {"Service":<30} {"SKU":<40} {"Net":>10} {"Usage":>15}') + print(' ' + '-' * 97) + for r in rows[:40]: + net = r['cost'] + r['credits'] + if abs(net) >= 0.01: + usage = f'{r["usage_amount"]:.1f} {r["usage_unit"]}' if r['usage_amount'] else '' + print(f' {r["service"][:29]:<30} {r["sku"][:39]:<40} {fmt_usd(net):>10} {usage:>15}') + + elif group_by == 'project': + rows = db.execute(''' + SELECT substr(date, 1, 7) as month, project_id, + SUM(cost) as cost, SUM(credits) as credits + FROM gcp_billing GROUP BY month, project_id ORDER BY month, cost DESC + ''').fetchall() + + current_month = None + for r in rows: + if r['month'] != current_month: + current_month = r['month'] + month_total = sum(row['cost'] + row['credits'] for row in rows if row['month'] == current_month) + print(f'\n {current_month} (net: {fmt_usd(month_total)})') + print(f' {"Project":<45} {"Net":>12}') + print(' ' + '-' * 59) + net = r['cost'] + r['credits'] + if abs(net) >= 0.01: + print(f' {r["project_id"]:<45} {fmt_usd(net):>12}') + + else: + # Default: just monthly totals + rows = db.execute(''' + SELECT substr(date, 1, 7) as month, + SUM(cost) as cost, SUM(credits) as credits, + COUNT(DISTINCT date) as days + FROM gcp_billing GROUP BY month ORDER BY month + ''').fetchall() + + print(f'\n {"Month":<10} {"Gross":>12} {"Credits":>12} {"Net":>12} {"Days":>6} {"Daily Avg":>12}') + print(' ' + '-' * 68) + grand_total = 0 + for r in rows: + net = r['cost'] + r['credits'] + daily = net / max(r['days'], 1) + grand_total += net + print(f' {r["month"]:<10} {fmt_usd(r["cost"]):>12} {fmt_usd(r["credits"]):>12} {fmt_usd(net):>12} {r["days"]:>6} {fmt_usd(daily):>12}') + print(' ' + '-' * 68) + print(f' {"TOTAL":<10} {"":>12} {"":>12} {fmt_usd(grand_total):>12}') + + +def cmd_daily(args): + """Show daily costs for a month.""" + db = get_db() + month = args.month + if not month: + row = db.execute('SELECT MAX(substr(date, 1, 7)) as m FROM gcp_billing').fetchone() + month = row['m'] if row else None + + if not month: + print('No data.') + return + + rows = db.execute(''' + SELECT date, SUM(cost) as cost, SUM(credits) as credits + FROM gcp_billing WHERE substr(date, 1, 7) = ? + GROUP BY date ORDER BY date + ''', (month,)).fetchall() + + total = 0 + print(f'\n {"Date":<12} {"Gross":>10} {"Credits":>10} {"Net":>10}') + print(' ' + '-' * 44) + for r in rows: + net = r['cost'] + r['credits'] + total += net + print(f' {r["date"]:<12} {fmt_usd(r["cost"]):>10} {fmt_usd(r["credits"]):>10} {fmt_usd(net):>10}') + print(' ' + '-' * 44) + print(f' {"TOTAL":<12} {"":>10} {"":>10} {fmt_usd(total):>10}') + + +def cmd_top(args): + """Show top cost items for a month.""" + db = get_db() + month = args.month + if not month: + row = db.execute('SELECT MAX(substr(date, 1, 7)) as m FROM gcp_billing').fetchone() + month = row['m'] if row else None + + if not month: + print('No data.') + return + + # Top services + services = db.execute(''' + SELECT service, SUM(cost + credits) as net, SUM(cost) as gross + FROM gcp_billing WHERE substr(date, 1, 7) = ? + GROUP BY service ORDER BY net DESC LIMIT 15 + ''', (month,)).fetchall() + + total = sum(r['net'] for r in services) + print(f'\n Top services for {month} (total: {fmt_usd(total)})') + print(f' {"Service":<45} {"Net":>12} {"% of Total":>10}') + print(' ' + '-' * 69) + for r in services: + pct = 100 * r['net'] / max(total, 0.01) + if abs(r['net']) >= 0.01: + print(f' {r["service"]:<45} {fmt_usd(r["net"]):>12} {pct:>9.1f}%') + + # Top SKUs + skus = db.execute(''' + SELECT service, sku, SUM(cost + credits) as net + FROM gcp_billing WHERE substr(date, 1, 7) = ? + GROUP BY service, sku ORDER BY net DESC LIMIT 20 + ''', (month,)).fetchall() + + print(f'\n Top SKUs for {month}') + print(f' {"Service":<25} {"SKU":<40} {"Net":>12}') + print(' ' + '-' * 79) + for r in skus: + if abs(r['net']) >= 0.01: + print(f' {r["service"][:24]:<25} {r["sku"][:39]:<40} {fmt_usd(r["net"]):>12}') + + +def cmd_compare(args): + """Compare billing export data vs usage metering estimates.""" + db = get_db() + + # Get billing export monthly totals + billing_rows = db.execute(''' + SELECT substr(date, 1, 7) as month, SUM(cost + credits) as net + FROM gcp_billing GROUP BY month ORDER BY month + ''').fetchall() + + if not billing_rows: + print('No billing export data cached. Run "fetch" first.') + return + + # Get usage metering estimates + try: + from billing import gcp as _gcp_billing + _gcp_billing._ensure_cached() + metering_data = _gcp_billing._cache.get('data', []) + except Exception as e: + print(f'Could not load usage metering data: {e}') + metering_data = [] + + metering_monthly = {} + for entry in metering_data: + month = entry['date'][:7] + day_total = sum(ns.get('total', 0) for ns in entry.get('namespaces', {}).values()) + metering_monthly[month] = metering_monthly.get(month, 0) + day_total + + print(f'\n {"Month":<10} {"Billing Export":>15} {"Usage Metering":>15} {"Ratio":>8}') + print(' ' + '-' * 50) + for r in billing_rows: + billing = r['net'] + metering = metering_monthly.get(r['month'], 0) + ratio = f'{billing / metering:.2f}x' if metering > 0 else '--' + print(f' {r["month"]:<10} {fmt_usd(billing):>15} {fmt_usd(metering):>15} {ratio:>8}') + + +def cmd_status(args): + """Show what data we have cached.""" + db = get_db() + meta = {r['key']: r['value'] for r in db.execute('SELECT * FROM gcp_billing_meta').fetchall()} + billing_count = db.execute('SELECT COUNT(*) as c FROM gcp_billing').fetchone()['c'] + billing_range = db.execute('SELECT MIN(date) as mn, MAX(date) as mx FROM gcp_billing').fetchone() + + print(f'\n Billing export cache:') + print(f' DB path: {DB_PATH}') + print(f' Table: {meta.get("table", "(not set)")}') + print(f' Last fetch: {meta.get("last_fetch", "(never)")}') + print(f' Rows: {billing_count}') + if billing_count: + print(f' Date range: {billing_range["mn"]} to {billing_range["mx"]}') + + # Also check billing export table status + try: + from google.cloud import bigquery + client = bigquery.Client(project=args.project) + table_id = 'testnet-440309.testnet440309billing.gcp_billing_export_v1_01EA8B_291C89_753ABC' + t = client.get_table(table_id) + print(f'\n BigQuery billing export:') + print(f' Table: {table_id}') + print(f' Rows: {t.num_rows}') + print(f' Modified: {t.modified}') + if t.num_rows > 0: + print(f' STATUS: Data available! Run "fetch --table {table_id}" to cache it.') + else: + print(f' STATUS: Not yet populated. GCP takes up to 24h after enabling export.') + except Exception as e: + print(f'\n BigQuery check failed: {e}') + + +def cmd_metering(args): + """Query both usage metering tables and compare with different approaches.""" + from google.cloud import bigquery + project = args.project + client = bigquery.Client(project=project) + months = args.months + + end_date = datetime.now(timezone.utc).date() + start_date = end_date - timedelta(days=months * 31) + + # Table names + usage_table = f'{project}.egress_consumption.gke_cluster_resource_usage' + consumption_table = f'{project}.egress_consumption.gke_cluster_resource_consumption' + + print(f'Date range: {start_date} to {end_date}') + + # 1. Current approach: usage table with our SKU pricing + print('\n=== Approach 1: gke_cluster_resource_usage (requests) with hardcoded SKU prices ===') + _query_metering_table(client, usage_table, start_date, end_date, 'REQUESTS') + + # 2. Consumption table with our SKU pricing + print('\n=== Approach 2: gke_cluster_resource_consumption (actual) with hardcoded SKU prices ===') + _query_metering_table(client, consumption_table, start_date, end_date, 'CONSUMPTION') + + # 3. Raw totals: what does each table report? + print('\n=== Approach 3: Raw resource totals from both tables ===') + for tname, label in [(usage_table, 'REQUESTS'), (consumption_table, 'CONSUMPTION')]: + query = f""" + SELECT + FORMAT_DATE('%Y-%m', DATE(start_time)) AS month, + resource_name, + SUM(usage.amount) AS total_amount, + usage.unit + FROM `{tname}` + WHERE DATE(start_time) BETWEEN @start AND @end + GROUP BY month, resource_name, usage.unit + ORDER BY month, resource_name + """ + job_config = bigquery.QueryJobConfig(query_parameters=[ + bigquery.ScalarQueryParameter('start', 'DATE', start_date.isoformat()), + bigquery.ScalarQueryParameter('end', 'DATE', end_date.isoformat()), + ]) + rows = list(client.query(query, job_config=job_config).result()) + print(f'\n {label} table raw resources:') + print(f' {"Month":<10} {"Resource":<20} {"Amount":>20} {"Unit":<15}') + print(' ' + '-' * 67) + for r in rows: + print(f' {r.month:<10} {r.resource_name:<20} {r.total_amount:>20,.0f} {r.unit:<15}') + + # 4. Count distinct SKUs + print('\n=== Approach 4: Distinct SKUs in usage table ===') + query = f""" + SELECT sku_id, resource_name, COUNT(*) as row_count, + SUM(usage.amount) as total_amount, usage.unit + FROM `{usage_table}` + WHERE DATE(start_time) BETWEEN @start AND @end + GROUP BY sku_id, resource_name, usage.unit + ORDER BY total_amount DESC + """ + job_config = bigquery.QueryJobConfig(query_parameters=[ + bigquery.ScalarQueryParameter('start', 'DATE', start_date.isoformat()), + bigquery.ScalarQueryParameter('end', 'DATE', end_date.isoformat()), + ]) + rows = list(client.query(query, job_config=job_config).result()) + # Import pricing to check + from billing.gcp import _SKU_PRICING + print(f' {"SKU ID":<20} {"Resource":<20} {"Rows":>10} {"Amount":>18} {"Unit":<12} {"Known?"}') + print(' ' + '-' * 90) + for r in rows: + known = 'YES' if r.sku_id in _SKU_PRICING else 'MISSING' + print(f' {r.sku_id:<20} {r.resource_name:<20} {r.row_count:>10,} {r.total_amount:>18,.0f} {r.unit:<12} {known}') + + +def _query_metering_table(client, table, start_date, end_date, label): + """Query a metering table and compute costs using our SKU pricing.""" + from google.cloud import bigquery + from billing.gcp import _SKU_PRICING, _usage_to_cost + + query = f""" + SELECT + FORMAT_DATE('%Y-%m', DATE(start_time)) AS month, + namespace, + sku_id, + resource_name, + SUM(usage.amount) AS total_usage + FROM `{table}` + WHERE DATE(start_time) BETWEEN @start AND @end + GROUP BY month, namespace, sku_id, resource_name + ORDER BY month, namespace + """ + job_config = bigquery.QueryJobConfig(query_parameters=[ + bigquery.ScalarQueryParameter('start', 'DATE', start_date.isoformat()), + bigquery.ScalarQueryParameter('end', 'DATE', end_date.isoformat()), + ]) + rows = list(client.query(query, job_config=job_config).result()) + + monthly = {} + monthly_by_cat = {} + missing_skus = set() + for r in rows: + cost, category = _usage_to_cost(r.sku_id, r.resource_name, float(r.total_usage)) + if r.sku_id not in _SKU_PRICING: + missing_skus.add(r.sku_id) + month = r.month + monthly[month] = monthly.get(month, 0) + cost + key = (month, category) + monthly_by_cat[key] = monthly_by_cat.get(key, 0) + cost + + print(f' {"Month":<10} {"Total":>12} {"compute_spot":>14} {"compute_od":>14} {"network":>10} {"storage":>10}') + print(' ' + '-' * 74) + for month in sorted(monthly.keys()): + total = monthly[month] + spot = monthly_by_cat.get((month, 'compute_spot'), 0) + od = monthly_by_cat.get((month, 'compute_ondemand'), 0) + net = monthly_by_cat.get((month, 'network'), 0) + stor = monthly_by_cat.get((month, 'storage'), 0) + print(f' {month:<10} {fmt_usd(total):>12} {fmt_usd(spot):>14} {fmt_usd(od):>14} {fmt_usd(net):>10} {fmt_usd(stor):>10}') + + if missing_skus: + print(f'\n WARNING: {len(missing_skus)} unknown SKU IDs (not priced): {", ".join(sorted(missing_skus)[:5])}...') + + +# ---- Main ---- + +def main(): + parser = argparse.ArgumentParser(description='Explore GCP billing data') + parser.add_argument('--project', default='testnet-440309', help='GCP project ID') + parser.add_argument('--table', default='', help='BigQuery billing export table') + sub = parser.add_subparsers(dest='command') + + sub.add_parser('discover', help='Find billing export tables') + + fetch_p = sub.add_parser('fetch', help='Fetch billing data from BigQuery') + fetch_p.add_argument('--months', type=int, default=6, help='How many months back to fetch') + + monthly_p = sub.add_parser('monthly', help='Monthly totals') + monthly_p.add_argument('--by', choices=['service', 'sku', 'project'], default='', help='Group by') + monthly_p.add_argument('--month', default='', help='Filter to month (YYYY-MM)') + + daily_p = sub.add_parser('daily', help='Daily costs') + daily_p.add_argument('--month', default='', help='Month to show (YYYY-MM)') + + top_p = sub.add_parser('top', help='Top cost items') + top_p.add_argument('--month', default='', help='Month to show (YYYY-MM)') + + sub.add_parser('compare', help='Compare billing export vs usage metering') + sub.add_parser('status', help='Show data status (what we have cached)') + + meter_p = sub.add_parser('metering', help='Query both metering tables directly and compare') + meter_p.add_argument('--months', type=int, default=6, help='How many months back') + + args = parser.parse_args() + + if not args.command: + parser.print_help() + sys.exit(1) + + cmds = { + 'discover': cmd_discover, + 'fetch': cmd_fetch, + 'monthly': cmd_monthly, + 'daily': cmd_daily, + 'top': cmd_top, + 'compare': cmd_compare, + 'metering': cmd_metering, + 'status': cmd_status, + } + cmds[args.command](args) + + +if __name__ == '__main__': + main() diff --git a/ci3/ci-metrics/billing/fetch_billing.py b/ci3/ci-metrics/billing/fetch_billing.py new file mode 100644 index 000000000000..271a788fc6bd --- /dev/null +++ b/ci3/ci-metrics/billing/fetch_billing.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Fetch namespace billing data from GKE resource consumption metering in BigQuery. + +Queries the GKE cluster resource consumption table which records CPU and memory +usage per namespace per pod. Actual GCP SKU prices (from the Cloud Billing +Catalog API) are applied to convert resource usage into dollar costs. + +Categories produced: + - compute_spot (Spot / Preemptible VM cores + RAM) + - compute_ondemand (On-demand VM cores + RAM) + +Usage: + # Fetch last 30 days + python fetch-billing.py + + # Specific range + python fetch-billing.py --from 2026-01-01 --to 2026-01-31 + + # Custom output directory + python fetch-billing.py --output-dir /tmp/billing + +Environment: + Requires Application Default Credentials or GOOGLE_APPLICATION_CREDENTIALS. + pip install google-cloud-bigquery +""" +import argparse +import json +import os +import sys +from datetime import datetime, timedelta + +from google.cloud import bigquery + +# ---- defaults ---- +DEFAULT_PROJECT = 'testnet-440309' +DEFAULT_DATASET = 'egress_consumption' +DEFAULT_TABLE_CONSUMPTION = 'gke_cluster_resource_consumption' +DEFAULT_TABLE_USAGE = 'gke_cluster_resource_usage' +DEFAULT_OUTPUT_DIR = os.path.join( + os.getenv('LOGS_DISK_PATH', '/logs-disk'), 'billing' +) + +# ---- SKU pricing ---- +# Prices sourced from GCP Cloud Billing Catalog API for us-west1. +SKU_PRICING = { + # Compute - Spot (per vCPU-hour / per GiB-hour) + 'E7FF-A0FB-FA82': {'price': 0.00497, 'resource': 'cpu', 'category': 'compute_spot'}, + '48AB-89F5-9112': {'price': 0.000668, 'resource': 'memory', 'category': 'compute_spot'}, + # Compute - On-demand T2D + 'EFE6-E23C-19CB': {'price': 0.027502, 'resource': 'cpu', 'category': 'compute_ondemand'}, + 'FB05-036A-8982': {'price': 0.003686, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Compute - On-demand N2 + 'BB77-5FDA-69D9': {'price': 0.031611, 'resource': 'cpu', 'category': 'compute_ondemand'}, + '5B01-D157-A097': {'price': 0.004237, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Compute - On-demand N2D + 'A03E-E620-7389': {'price': 0.027502, 'resource': 'cpu', 'category': 'compute_ondemand'}, + '5535-6D2D-4B50': {'price': 0.003686, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Network Egress (per GiB) + '0C3C-6B13-B1E8': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + '6B8F-E63D-832B': {'price': 0.0, 'resource': 'networkEgress', 'category': 'network'}, + '92CB-C25F-B1D1': {'price': 0.0, 'resource': 'networkEgress', 'category': 'network'}, + '984A-1F27-2D1F': {'price': 0.04, 'resource': 'networkEgress', 'category': 'network'}, + '9DE9-9092-B3BC': {'price': 0.20, 'resource': 'networkEgress', 'category': 'network'}, + 'C863-37DA-506E': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + 'C8EA-1A86-3D28': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + 'DE9E-AFBC-A15A': {'price': 0.01, 'resource': 'networkEgress', 'category': 'network'}, + 'DFA5-B5C6-36D6': {'price': 0.085, 'resource': 'networkEgress', 'category': 'network'}, + 'F274-1692-F213': {'price': 0.08, 'resource': 'networkEgress', 'category': 'network'}, + 'FDBC-6E3B-D4D8': {'price': 0.15, 'resource': 'networkEgress', 'category': 'network'}, + # Storage (per GiB-month) + 'D973-5D65-BAB2': {'price': 0.04, 'resource': 'storage', 'category': 'storage'}, +} + + +def usage_to_cost(sku_id: str, resource_name: str, amount: float) -> tuple[float, str]: + """Convert raw usage amount to dollar cost. Returns (cost_usd, category).""" + info = SKU_PRICING.get(sku_id) + if not info: + return 0.0, 'other' + + price = info['price'] + if resource_name == 'cpu': + return (amount / 3600.0) * price, info['category'] + elif resource_name == 'memory': + return (amount / 3600.0 / (1024 ** 3)) * price, info['category'] + elif resource_name.startswith('networkEgress'): + return (amount / (1024 ** 3)) * price, info['category'] + elif resource_name == 'storage': + gib_months = amount / (1024 ** 3) / (730 * 3600) + return gib_months * price, info['category'] + return 0.0, info['category'] + + +# ---- BigQuery query ---- + +def fetch_usage_rows( + client: bigquery.Client, + project: str, + dataset: str, + date_from: str, + date_to: str, +) -> list[dict]: + """Query both metering tables for daily usage by namespace + SKU.""" + consumption = f'{project}.{dataset}.{DEFAULT_TABLE_CONSUMPTION}' + usage = f'{project}.{dataset}.{DEFAULT_TABLE_USAGE}' + query = f""" + SELECT date, namespace, sku_id, resource_name, SUM(total_usage) AS total_usage FROM ( + SELECT DATE(start_time) AS date, namespace, sku_id, resource_name, SUM(usage.amount) AS total_usage + FROM `{consumption}` + WHERE DATE(start_time) BETWEEN @date_from AND @date_to + GROUP BY date, namespace, sku_id, resource_name + UNION ALL + SELECT DATE(start_time) AS date, namespace, sku_id, resource_name, SUM(usage.amount) AS total_usage + FROM `{usage}` + WHERE DATE(start_time) BETWEEN @date_from AND @date_to + AND resource_name IN ('networkEgress', 'storage') + GROUP BY date, namespace, sku_id, resource_name + ) + GROUP BY date, namespace, sku_id, resource_name + ORDER BY date, namespace + """ + job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter('date_from', 'DATE', date_from), + bigquery.ScalarQueryParameter('date_to', 'DATE', date_to), + ] + ) + rows = client.query(query, job_config=job_config).result() + return [dict(row) for row in rows] + + +# ---- aggregate into daily JSON ---- + +def build_daily_files(rows: list[dict]) -> tuple[dict[str, dict], set[str]]: + """Convert raw usage rows into daily billing JSON structures. + + Returns (days_dict, unknown_skus). + """ + days: dict[str, dict] = {} + unknown_skus: set[str] = set() + + for row in rows: + date_str = ( + row['date'].isoformat() + if hasattr(row['date'], 'isoformat') + else str(row['date']) + ) + ns = row['namespace'] + sku_id = row['sku_id'] + resource_name = row['resource_name'] + amount = float(row['total_usage']) + + cost, category = usage_to_cost(sku_id, resource_name, amount) + + if sku_id not in SKU_PRICING: + unknown_skus.add(sku_id) + + if cost <= 0: + continue + + if date_str not in days: + days[date_str] = {'date': date_str, 'namespaces': {}} + if ns not in days[date_str]['namespaces']: + days[date_str]['namespaces'][ns] = {'total': 0, 'breakdown': {}} + + entry = days[date_str]['namespaces'][ns] + entry['breakdown'][category] = ( + entry['breakdown'].get(category, 0) + cost + ) + entry['total'] += cost + + # Round + for day in days.values(): + for ns_data in day['namespaces'].values(): + ns_data['total'] = round(ns_data['total'], 4) + ns_data['breakdown'] = { + k: round(v, 4) for k, v in ns_data['breakdown'].items() + } + + return days, unknown_skus + + +def write_files(days: dict[str, dict], output_dir: str) -> int: + os.makedirs(output_dir, exist_ok=True) + count = 0 + for date_str, data in sorted(days.items()): + filepath = os.path.join(output_dir, f'{date_str}.json') + with open(filepath, 'w') as f: + json.dump(data, f, indent=2) + count += 1 + return count + + +# ---- CLI ---- + +def main(): + parser = argparse.ArgumentParser( + description='Fetch GKE namespace compute billing from resource consumption metering' + ) + today = datetime.utcnow().strftime('%Y-%m-%d') + default_from = (datetime.utcnow() - timedelta(days=30)).strftime('%Y-%m-%d') + + parser.add_argument('--from', dest='date_from', default=default_from, + help='Start date YYYY-MM-DD (default: 30 days ago)') + parser.add_argument('--to', dest='date_to', default=today, + help='End date YYYY-MM-DD (default: today)') + parser.add_argument('--project', default=DEFAULT_PROJECT, + help=f'GCP project ID (default: {DEFAULT_PROJECT})') + parser.add_argument('--dataset', default=DEFAULT_DATASET, + help=f'BigQuery dataset (default: {DEFAULT_DATASET})') + parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR, + help=f'Output directory (default: {DEFAULT_OUTPUT_DIR})') + args = parser.parse_args() + + print(f'Connecting to BigQuery ({args.project})...') + client = bigquery.Client(project=args.project) + + print(f'Fetching metering data {args.date_from} to {args.date_to}...') + print(f' consumption: {args.project}.{args.dataset}.{DEFAULT_TABLE_CONSUMPTION}') + print(f' usage: {args.project}.{args.dataset}.{DEFAULT_TABLE_USAGE}') + rows = fetch_usage_rows( + client, args.project, args.dataset, + args.date_from, args.date_to, + ) + print(f'Got {len(rows)} aggregated rows') + + if not rows: + print('No metering data found. Check that:') + print(' 1. GKE resource consumption metering is enabled') + print(' 2. The date range has data') + return + + days, unknown_skus = build_daily_files(rows) + count = write_files(days, args.output_dir) + print(f'Wrote {count} daily billing files to {args.output_dir}') + + if unknown_skus: + print(f'\nWARNING: {len(unknown_skus)} unknown SKU(s) had zero cost assigned:') + for s in sorted(unknown_skus): + print(f' {s}') + print('Add these to SKU_PRICING in fetch-billing.py with prices from') + print('the GCP Cloud Billing Catalog API.') + + # Summary + total = sum( + ns['total'] for day in days.values() + for ns in day['namespaces'].values() + ) + ns_set: set[str] = set() + cat_set: set[str] = set() + for day in days.values(): + for ns_name, ns_data in day['namespaces'].items(): + ns_set.add(ns_name) + cat_set.update(ns_data['breakdown'].keys()) + + print(f'\nTotal cost: ${total:,.2f}') + print(f'Namespaces ({len(ns_set)}): {sorted(ns_set)}') + print(f'Categories: {sorted(cat_set)}') + + +if __name__ == '__main__': + main() diff --git a/ci3/ci-metrics/billing/gcp.py b/ci3/ci-metrics/billing/gcp.py new file mode 100644 index 000000000000..5254e20bbbf0 --- /dev/null +++ b/ci3/ci-metrics/billing/gcp.py @@ -0,0 +1,289 @@ +"""Namespace billing helpers for rkapp. + +Fetches GKE namespace billing from BigQuery with in-memory cache. +Route definitions remain in rk.py; this module provides the logic. + +SKU pricing: Queries the Cloud Billing pricing export table in BigQuery +if available, otherwise falls back to hardcoded rates. To enable the +pricing export: + 1. Go to GCP Console > Billing > Billing export + 2. Enable "Detailed usage cost" and "Pricing" exports + 3. Set the dataset to the _BQ_DATASET below +""" +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# BigQuery defaults +_BQ_PROJECT = 'testnet-440309' +_BQ_DATASET = 'egress_consumption' +_BQ_TABLE_USAGE = 'gke_cluster_resource_usage' +_BQ_TABLE_PRICING = 'cloud_pricing_export' + +# Hardcoded fallback SKU pricing (us-west1). +# cpu: price per vCPU-hour, memory: price per GiB-hour +# network: price per GiB, storage: price per GiB-month +_HARDCODED_SKU_PRICING = { + # Compute - Spot + 'E7FF-A0FB-FA82': {'price': 0.00497, 'resource': 'cpu', 'category': 'compute_spot'}, + '48AB-89F5-9112': {'price': 0.000668, 'resource': 'memory', 'category': 'compute_spot'}, + # Compute - On-demand T2D + 'EFE6-E23C-19CB': {'price': 0.027502, 'resource': 'cpu', 'category': 'compute_ondemand'}, + 'FB05-036A-8982': {'price': 0.003686, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Compute - On-demand N2 + 'BB77-5FDA-69D9': {'price': 0.031611, 'resource': 'cpu', 'category': 'compute_ondemand'}, + '5B01-D157-A097': {'price': 0.004237, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Compute - On-demand N2D + 'A03E-E620-7389': {'price': 0.027502, 'resource': 'cpu', 'category': 'compute_ondemand'}, + '5535-6D2D-4B50': {'price': 0.003686, 'resource': 'memory', 'category': 'compute_ondemand'}, + # Network Egress (price per GiB) + '0C3C-6B13-B1E8': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + '6B8F-E63D-832B': {'price': 0.0, 'resource': 'networkEgress', 'category': 'network'}, + '92CB-C25F-B1D1': {'price': 0.0, 'resource': 'networkEgress', 'category': 'network'}, + '984A-1F27-2D1F': {'price': 0.04, 'resource': 'networkEgress', 'category': 'network'}, + '9DE9-9092-B3BC': {'price': 0.20, 'resource': 'networkEgress', 'category': 'network'}, + 'C863-37DA-506E': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + 'C8EA-1A86-3D28': {'price': 0.02, 'resource': 'networkEgress', 'category': 'network'}, + 'DE9E-AFBC-A15A': {'price': 0.01, 'resource': 'networkEgress', 'category': 'network'}, + 'DFA5-B5C6-36D6': {'price': 0.085, 'resource': 'networkEgress', 'category': 'network'}, + 'F274-1692-F213': {'price': 0.08, 'resource': 'networkEgress', 'category': 'network'}, + 'FDBC-6E3B-D4D8': {'price': 0.15, 'resource': 'networkEgress', 'category': 'network'}, + # Storage (price per GiB-month) + 'D973-5D65-BAB2': {'price': 0.04, 'resource': 'storage', 'category': 'storage'}, +} + +# Resource name to category mapping for SKUs discovered from BigQuery +_RESOURCE_CATEGORIES = { + ('cpu', True): 'compute_spot', + ('cpu', False): 'compute_ondemand', + ('memory', True): 'compute_spot', + ('memory', False): 'compute_ondemand', +} + +# Active SKU pricing — updated from BigQuery if available +_SKU_PRICING = dict(_HARDCODED_SKU_PRICING) + +# In-memory caches +_cache = {'data': [], 'ts': 0} +_cache_lock = threading.Lock() +_CACHE_TTL = 6 * 3600 # 6 hours + +_pricing_cache = {'ts': 0} +_pricing_lock = threading.Lock() +_PRICING_CACHE_TTL = 24 * 3600 # 24 hours + + +def _refresh_sku_pricing(): + """Try to fetch SKU pricing from BigQuery pricing export table.""" + global _SKU_PRICING + now = time.time() + if _pricing_cache['ts'] and now - _pricing_cache['ts'] < _PRICING_CACHE_TTL: + return + if not _pricing_lock.acquire(blocking=False): + return + try: + if _pricing_cache['ts'] and time.time() - _pricing_cache['ts'] < _PRICING_CACHE_TTL: + return + from google.cloud import bigquery + client = bigquery.Client(project=_BQ_PROJECT) + table = f'{_BQ_PROJECT}.{_BQ_DATASET}.{_BQ_TABLE_PRICING}' + + # Get the known SKU IDs we need pricing for + sku_ids = list(_HARDCODED_SKU_PRICING.keys()) + placeholders = ', '.join(f"'{s}'" for s in sku_ids) + + query = f""" + SELECT sku.id AS sku_id, + pricing.effective_price AS price, + sku.description AS description + FROM `{table}` + WHERE sku.id IN ({placeholders}) + AND service.description = 'Compute Engine' + QUALIFY ROW_NUMBER() OVER (PARTITION BY sku.id ORDER BY export_time DESC) = 1 + """ + rows = list(client.query(query).result()) + if rows: + updated = dict(_HARDCODED_SKU_PRICING) + for row in rows: + sid = row.sku_id + if sid in updated: + updated[sid] = {**updated[sid], 'price': float(row.price)} + _SKU_PRICING = updated + _pricing_cache['ts'] = time.time() + print(f"[rk_billing] Updated {len(rows)} SKU prices from BigQuery") + else: + _pricing_cache['ts'] = time.time() + print("[rk_billing] No pricing rows returned, using hardcoded rates") + except Exception as e: + # Table probably doesn't exist yet — use hardcoded rates + _pricing_cache['ts'] = time.time() + print(f"[rk_billing] SKU pricing query failed (using hardcoded): {e}") + finally: + _pricing_lock.release() + + +# ---- BigQuery fetch ---- + +def _usage_to_cost(sku_id, resource_name, amount): + info = _SKU_PRICING.get(sku_id) + if not info: + return 0.0, 'other' + price = info['price'] + if resource_name == 'cpu': + # cpu-seconds -> hours + return (amount / 3600.0) * price, info['category'] + elif resource_name == 'memory': + # byte-seconds -> GiB-hours + return (amount / 3600.0 / (1024 ** 3)) * price, info['category'] + elif resource_name.startswith('networkEgress'): + # bytes -> GiB + return (amount / (1024 ** 3)) * price, info['category'] + elif resource_name == 'storage': + # byte-seconds -> GiB-months (730 hours/month) + gib_months = amount / (1024 ** 3) / (730 * 3600) + return gib_months * price, info['category'] + return 0.0, info['category'] + + +def _fetch_from_bigquery(date_from_str, date_to_str): + """Query BigQuery for usage data, return list of daily billing entries.""" + try: + from google.cloud import bigquery + except ImportError: + print("[rk_billing] google-cloud-bigquery not installed") + return [] + + try: + client = bigquery.Client(project=_BQ_PROJECT) + # Use the usage table for all resources (actual consumption, not just requests). + # The consumption table only records resource *requests* which can be far lower + # than actual usage (e.g. prove-n-tps-real: $2.87 requests vs $138.72 actual). + usage = f'{_BQ_PROJECT}.{_BQ_DATASET}.{_BQ_TABLE_USAGE}' + query = f""" + SELECT DATE(start_time) AS date, namespace, sku_id, resource_name, + SUM(usage.amount) AS total_usage + FROM `{usage}` + WHERE DATE(start_time) BETWEEN @date_from AND @date_to + GROUP BY date, namespace, sku_id, resource_name + ORDER BY date, namespace + """ + job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter('date_from', 'DATE', date_from_str), + bigquery.ScalarQueryParameter('date_to', 'DATE', date_to_str), + ] + ) + rows = list(client.query(query, job_config=job_config).result()) + except Exception as e: + print(f"[rk_billing] BigQuery fetch failed: {e}") + return [] + + # Build daily structures + days = {} + for row in rows: + date_str = row.date.isoformat() if hasattr(row.date, 'isoformat') else str(row.date) + ns = row.namespace + cost, category = _usage_to_cost(row.sku_id, row.resource_name, float(row.total_usage)) + if cost <= 0: + continue + if date_str not in days: + days[date_str] = {'date': date_str, 'namespaces': {}} + if ns not in days[date_str]['namespaces']: + days[date_str]['namespaces'][ns] = {'total': 0, 'breakdown': {}} + entry = days[date_str]['namespaces'][ns] + entry['breakdown'][category] = entry['breakdown'].get(category, 0) + cost + entry['total'] += cost + + # Round values + for data in days.values(): + for ns_data in data['namespaces'].values(): + ns_data['total'] = round(ns_data['total'], 4) + ns_data['breakdown'] = {k: round(v, 4) for k, v in ns_data['breakdown'].items()} + + return sorted(days.values(), key=lambda x: x['date']) + + +def _ensure_cached(): + now = time.time() + if _cache['data'] and now - _cache['ts'] < _CACHE_TTL: + return + if not _cache_lock.acquire(blocking=False): + return + try: + yesterday = datetime.now(timezone.utc).date() - timedelta(days=1) + date_from = (yesterday - timedelta(days=365)).isoformat() + date_to = yesterday.isoformat() + print(f"[rk_billing] Fetching billing data from BigQuery ({date_from} to {date_to})...") + data = _fetch_from_bigquery(date_from, date_to) + if data: + _cache['data'] = data + _cache['ts'] = now + print(f"[rk_billing] Cached {len(data)} days of billing data") + finally: + _cache_lock.release() + + +# ---- Public API ---- + +def get_billing_files_in_range(date_from, date_to): + """Return billing data for dates in range. Fetches from BigQuery with in-memory cache.""" + # Refresh SKU pricing from BigQuery (async, falls back to hardcoded) + threading.Thread(target=_refresh_sku_pricing, daemon=True).start() + + if not _cache['data']: + _ensure_cached() # block on first load so dashboard isn't empty + else: + threading.Thread(target=_ensure_cached, daemon=True).start() + + # Convert datetime args to date strings for filtering + from_str = date_from.strftime('%Y-%m-%d') if hasattr(date_from, 'strftime') else str(date_from) + to_str = date_to.strftime('%Y-%m-%d') if hasattr(date_to, 'strftime') else str(date_to) + + return [e for e in _cache['data'] if from_str <= e['date'] <= to_str] + + +def _merge_ns_billing(target, ns_data): + target['total'] += ns_data.get('total', 0) + for cat, val in ns_data.get('breakdown', {}).items(): + target['breakdown'][cat] = target['breakdown'].get(cat, 0) + val + + +def aggregate_billing_weekly(daily_data): + if not daily_data: + return [] + weeks = {} + for entry in daily_data: + d = datetime.strptime(entry['date'], '%Y-%m-%d') + week_start = d - timedelta(days=d.weekday()) + week_key = week_start.strftime('%Y-%m-%d') + if week_key not in weeks: + weeks[week_key] = {'date': week_key, 'namespaces': {}} + for ns, ns_data in entry.get('namespaces', {}).items(): + if ns not in weeks[week_key]['namespaces']: + weeks[week_key]['namespaces'][ns] = {'total': 0, 'breakdown': {}} + _merge_ns_billing(weeks[week_key]['namespaces'][ns], ns_data) + return sorted(weeks.values(), key=lambda x: x['date']) + + +def aggregate_billing_monthly(daily_data): + if not daily_data: + return [] + months = {} + for entry in daily_data: + month_key = entry['date'][:7] + '-01' + if month_key not in months: + months[month_key] = {'date': month_key, 'namespaces': {}} + for ns, ns_data in entry.get('namespaces', {}).items(): + if ns not in months[month_key]['namespaces']: + months[month_key]['namespaces'][ns] = {'total': 0, 'breakdown': {}} + _merge_ns_billing(months[month_key]['namespaces'][ns], ns_data) + return sorted(months.values(), key=lambda x: x['date']) + + +def serve_billing_dashboard(): + billing_html_path = Path(__file__).parent / 'billing-dashboard.html' + if billing_html_path.exists(): + with billing_html_path.open('r') as f: + return f.read() + return None diff --git a/ci3/ci-metrics/ci-run-seed.json.gz b/ci3/ci-metrics/ci-run-seed.json.gz new file mode 100644 index 000000000000..a971ad10d38b Binary files /dev/null and b/ci3/ci-metrics/ci-run-seed.json.gz differ diff --git a/ci3/ci-metrics/db.py b/ci3/ci-metrics/db.py new file mode 100644 index 000000000000..93e970fe3a56 --- /dev/null +++ b/ci3/ci-metrics/db.py @@ -0,0 +1,107 @@ +"""SQLite database for CI metrics storage. + +Stores test events (from Redis pub/sub) and merge queue daily stats +(backfilled from GitHub API). +""" +import os +import sqlite3 +import threading + +_DB_PATH = os.path.join(os.getenv('LOGS_DISK_PATH', '/logs-disk'), 'metrics.db') +_local = threading.local() + +SCHEMA = """ +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS test_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + status TEXT NOT NULL, + test_cmd TEXT NOT NULL, + log_url TEXT, + ref_name TEXT NOT NULL, + commit_hash TEXT, + commit_author TEXT, + commit_msg TEXT, + exit_code INTEGER, + duration_secs REAL, + is_scenario INTEGER DEFAULT 0, + owners TEXT, + flake_group_id TEXT, + dashboard TEXT NOT NULL DEFAULT '', + timestamp TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_test_events_status ON test_events(status); +CREATE INDEX IF NOT EXISTS idx_test_events_ts ON test_events(timestamp); +CREATE INDEX IF NOT EXISTS idx_test_events_cmd ON test_events(test_cmd); +CREATE INDEX IF NOT EXISTS idx_test_events_dashboard ON test_events(dashboard); + +CREATE TABLE IF NOT EXISTS merge_queue_daily ( + date TEXT PRIMARY KEY, + total INTEGER NOT NULL DEFAULT 0, + success INTEGER NOT NULL DEFAULT 0, + failure INTEGER NOT NULL DEFAULT 0, + cancelled INTEGER NOT NULL DEFAULT 0, + in_progress INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS ci_runs ( + dashboard TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + timestamp_ms INTEGER NOT NULL, + complete_ms INTEGER, + status TEXT, + author TEXT, + pr_number INTEGER, + instance_type TEXT, + instance_vcpus INTEGER, + spot INTEGER DEFAULT 0, + cost_usd REAL, + job_id TEXT DEFAULT '', + arch TEXT DEFAULT '', + synced_at TEXT NOT NULL, + PRIMARY KEY (dashboard, timestamp_ms, name) +); +CREATE INDEX IF NOT EXISTS idx_ci_runs_ts ON ci_runs(timestamp_ms); +CREATE INDEX IF NOT EXISTS idx_ci_runs_name ON ci_runs(name); +CREATE INDEX IF NOT EXISTS idx_ci_runs_dashboard ON ci_runs(dashboard); +""" + + +_MIGRATIONS = [ + # Add columns introduced after initial schema + "ALTER TABLE ci_runs ADD COLUMN instance_vcpus INTEGER", + "ALTER TABLE ci_runs ADD COLUMN job_id TEXT DEFAULT ''", + "ALTER TABLE ci_runs ADD COLUMN arch TEXT DEFAULT ''", + "CREATE INDEX IF NOT EXISTS idx_ci_runs_dashboard ON ci_runs(dashboard)", +] + + +def get_db() -> sqlite3.Connection: + conn = getattr(_local, 'conn', None) + if conn is None: + os.makedirs(os.path.dirname(_DB_PATH), exist_ok=True) + conn = sqlite3.connect(_DB_PATH) + conn.execute('PRAGMA busy_timeout = 5000') + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + # Run migrations (ignore "duplicate column" errors for idempotency) + for sql in _MIGRATIONS: + try: + conn.execute(sql) + except sqlite3.OperationalError: + pass + conn.commit() + _local.conn = conn + return conn + + +def query(sql: str, params=()) -> list[dict]: + conn = get_db() + rows = conn.execute(sql, params).fetchall() + return [dict(r) for r in rows] + + +def execute(sql: str, params=()): + conn = get_db() + conn.execute(sql, params) + conn.commit() diff --git a/ci3/ci-metrics/ec2_pricing.py b/ci3/ci-metrics/ec2_pricing.py new file mode 100644 index 000000000000..ace55ea4f40a --- /dev/null +++ b/ci3/ci-metrics/ec2_pricing.py @@ -0,0 +1,232 @@ +"""EC2 instance pricing: live on-demand + spot rates with TTL cache. + +Queries the AWS Pricing API (on-demand) and EC2 describe_spot_price_history +(spot) for us-east-2 instance rates. Caches results for 24 hours and falls +back to hardcoded values if the APIs are unavailable. + +Exports: + get_instance_rate(instance_type, is_spot) -> float + get_fallback_vcpu_rate(is_spot) -> float +""" +import json +import threading +import time +from datetime import datetime, timezone + +# ---- Hardcoded fallback rates (us-east-2, USD/hr) ---- + +_HARDCODED_RATES = { + ('m6a.48xlarge', True): 8.31, # spot + ('m6a.48xlarge', False): 16.56, # on-demand + ('m6a.32xlarge', True): 5.54, + ('m6a.32xlarge', False): 11.04, + ('m6a.16xlarge', True): 2.77, + ('m6a.16xlarge', False): 5.52, + ('m7a.48xlarge', True): 8.31, + ('m7a.48xlarge', False): 16.56, + ('m7a.16xlarge', True): 2.77, + ('m7a.16xlarge', False): 5.52, + ('m7i.48xlarge', True): 8.31, + ('m7i.48xlarge', False): 16.56, + ('r7g.16xlarge', True): 1.97, + ('r7g.16xlarge', False): 3.94, +} +_FALLBACK_VCPU_HOUR = {True: 0.0433, False: 0.0864} + +# ---- Cache state ---- + +_REGION = 'us-east-2' +_LOCATION = 'US East (Ohio)' # Pricing API uses location names, not codes +_CACHE_TTL = 24 * 3600 # 24 hours + +_cache = { + 'ondemand': {}, # instance_type -> USD/hr + 'spot': {}, # instance_type -> USD/hr + 'ts': 0, # last successful fetch time +} +_cache_lock = threading.Lock() + + +# ---- On-demand pricing (AWS Pricing API) ---- + +def _fetch_ondemand_rate(pricing_client, instance_type: str) -> float | None: + """Fetch on-demand hourly rate for a single instance type from AWS Pricing API. + + The Pricing API is only available in us-east-1 and ap-south-1. + """ + try: + response = pricing_client.get_products( + ServiceCode='AmazonEC2', + Filters=[ + {'Type': 'TERM_MATCH', 'Field': 'instanceType', 'Value': instance_type}, + {'Type': 'TERM_MATCH', 'Field': 'location', 'Value': _LOCATION}, + {'Type': 'TERM_MATCH', 'Field': 'operatingSystem', 'Value': 'Linux'}, + {'Type': 'TERM_MATCH', 'Field': 'preInstalledSw', 'Value': 'NA'}, + {'Type': 'TERM_MATCH', 'Field': 'tenancy', 'Value': 'Shared'}, + {'Type': 'TERM_MATCH', 'Field': 'capacitystatus', 'Value': 'Used'}, + ], + MaxResults=10, + ) + for price_item in response.get('PriceList', []): + product = json.loads(price_item) if isinstance(price_item, str) else price_item + on_demand = product.get('terms', {}).get('OnDemand', {}) + for term in on_demand.values(): + for dim in term.get('priceDimensions', {}).values(): + price = dim.get('pricePerUnit', {}).get('USD') + if price and float(price) > 0: + return float(price) + except Exception as e: + print(f"[ec2_pricing] on-demand fetch error for {instance_type}: {e}") + return None + + +def _fetch_all_ondemand(instance_types: list[str]) -> dict[str, float]: + """Fetch on-demand rates for all instance types. Returns {type: rate}.""" + try: + import boto3 + except ImportError: + print("[ec2_pricing] boto3 not installed, skipping on-demand fetch") + return {} + + results = {} + try: + # Pricing API is only in us-east-1 and ap-south-1 + pricing = boto3.client('pricing', region_name='us-east-1') + for itype in instance_types: + rate = _fetch_ondemand_rate(pricing, itype) + if rate is not None: + results[itype] = rate + except Exception as e: + print(f"[ec2_pricing] on-demand client error: {e}") + return results + + +# ---- Spot pricing (EC2 describe_spot_price_history) ---- + +def _fetch_all_spot(instance_types: list[str]) -> dict[str, float]: + """Fetch current spot prices for all instance types. Returns {type: rate}. + + Uses describe_spot_price_history with StartTime=now to get the most recent + price. Takes the minimum across availability zones. + """ + try: + import boto3 + except ImportError: + print("[ec2_pricing] boto3 not installed, skipping spot fetch") + return {} + + results = {} + try: + ec2 = boto3.client('ec2', region_name=_REGION) + for itype in instance_types: + try: + response = ec2.describe_spot_price_history( + InstanceTypes=[itype], + ProductDescriptions=['Linux/UNIX'], + StartTime=datetime.now(timezone.utc), + MaxResults=10, + ) + prices = [] + for entry in response.get('SpotPriceHistory', []): + try: + prices.append(float(entry['SpotPrice'])) + except (KeyError, ValueError): + continue + if prices: + # Use the minimum AZ price (what our fleet would target) + results[itype] = min(prices) + except Exception as e: + print(f"[ec2_pricing] spot fetch error for {itype}: {e}") + except Exception as e: + print(f"[ec2_pricing] spot client error: {e}") + return results + + +# ---- Cache refresh ---- + +def _get_known_instance_types() -> list[str]: + """Return the set of instance types we need pricing for.""" + return sorted({itype for itype, _ in _HARDCODED_RATES}) + + +def _refresh_cache(): + """Fetch fresh pricing data and update the cache. Thread-safe.""" + now = time.time() + if _cache['ts'] and now - _cache['ts'] < _CACHE_TTL: + return + if not _cache_lock.acquire(blocking=False): + return # another thread is already refreshing + try: + # Double-check after acquiring lock + if _cache['ts'] and time.time() - _cache['ts'] < _CACHE_TTL: + return + + instance_types = _get_known_instance_types() + ondemand = _fetch_all_ondemand(instance_types) + spot = _fetch_all_spot(instance_types) + + # Only update cache if we got at least some data + if ondemand or spot: + if ondemand: + _cache['ondemand'] = ondemand + if spot: + _cache['spot'] = spot + _cache['ts'] = time.time() + print(f"[ec2_pricing] Cache refreshed: {len(ondemand)} on-demand, {len(spot)} spot rates") + else: + print("[ec2_pricing] No pricing data returned, keeping existing cache/fallbacks") + except Exception as e: + print(f"[ec2_pricing] Cache refresh error: {e}") + finally: + _cache_lock.release() + + +def _ensure_cached(): + """Ensure cache is populated. Blocks on first call, async refresh after.""" + if not _cache['ts']: + _refresh_cache() # block on first load + else: + threading.Thread(target=_refresh_cache, daemon=True).start() + + +# ---- Public API ---- + +def get_instance_rate(instance_type: str, is_spot: bool) -> float: + """Get the hourly rate for an EC2 instance type. + + Tries live pricing cache first, falls back to hardcoded rates. + + Args: + instance_type: EC2 instance type (e.g. 'm6a.48xlarge') + is_spot: True for spot pricing, False for on-demand + + Returns: + Hourly rate in USD. + """ + _ensure_cached() + + # Try live cache + cache_key = 'spot' if is_spot else 'ondemand' + rate = _cache[cache_key].get(instance_type) + if rate is not None: + return rate + + # Fall back to hardcoded + rate = _HARDCODED_RATES.get((instance_type, is_spot)) + if rate is not None: + return rate + + # Unknown instance type -- return 0 (caller should use vCPU fallback) + return 0.0 + + +def get_fallback_vcpu_rate(is_spot: bool) -> float: + """Get the per-vCPU hourly rate for unknown instance types. + + Args: + is_spot: True for spot, False for on-demand + + Returns: + Per-vCPU hourly rate in USD. + """ + return _FALLBACK_VCPU_HOUR[is_spot] diff --git a/ci3/ci-metrics/github_data.py b/ci3/ci-metrics/github_data.py new file mode 100644 index 000000000000..8824d187cb81 --- /dev/null +++ b/ci3/ci-metrics/github_data.py @@ -0,0 +1,666 @@ +"""GitHub API polling with in-memory cache. + +Fetches PR lifecycle, deployment runs, branch lag, and merge queue stats via `gh` CLI. +Most data cached in memory with TTL. Merge queue stats persisted to SQLite daily. +""" +import json +import subprocess +import threading +import time +from datetime import datetime, timedelta, timezone + +REPO = 'AztecProtocol/aztec-packages' + +BRANCH_PAIRS = [ + ('next', 'staging-public'), + ('next', 'testnet'), + ('staging-public', 'testnet'), +] + +DEPLOY_WORKFLOWS = [ + 'deploy-staging-networks.yml', + 'deploy-network.yml', + 'deploy-next-net.yml', +] + +_CACHE_TTL = 3600 # 1 hour +_pr_cache = {'data': [], 'ts': 0} +_deploy_cache = {'data': [], 'ts': 0} +_lag_cache = {'data': [], 'ts': 0} +_pr_author_cache = {} # {pr_number: {'author': str, 'title': str, 'branch': str}} +_pr_lock = threading.Lock() +_deploy_lock = threading.Lock() +_lag_lock = threading.Lock() + + +def _gh(args: list[str]) -> str | None: + try: + result = subprocess.run( + ['gh'] + args, + capture_output=True, text=True, timeout=30 + ) + if result.returncode == 0: + return result.stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired) as e: + print(f"[rk_github] gh error: {e}") + return None + + +# ---- PR lifecycle ---- + +def _fetch_and_process_prs() -> list[dict]: + out = _gh([ + 'pr', 'list', '--repo', REPO, '--state', 'merged', + '--limit', '500', + '--json', 'number,author,title,createdAt,mergedAt,closedAt,baseRefName,' + 'headRefName,additions,deletions,changedFiles,isDraft,reviewDecision,labels' + ]) + if not out: + return [] + try: + prs = json.loads(out) + except json.JSONDecodeError: + return [] + + for pr in prs: + author = pr.get('author', {}) + if isinstance(author, dict): + pr['author'] = author.get('login', 'unknown') + # Extract label names from label objects + labels = pr.get('labels', []) + if labels and isinstance(labels[0], dict): + pr['labels'] = [l.get('name', '') for l in labels] + created = pr.get('createdAt', '') + merged = pr.get('mergedAt') + if created and merged: + try: + c = datetime.fromisoformat(created.replace('Z', '+00:00')) + m = datetime.fromisoformat(merged.replace('Z', '+00:00')) + pr['merge_time_hrs'] = round((m - c).total_seconds() / 3600, 2) + except (ValueError, TypeError): + pr['merge_time_hrs'] = None + else: + pr['merge_time_hrs'] = None + pr['merged_date'] = merged[:10] if merged else None + pr['size'] = (pr.get('additions', 0) or 0) + (pr.get('deletions', 0) or 0) + return prs + + +def _ensure_prs(): + now = time.time() + if _pr_cache['data'] and now - _pr_cache['ts'] < _CACHE_TTL: + return + if not _pr_lock.acquire(blocking=False): + return + try: + prs = _fetch_and_process_prs() + if prs: + _pr_cache['data'] = prs + _pr_cache['ts'] = now + finally: + _pr_lock.release() + + +# ---- Deployments ---- + +def _fetch_all_deploys() -> list[dict]: + all_runs = [] + for workflow in DEPLOY_WORKFLOWS: + out = _gh([ + 'run', 'list', '--repo', REPO, + '--workflow', workflow, '--limit', '50', + '--json', 'databaseId,status,conclusion,createdAt,updatedAt,headBranch,name' + ]) + if not out: + continue + try: + runs = json.loads(out) + except json.JSONDecodeError: + continue + for run in runs: + started = run.get('createdAt', '') + completed = run.get('updatedAt') + duration = None + if started and completed: + try: + s = datetime.fromisoformat(started.replace('Z', '+00:00')) + c = datetime.fromisoformat(completed.replace('Z', '+00:00')) + duration = round((c - s).total_seconds(), 1) + except (ValueError, TypeError): + pass + all_runs.append({ + 'run_id': str(run.get('databaseId', '')), + 'workflow_name': workflow.replace('.yml', ''), + 'ref_name': run.get('headBranch', ''), + 'status': run.get('conclusion', run.get('status', 'unknown')), + 'started_at': started, + 'completed_at': completed, + 'duration_secs': duration, + 'started_date': started[:10] if started else None, + }) + return all_runs + + +def _ensure_deploys(): + now = time.time() + if _deploy_cache['data'] and now - _deploy_cache['ts'] < _CACHE_TTL: + return + if not _deploy_lock.acquire(blocking=False): + return + try: + deploys = _fetch_all_deploys() + if deploys: + _deploy_cache['data'] = deploys + _deploy_cache['ts'] = now + finally: + _deploy_lock.release() + + +# ---- Branch lag ---- + +def _fetch_branch_lag() -> list[dict]: + results = [] + today = datetime.now(timezone.utc).date().isoformat() + for source, target in BRANCH_PAIRS: + out = _gh([ + 'api', f'repos/{REPO}/compare/{target}...{source}', + '--jq', '.ahead_by' + ]) + if not out: + continue + try: + commits_behind = int(out) + except (ValueError, TypeError): + continue + + days_behind = None + out2 = _gh([ + 'api', f'repos/{REPO}/compare/{target}...{source}', + '--jq', '.commits[0].commit.committer.date' + ]) + if out2: + try: + oldest = datetime.fromisoformat(out2.replace('Z', '+00:00')) + days_behind = round((datetime.now(timezone.utc) - oldest).total_seconds() / 86400, 1) + except (ValueError, TypeError): + pass + + results.append({ + 'date': today, + 'source': source, + 'target': target, + 'commits_behind': commits_behind, + 'days_behind': days_behind, + }) + return results + + +def _ensure_lag(): + now = time.time() + if _lag_cache['data'] and now - _lag_cache['ts'] < _CACHE_TTL: + return + if not _lag_lock.acquire(blocking=False): + return + try: + lag = _fetch_branch_lag() + if lag: + _lag_cache['data'] = lag + _lag_cache['ts'] = now + finally: + _lag_lock.release() + + +# ---- Query functions for API endpoints ---- + +def get_deployment_speed(date_from: str, date_to: str, workflow: str = '') -> dict: + if not _deploy_cache['data']: + _ensure_deploys() + else: + threading.Thread(target=_ensure_deploys, daemon=True).start() + deploys = [d for d in _deploy_cache['data'] + if d.get('started_date') and date_from <= d['started_date'] <= date_to] + if workflow: + deploys = [d for d in deploys if d['workflow_name'] == workflow] + + # Group by date + by_date_map = {} + for d in deploys: + date = d['started_date'] + if date not in by_date_map: + by_date_map[date] = {'durations': [], 'success': 0, 'failure': 0, 'count': 0} + by_date_map[date]['count'] += 1 + if d['duration_secs'] is not None: + by_date_map[date]['durations'].append(d['duration_secs'] / 60.0) + if d['status'] == 'success': + by_date_map[date]['success'] += 1 + elif d['status'] == 'failure': + by_date_map[date]['failure'] += 1 + + by_date = [] + for date in sorted(by_date_map): + b = by_date_map[date] + durs = sorted(b['durations']) + by_date.append({ + 'date': date, + 'median_mins': round(durs[len(durs)//2], 1) if durs else None, + 'p95_mins': round(durs[int(len(durs)*0.95)], 1) if durs else None, + 'count': b['count'], + 'success': b['success'], + 'failure': b['failure'], + }) + + all_durs = sorted([d['duration_secs']/60.0 for d in deploys if d['duration_secs'] is not None]) + total = len(deploys) + success = sum(1 for d in deploys if d['status'] == 'success') + + recent = [{'run_id': d['run_id'], 'workflow_name': d['workflow_name'], + 'status': d['status'], 'duration_mins': round(d['duration_secs']/60.0, 1) if d['duration_secs'] else None, + 'started_at': d['started_at'], 'ref_name': d['ref_name']} + for d in sorted(deploys, key=lambda x: x['started_at'], reverse=True)[:50]] + + return { + 'by_date': by_date, + 'summary': { + 'median_mins': round(all_durs[len(all_durs)//2], 1) if all_durs else None, + 'p95_mins': round(all_durs[int(len(all_durs)*0.95)], 1) if all_durs else None, + 'success_rate': round(100.0 * success / max(total, 1), 1), + 'total': total, + }, + 'recent': recent, + } + + +def get_branch_lag(date_from: str, date_to: str) -> dict: + if not _lag_cache['data']: + _ensure_lag() + else: + threading.Thread(target=_ensure_lag, daemon=True).start() + pairs = [] + for source, target in BRANCH_PAIRS: + matching = [l for l in _lag_cache['data'] + if l['source'] == source and l['target'] == target] + current = matching[-1] if matching else {'commits_behind': 0, 'days_behind': 0} + pairs.append({ + 'source': source, + 'target': target, + 'current': {'commits_behind': current.get('commits_behind', 0), + 'days_behind': current.get('days_behind', 0)}, + 'history': [{'date': l['date'], 'commits_behind': l['commits_behind'], + 'days_behind': l['days_behind']} for l in matching], + }) + return {'pairs': pairs} + + +def get_pr_author(pr_number) -> dict | None: + """Look up PR author/title by number. Results are cached permanently (PR data doesn't change).""" + pr_number = int(pr_number) if pr_number else None + if not pr_number: + return None + if pr_number in _pr_author_cache: + return _pr_author_cache[pr_number] + + # Check merged PR cache first (already fetched) + for pr in _pr_cache.get('data', []): + if pr.get('number') == pr_number: + info = {'author': pr.get('author', 'unknown'), 'title': pr.get('title', ''), + 'branch': pr.get('headRefName', ''), + 'additions': pr.get('additions', 0), 'deletions': pr.get('deletions', 0)} + _pr_author_cache[pr_number] = info + return info + + # Fetch from GitHub API + out = _gh(['pr', 'view', str(pr_number), '--repo', REPO, + '--json', 'author,title,headRefName,additions,deletions']) + if out: + try: + data = json.loads(out) + author = data.get('author', {}) + if isinstance(author, dict): + author = author.get('login', 'unknown') + info = {'author': author, 'title': data.get('title', ''), + 'branch': data.get('headRefName', ''), + 'additions': data.get('additions', 0), 'deletions': data.get('deletions', 0)} + _pr_author_cache[pr_number] = info + return info + except (json.JSONDecodeError, KeyError): + pass + return None + + +def batch_get_pr_authors(pr_numbers: set) -> dict: + """Fetch authors for multiple PR numbers, using cache. Returns {pr_number: info}.""" + result = {} + to_fetch = [] + for prn in pr_numbers: + if not prn: + continue + prn = int(prn) + if prn in _pr_author_cache: + result[prn] = _pr_author_cache[prn] + else: + to_fetch.append(prn) + + # Check merged PR cache first + for pr in _pr_cache.get('data', []): + num = pr.get('number') + if num in to_fetch: + info = {'author': pr.get('author', 'unknown'), 'title': pr.get('title', ''), + 'branch': pr.get('headRefName', ''), + 'additions': pr.get('additions', 0), 'deletions': pr.get('deletions', 0)} + _pr_author_cache[num] = info + result[num] = info + to_fetch.remove(num) + + # Fetch remaining individually (with a cap to avoid API abuse) + for prn in to_fetch[:50]: + info = get_pr_author(prn) + if info: + result[prn] = info + + return result + + +def get_branch_pr_map() -> dict: + """Return {branch_name: pr_number} from the PR cache. Call _ensure_prs first.""" + if not _pr_cache['data']: + _ensure_prs() + else: + threading.Thread(target=_ensure_prs, daemon=True).start() + return {pr['headRefName']: pr['number'] + for pr in _pr_cache.get('data', []) + if pr.get('headRefName')} + + +def get_pr_metrics(date_from: str, date_to: str, author: str = '', + ci_runs: list = None) -> dict: + """Get PR metrics. ci_runs should be passed from the caller (read from Redis).""" + if not _pr_cache['data']: + _ensure_prs() + else: + threading.Thread(target=_ensure_prs, daemon=True).start() + + prs = [p for p in _pr_cache['data'] + if p.get('merged_date') and date_from <= p['merged_date'] <= date_to] + if author: + prs = [p for p in prs if p.get('author') == author] + + # Compute per-PR CI cost and duration from ci_runs + pr_costs = {} + pr_run_counts = {} + pr_ci_time = {} # total CI compute hours per PR + if ci_runs: + for run in ci_runs: + prn = run.get('pr_number') + if not prn: + continue + if run.get('cost_usd') is not None: + pr_costs[prn] = pr_costs.get(prn, 0) + run['cost_usd'] + pr_run_counts[prn] = pr_run_counts.get(prn, 0) + 1 + c = run.get('complete') + t = run.get('timestamp') + if c and t: + pr_ci_time[prn] = pr_ci_time.get(prn, 0) + (c - t) / 3_600_000 + + for pr in prs: + prn = pr.get('number') + pr['ci_cost_usd'] = round(pr_costs.get(prn, 0), 2) + pr['ci_runs_count'] = pr_run_counts.get(prn, 0) + pr['ci_time_hrs'] = round(pr_ci_time.get(prn, 0), 2) + + # Group by date + by_date_map = {} + for pr in prs: + date = pr['merged_date'] + if date not in by_date_map: + by_date_map[date] = {'costs': [], 'merge_times': [], 'ci_times': [], + 'run_counts': [], 'count': 0} + by_date_map[date]['count'] += 1 + by_date_map[date]['costs'].append(pr['ci_cost_usd']) + by_date_map[date]['ci_times'].append(pr.get('ci_time_hrs', 0)) + by_date_map[date]['run_counts'].append(pr.get('ci_runs_count', 0)) + if pr.get('merge_time_hrs') is not None: + by_date_map[date]['merge_times'].append(pr['merge_time_hrs']) + + def _median(vals): + s = sorted(vals) + n = len(s) + if n == 0: + return None + if n % 2 == 1: + return s[n // 2] + return (s[n // 2 - 1] + s[n // 2]) / 2 + + by_date = [] + for d, v in sorted(by_date_map.items()): + by_date.append({ + 'date': d, + 'pr_count': v['count'], + 'avg_cost': round(sum(v['costs']) / max(len(v['costs']), 1), 2), + 'median_merge_time_hrs': round(_median(v['merge_times']), 1) if v['merge_times'] else None, + 'avg_ci_time_hrs': round(sum(v['ci_times']) / max(len(v['ci_times']), 1), 2), + 'avg_runs': round(sum(v['run_counts']) / max(len(v['run_counts']), 1), 1), + }) + + # By author (all PRs in range, not filtered by author) + all_prs_in_range = [p for p in _pr_cache['data'] + if p.get('merged_date') and date_from <= p['merged_date'] <= date_to] + + author_map = {} + for pr in all_prs_in_range: + prn = pr.get('number') + a = pr.get('author', 'unknown') + if a not in author_map: + author_map[a] = {'total_cost': 0, 'pr_count': 0, 'merge_times': [], + 'total_ci_time': 0, 'total_runs': 0} + author_map[a]['total_cost'] += round(pr_costs.get(prn, 0), 2) + author_map[a]['pr_count'] += 1 + author_map[a]['total_ci_time'] += round(pr_ci_time.get(prn, 0), 2) + author_map[a]['total_runs'] += pr_run_counts.get(prn, 0) + if pr.get('merge_time_hrs') is not None: + author_map[a]['merge_times'].append(pr['merge_time_hrs']) + + by_author = [] + for a, v in sorted(author_map.items(), key=lambda x: -x[1]['total_cost'])[:20]: + by_author.append({ + 'author': a, + 'total_cost': round(v['total_cost'], 2), + 'pr_count': v['pr_count'], + 'avg_merge_time_hrs': round(_median(v['merge_times']), 1) if v['merge_times'] else None, + 'avg_ci_time_hrs': round(v['total_ci_time'] / max(v['pr_count'], 1), 2), + 'avg_runs_per_pr': round(v['total_runs'] / max(v['pr_count'], 1), 1), + }) + + all_costs = [p.get('ci_cost_usd', 0) for p in prs] + all_merge = [p['merge_time_hrs'] for p in prs if p.get('merge_time_hrs') is not None] + all_run_counts = [p.get('ci_runs_count', 0) for p in prs] + all_ci_times = [p.get('ci_time_hrs', 0) for p in prs] + + return { + 'by_date': by_date, + 'by_author': by_author, + 'summary': { + 'avg_cost_per_pr': round(sum(all_costs)/max(len(all_costs),1), 2) if all_costs else 0, + 'median_merge_time_hrs': round(_median(all_merge), 1) if all_merge else None, + 'total_prs': len(prs), + 'total_cost': round(sum(all_costs), 2), + 'avg_ci_runs_per_pr': round(sum(all_run_counts)/max(len(all_run_counts),1), 1) if all_run_counts else 0, + 'avg_ci_time_hrs': round(sum(all_ci_times)/max(len(all_ci_times),1), 2) if all_ci_times else 0, + }, + } + + +# ---- Merge queue failure rate ---- + +CI3_WORKFLOW = 'ci3.yml' + +def _fetch_merge_queue_runs(date_str: str) -> dict: + """Fetch merge_group workflow runs for a single date. Returns daily summary.""" + out = _gh([ + 'api', '--paginate', + f'repos/{REPO}/actions/workflows/{CI3_WORKFLOW}/runs' + f'?event=merge_group&created={date_str}&per_page=100', + '--jq', '.workflow_runs[] | [.conclusion, .status] | @tsv', + ]) + summary = {'date': date_str, 'total': 0, 'success': 0, 'failure': 0, + 'cancelled': 0, 'in_progress': 0} + if not out: + return summary + for line in out.strip().split('\n'): + if not line.strip(): + continue + parts = line.split('\t') + conclusion = parts[0] if parts[0] else '' + status = parts[1] if len(parts) > 1 else '' + summary['total'] += 1 + if conclusion == 'success': + summary['success'] += 1 + elif conclusion == 'failure': + summary['failure'] += 1 + elif conclusion == 'cancelled': + summary['cancelled'] += 1 + elif status in ('in_progress', 'queued', 'waiting'): + summary['in_progress'] += 1 + else: + summary['failure'] += 1 # treat unknown conclusions as failures + return summary + + +def _load_backfill_json(): + """Load seed data from merge-queue-backfill.json if SQLite is empty.""" + import db + from pathlib import Path + conn = db.get_db() + + count = conn.execute('SELECT COUNT(*) as c FROM merge_queue_daily').fetchone()['c'] + if count > 0: + return + + seed = Path(__file__).parent / 'merge-queue-backfill.json' + if not seed.exists(): + return + + import json + with seed.open() as f: + data = json.load(f) + + print(f"[rk_github] Loading {len(data)} days from merge-queue-backfill.json...") + for ds, summary in data.items(): + conn.execute( + 'INSERT OR REPLACE INTO merge_queue_daily (date, total, success, failure, cancelled, in_progress) ' + 'VALUES (?, ?, ?, ?, ?, ?)', + (ds, summary['total'], summary['success'], summary['failure'], + summary['cancelled'], summary['in_progress'])) + conn.commit() + + +def _backfill_merge_queue(): + """Backfill missing merge queue daily stats into SQLite.""" + import db + conn = db.get_db() + + # Load seed data on first run + _load_backfill_json() + + # Find which dates we already have + existing = {row['date'] for row in + conn.execute('SELECT date FROM merge_queue_daily').fetchall()} + + yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).date() + # Backfill up to 365 days + start = yesterday - timedelta(days=365) + current = start + + missing = [] + while current <= yesterday: + ds = current.isoformat() + if ds not in existing: + missing.append(ds) + current += timedelta(days=1) + + if not missing: + return + + print(f"[rk_github] Backfilling {len(missing)} days of merge queue stats...") + for ds in missing: + summary = _fetch_merge_queue_runs(ds) + if summary['total'] == 0: + conn.execute( + 'INSERT OR REPLACE INTO merge_queue_daily (date, total, success, failure, cancelled, in_progress) ' + 'VALUES (?, 0, 0, 0, 0, 0)', (ds,)) + else: + conn.execute( + 'INSERT OR REPLACE INTO merge_queue_daily (date, total, success, failure, cancelled, in_progress) ' + 'VALUES (?, ?, ?, ?, ?, ?)', + (ds, summary['total'], summary['success'], summary['failure'], + summary['cancelled'], summary['in_progress'])) + conn.commit() + + +def refresh_merge_queue_today(): + """Refresh today's (and yesterday's) merge queue stats. Called periodically.""" + import db + conn = db.get_db() + today = datetime.now(timezone.utc).date().isoformat() + yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).date().isoformat() + + for ds in [yesterday, today]: + summary = _fetch_merge_queue_runs(ds) + conn.execute( + 'INSERT OR REPLACE INTO merge_queue_daily (date, total, success, failure, cancelled, in_progress) ' + 'VALUES (?, ?, ?, ?, ?, ?)', + (ds, summary['total'], summary['success'], summary['failure'], + summary['cancelled'], summary['in_progress'])) + conn.commit() + + +_mq_backfill_lock = threading.Lock() +_mq_last_refresh = 0 +_MQ_REFRESH_TTL = 3600 # refresh today's data every hour + + +def ensure_merge_queue_data(): + """Ensure merge queue data is backfilled and today is fresh.""" + global _mq_last_refresh + now = time.time() + if now - _mq_last_refresh < _MQ_REFRESH_TTL: + return + if not _mq_backfill_lock.acquire(blocking=False): + return + try: + _backfill_merge_queue() + refresh_merge_queue_today() + _mq_last_refresh = now + finally: + _mq_backfill_lock.release() + + +def get_merge_queue_stats(date_from: str, date_to: str) -> dict: + """Get merge queue failure rate by day. Triggers backfill if needed.""" + # Ensure data is populated (async after first load) + import db + conn = db.get_db() + count = conn.execute('SELECT COUNT(*) as c FROM merge_queue_daily').fetchone()['c'] + if count == 0: + ensure_merge_queue_data() # block on first load + else: + threading.Thread(target=ensure_merge_queue_data, daemon=True).start() + + rows = db.query( + 'SELECT date, total, success, failure, cancelled, in_progress ' + 'FROM merge_queue_daily WHERE date >= ? AND date <= ? ORDER BY date', + (date_from, date_to)) + + total_runs = sum(r['total'] for r in rows) + total_fail = sum(r['failure'] for r in rows) + total_success = sum(r['success'] for r in rows) + + return { + 'by_date': rows, + 'summary': { + 'total_runs': total_runs, + 'total_success': total_success, + 'total_failure': total_fail, + 'failure_rate': round(total_fail / max(total_runs, 1) * 100, 1), + 'days': len([r for r in rows if r['total'] > 0]), + }, + } diff --git a/ci3/ci-metrics/merge-queue-backfill.json b/ci3/ci-metrics/merge-queue-backfill.json new file mode 100644 index 000000000000..079077590581 --- /dev/null +++ b/ci3/ci-metrics/merge-queue-backfill.json @@ -0,0 +1,2564 @@ +{ + "2025-02-10": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-11": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-12": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-13": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-14": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-15": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-16": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-17": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-18": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-19": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-20": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-21": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-22": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-23": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-24": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-25": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-26": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-27": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-02-28": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-01": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-02": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-03": { + "total": 1, + "success": 0, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-04": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-05": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-06": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-07": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-08": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-09": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-10": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-11": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-12": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-13": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-14": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-15": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-16": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-17": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-18": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-19": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-20": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-21": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-22": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-23": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-24": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-25": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-26": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-27": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-28": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-29": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-30": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-03-31": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-01": { + "total": 3, + "success": 2, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-02": { + "total": 31, + "success": 19, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-03": { + "total": 113, + "success": 58, + "failure": 55, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-04": { + "total": 69, + "success": 50, + "failure": 19, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-05": { + "total": 4, + "success": 4, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-06": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-07": { + "total": 42, + "success": 32, + "failure": 10, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-08": { + "total": 27, + "success": 19, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-09": { + "total": 29, + "success": 26, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-10": { + "total": 42, + "success": 35, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-11": { + "total": 51, + "success": 36, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-12": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-13": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-14": { + "total": 24, + "success": 19, + "failure": 4, + "cancelled": 1, + "in_progress": 0 + }, + "2025-04-15": { + "total": 41, + "success": 22, + "failure": 19, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-16": { + "total": 26, + "success": 21, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-17": { + "total": 29, + "success": 28, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-18": { + "total": 10, + "success": 10, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-19": { + "total": 4, + "success": 4, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-20": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-21": { + "total": 5, + "success": 5, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-22": { + "total": 49, + "success": 33, + "failure": 15, + "cancelled": 1, + "in_progress": 0 + }, + "2025-04-23": { + "total": 32, + "success": 28, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-24": { + "total": 29, + "success": 26, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-25": { + "total": 28, + "success": 26, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-26": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-27": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-28": { + "total": 26, + "success": 20, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-29": { + "total": 60, + "success": 26, + "failure": 34, + "cancelled": 0, + "in_progress": 0 + }, + "2025-04-30": { + "total": 47, + "success": 33, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-01": { + "total": 31, + "success": 27, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-02": { + "total": 8, + "success": 8, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-03": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-04": { + "total": 7, + "success": 7, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-05": { + "total": 14, + "success": 11, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-06": { + "total": 18, + "success": 16, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-07": { + "total": 22, + "success": 20, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-08": { + "total": 18, + "success": 15, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-09": { + "total": 36, + "success": 27, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-10": { + "total": 2, + "success": 1, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-11": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-12": { + "total": 47, + "success": 30, + "failure": 17, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-13": { + "total": 134, + "success": 65, + "failure": 69, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-14": { + "total": 51, + "success": 34, + "failure": 17, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-15": { + "total": 22, + "success": 9, + "failure": 12, + "cancelled": 1, + "in_progress": 0 + }, + "2025-05-16": { + "total": 21, + "success": 15, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-17": { + "total": 2, + "success": 1, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-18": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-19": { + "total": 10, + "success": 9, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-20": { + "total": 30, + "success": 15, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-21": { + "total": 26, + "success": 12, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-22": { + "total": 51, + "success": 21, + "failure": 30, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-23": { + "total": 67, + "success": 13, + "failure": 53, + "cancelled": 1, + "in_progress": 0 + }, + "2025-05-24": { + "total": 5, + "success": 2, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-25": { + "total": 5, + "success": 0, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-26": { + "total": 10, + "success": 7, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-27": { + "total": 61, + "success": 12, + "failure": 49, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-28": { + "total": 56, + "success": 15, + "failure": 41, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-29": { + "total": 77, + "success": 24, + "failure": 52, + "cancelled": 1, + "in_progress": 0 + }, + "2025-05-30": { + "total": 25, + "success": 15, + "failure": 10, + "cancelled": 0, + "in_progress": 0 + }, + "2025-05-31": { + "total": 6, + "success": 3, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-01": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-02": { + "total": 50, + "success": 20, + "failure": 29, + "cancelled": 1, + "in_progress": 0 + }, + "2025-06-03": { + "total": 57, + "success": 22, + "failure": 35, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-04": { + "total": 219, + "success": 22, + "failure": 196, + "cancelled": 1, + "in_progress": 0 + }, + "2025-06-05": { + "total": 166, + "success": 19, + "failure": 147, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-06": { + "total": 73, + "success": 27, + "failure": 45, + "cancelled": 1, + "in_progress": 0 + }, + "2025-06-07": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-08": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-09": { + "total": 124, + "success": 31, + "failure": 93, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-10": { + "total": 44, + "success": 29, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-11": { + "total": 19, + "success": 16, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-12": { + "total": 26, + "success": 14, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-13": { + "total": 29, + "success": 24, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-14": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-15": { + "total": 1, + "success": 0, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-16": { + "total": 44, + "success": 21, + "failure": 23, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-17": { + "total": 29, + "success": 15, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-18": { + "total": 38, + "success": 25, + "failure": 13, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-19": { + "total": 15, + "success": 11, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-20": { + "total": 27, + "success": 21, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-21": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-22": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-23": { + "total": 30, + "success": 14, + "failure": 16, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-24": { + "total": 26, + "success": 17, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-25": { + "total": 26, + "success": 20, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-26": { + "total": 44, + "success": 21, + "failure": 22, + "cancelled": 1, + "in_progress": 0 + }, + "2025-06-27": { + "total": 18, + "success": 13, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-28": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-29": { + "total": 3, + "success": 3, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-06-30": { + "total": 27, + "success": 17, + "failure": 10, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-01": { + "total": 26, + "success": 12, + "failure": 13, + "cancelled": 1, + "in_progress": 0 + }, + "2025-07-02": { + "total": 42, + "success": 25, + "failure": 17, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-03": { + "total": 17, + "success": 12, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-04": { + "total": 15, + "success": 12, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-05": { + "total": 4, + "success": 3, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-06": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-07": { + "total": 20, + "success": 14, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-08": { + "total": 33, + "success": 19, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-09": { + "total": 19, + "success": 13, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-10": { + "total": 22, + "success": 14, + "failure": 7, + "cancelled": 1, + "in_progress": 0 + }, + "2025-07-11": { + "total": 6, + "success": 6, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-12": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-13": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-14": { + "total": 29, + "success": 21, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-15": { + "total": 49, + "success": 22, + "failure": 27, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-16": { + "total": 47, + "success": 21, + "failure": 26, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-17": { + "total": 18, + "success": 10, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-18": { + "total": 13, + "success": 12, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-19": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-20": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-21": { + "total": 26, + "success": 22, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-22": { + "total": 25, + "success": 19, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-23": { + "total": 33, + "success": 16, + "failure": 15, + "cancelled": 2, + "in_progress": 0 + }, + "2025-07-24": { + "total": 61, + "success": 26, + "failure": 35, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-25": { + "total": 35, + "success": 17, + "failure": 16, + "cancelled": 2, + "in_progress": 0 + }, + "2025-07-26": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-27": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-28": { + "total": 23, + "success": 22, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-29": { + "total": 52, + "success": 21, + "failure": 31, + "cancelled": 0, + "in_progress": 0 + }, + "2025-07-30": { + "total": 30, + "success": 15, + "failure": 14, + "cancelled": 1, + "in_progress": 0 + }, + "2025-07-31": { + "total": 35, + "success": 23, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-01": { + "total": 13, + "success": 13, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-02": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-03": { + "total": 4, + "success": 4, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-04": { + "total": 16, + "success": 15, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-05": { + "total": 14, + "success": 10, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-06": { + "total": 23, + "success": 16, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-07": { + "total": 19, + "success": 7, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-08": { + "total": 24, + "success": 15, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-09": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-10": { + "total": 4, + "success": 2, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-11": { + "total": 13, + "success": 12, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-12": { + "total": 9, + "success": 9, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-13": { + "total": 14, + "success": 12, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-14": { + "total": 18, + "success": 16, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-15": { + "total": 38, + "success": 30, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-16": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-17": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-18": { + "total": 19, + "success": 12, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-19": { + "total": 11, + "success": 7, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-20": { + "total": 11, + "success": 9, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-21": { + "total": 19, + "success": 15, + "failure": 3, + "cancelled": 1, + "in_progress": 0 + }, + "2025-08-22": { + "total": 32, + "success": 24, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-23": { + "total": 6, + "success": 5, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-24": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-25": { + "total": 13, + "success": 11, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-26": { + "total": 17, + "success": 10, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-27": { + "total": 20, + "success": 11, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-28": { + "total": 36, + "success": 18, + "failure": 17, + "cancelled": 1, + "in_progress": 0 + }, + "2025-08-29": { + "total": 39, + "success": 28, + "failure": 11, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-30": { + "total": 4, + "success": 2, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-08-31": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-01": { + "total": 20, + "success": 15, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-02": { + "total": 25, + "success": 16, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-03": { + "total": 30, + "success": 19, + "failure": 11, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-04": { + "total": 29, + "success": 15, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-05": { + "total": 32, + "success": 14, + "failure": 18, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-06": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-07": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-08": { + "total": 18, + "success": 12, + "failure": 5, + "cancelled": 1, + "in_progress": 0 + }, + "2025-09-09": { + "total": 25, + "success": 14, + "failure": 11, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-10": { + "total": 38, + "success": 23, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-11": { + "total": 39, + "success": 18, + "failure": 21, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-12": { + "total": 34, + "success": 21, + "failure": 13, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-13": { + "total": 1, + "success": 0, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-14": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-15": { + "total": 22, + "success": 11, + "failure": 11, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-16": { + "total": 25, + "success": 15, + "failure": 10, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-17": { + "total": 24, + "success": 17, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-18": { + "total": 24, + "success": 17, + "failure": 6, + "cancelled": 1, + "in_progress": 0 + }, + "2025-09-19": { + "total": 16, + "success": 9, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-20": { + "total": 8, + "success": 3, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-21": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-22": { + "total": 45, + "success": 19, + "failure": 26, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-23": { + "total": 23, + "success": 17, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-24": { + "total": 17, + "success": 13, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-25": { + "total": 47, + "success": 26, + "failure": 21, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-26": { + "total": 22, + "success": 21, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-27": { + "total": 4, + "success": 3, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-28": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-29": { + "total": 20, + "success": 12, + "failure": 8, + "cancelled": 0, + "in_progress": 0 + }, + "2025-09-30": { + "total": 46, + "success": 21, + "failure": 25, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-01": { + "total": 23, + "success": 16, + "failure": 6, + "cancelled": 1, + "in_progress": 0 + }, + "2025-10-02": { + "total": 30, + "success": 17, + "failure": 13, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-03": { + "total": 10, + "success": 9, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-04": { + "total": 4, + "success": 4, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-05": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-06": { + "total": 25, + "success": 9, + "failure": 15, + "cancelled": 1, + "in_progress": 0 + }, + "2025-10-07": { + "total": 42, + "success": 12, + "failure": 29, + "cancelled": 1, + "in_progress": 0 + }, + "2025-10-08": { + "total": 21, + "success": 11, + "failure": 10, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-09": { + "total": 61, + "success": 2, + "failure": 59, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-10": { + "total": 47, + "success": 13, + "failure": 34, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-11": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-12": { + "total": 1, + "success": 0, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-13": { + "total": 32, + "success": 18, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-14": { + "total": 31, + "success": 16, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-15": { + "total": 33, + "success": 22, + "failure": 11, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-16": { + "total": 19, + "success": 12, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-17": { + "total": 20, + "success": 12, + "failure": 7, + "cancelled": 1, + "in_progress": 0 + }, + "2025-10-18": { + "total": 1, + "success": 0, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-19": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-20": { + "total": 37, + "success": 14, + "failure": 23, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-21": { + "total": 21, + "success": 12, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-22": { + "total": 24, + "success": 11, + "failure": 13, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-23": { + "total": 61, + "success": 17, + "failure": 44, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-24": { + "total": 30, + "success": 18, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-25": { + "total": 3, + "success": 3, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-26": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-27": { + "total": 9, + "success": 9, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-28": { + "total": 18, + "success": 16, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-29": { + "total": 19, + "success": 14, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-30": { + "total": 17, + "success": 16, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-10-31": { + "total": 15, + "success": 14, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-01": { + "total": 4, + "success": 1, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-02": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-03": { + "total": 14, + "success": 13, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-04": { + "total": 19, + "success": 16, + "failure": 1, + "cancelled": 2, + "in_progress": 0 + }, + "2025-11-05": { + "total": 13, + "success": 10, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-06": { + "total": 24, + "success": 11, + "failure": 13, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-07": { + "total": 19, + "success": 14, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-08": { + "total": 3, + "success": 2, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-09": { + "total": 2, + "success": 1, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-10": { + "total": 47, + "success": 13, + "failure": 33, + "cancelled": 1, + "in_progress": 0 + }, + "2025-11-11": { + "total": 15, + "success": 11, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-12": { + "total": 42, + "success": 22, + "failure": 20, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-13": { + "total": 17, + "success": 12, + "failure": 4, + "cancelled": 1, + "in_progress": 0 + }, + "2025-11-14": { + "total": 22, + "success": 15, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-15": { + "total": 3, + "success": 3, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-16": { + "total": 3, + "success": 3, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-17": { + "total": 9, + "success": 7, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-18": { + "total": 19, + "success": 12, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-19": { + "total": 18, + "success": 13, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-20": { + "total": 9, + "success": 8, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-21": { + "total": 16, + "success": 12, + "failure": 3, + "cancelled": 1, + "in_progress": 0 + }, + "2025-11-22": { + "total": 5, + "success": 2, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-23": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-24": { + "total": 8, + "success": 7, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-25": { + "total": 11, + "success": 10, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-26": { + "total": 17, + "success": 16, + "failure": 0, + "cancelled": 1, + "in_progress": 0 + }, + "2025-11-27": { + "total": 17, + "success": 15, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-28": { + "total": 11, + "success": 6, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-29": { + "total": 2, + "success": 2, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-11-30": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-01": { + "total": 13, + "success": 12, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-02": { + "total": 8, + "success": 8, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-03": { + "total": 17, + "success": 10, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-04": { + "total": 11, + "success": 8, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-05": { + "total": 12, + "success": 11, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-06": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-07": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-08": { + "total": 17, + "success": 14, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-09": { + "total": 23, + "success": 14, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-10": { + "total": 43, + "success": 21, + "failure": 20, + "cancelled": 2, + "in_progress": 0 + }, + "2025-12-11": { + "total": 28, + "success": 19, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-12": { + "total": 14, + "success": 12, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-13": { + "total": 2, + "success": 0, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-14": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-15": { + "total": 41, + "success": 15, + "failure": 26, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-16": { + "total": 25, + "success": 21, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-17": { + "total": 10, + "success": 8, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-18": { + "total": 20, + "success": 14, + "failure": 5, + "cancelled": 1, + "in_progress": 0 + }, + "2025-12-19": { + "total": 13, + "success": 11, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-20": { + "total": 7, + "success": 3, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-21": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-22": { + "total": 20, + "success": 16, + "failure": 3, + "cancelled": 1, + "in_progress": 0 + }, + "2025-12-23": { + "total": 28, + "success": 19, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-24": { + "total": 13, + "success": 8, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-25": { + "total": 3, + "success": 1, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-26": { + "total": 6, + "success": 3, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-27": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-28": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-29": { + "total": 4, + "success": 2, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-30": { + "total": 3, + "success": 1, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2025-12-31": { + "total": 2, + "success": 1, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-01": { + "total": 2, + "success": 1, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-02": { + "total": 12, + "success": 8, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-03": { + "total": 3, + "success": 1, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-04": { + "total": 3, + "success": 3, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-05": { + "total": 34, + "success": 27, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-06": { + "total": 45, + "success": 25, + "failure": 20, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-07": { + "total": 17, + "success": 13, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-08": { + "total": 36, + "success": 24, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-09": { + "total": 25, + "success": 17, + "failure": 7, + "cancelled": 1, + "in_progress": 0 + }, + "2026-01-10": { + "total": 5, + "success": 2, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-11": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-12": { + "total": 32, + "success": 17, + "failure": 15, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-13": { + "total": 44, + "success": 22, + "failure": 22, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-14": { + "total": 114, + "success": 32, + "failure": 82, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-15": { + "total": 54, + "success": 22, + "failure": 31, + "cancelled": 1, + "in_progress": 0 + }, + "2026-01-16": { + "total": 70, + "success": 27, + "failure": 40, + "cancelled": 3, + "in_progress": 0 + }, + "2026-01-17": { + "total": 6, + "success": 4, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-18": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-19": { + "total": 28, + "success": 25, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-20": { + "total": 42, + "success": 30, + "failure": 12, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-21": { + "total": 51, + "success": 31, + "failure": 20, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-22": { + "total": 32, + "success": 25, + "failure": 5, + "cancelled": 2, + "in_progress": 0 + }, + "2026-01-23": { + "total": 28, + "success": 25, + "failure": 3, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-24": { + "total": 6, + "success": 4, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-25": { + "total": 3, + "success": 2, + "failure": 1, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-26": { + "total": 89, + "success": 33, + "failure": 56, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-27": { + "total": 24, + "success": 21, + "failure": 2, + "cancelled": 1, + "in_progress": 0 + }, + "2026-01-28": { + "total": 48, + "success": 28, + "failure": 20, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-29": { + "total": 24, + "success": 18, + "failure": 6, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-30": { + "total": 31, + "success": 24, + "failure": 7, + "cancelled": 0, + "in_progress": 0 + }, + "2026-01-31": { + "total": 1, + "success": 1, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-01": { + "total": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-02": { + "total": 14, + "success": 12, + "failure": 2, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-03": { + "total": 27, + "success": 18, + "failure": 9, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-04": { + "total": 30, + "success": 16, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-05": { + "total": 33, + "success": 19, + "failure": 14, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-06": { + "total": 20, + "success": 15, + "failure": 5, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-07": { + "total": 8, + "success": 4, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-08": { + "total": 5, + "success": 2, + "failure": 2, + "cancelled": 1, + "in_progress": 0 + }, + "2026-02-09": { + "total": 15, + "success": 11, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + }, + "2026-02-10": { + "total": 24, + "success": 20, + "failure": 4, + "cancelled": 0, + "in_progress": 0 + } +} \ No newline at end of file diff --git a/ci3/ci-metrics/metrics.py b/ci3/ci-metrics/metrics.py new file mode 100644 index 000000000000..5c0d1610e06b --- /dev/null +++ b/ci3/ci-metrics/metrics.py @@ -0,0 +1,602 @@ +"""CI metrics: direct Redis reads + test event listener. + +Reads CI run data directly from Redis sorted sets on each request. +Test events stored in SQLite since they only arrive via pub/sub. +CI runs periodically synced from Redis to SQLite for flake correlation. +""" +import json +import re +import time +import threading +from datetime import datetime, timedelta, timezone + +import db +import github_data +import ec2_pricing + +SECTIONS = ['next', 'prs', 'master', 'staging', 'releases', 'nightly', 'network', 'deflake', 'local'] + +_PR_RE = re.compile(r'(?:pr-|#)(\d+)', re.IGNORECASE) +_ANSI_RE = re.compile(r'\x1b\[[^m]*m|\x1b\]8;;[^\x07]*\x07') +_URL_PR_RE = re.compile(r'/pull/(\d+)') + + +def compute_run_cost(data: dict) -> float | None: + complete = data.get('complete') + ts = data.get('timestamp') + if not complete or not ts: + return None + hours = (complete - ts) / 3_600_000 + instance_type = data.get('instance_type', 'unknown') + is_spot = bool(data.get('spot')) + rate = ec2_pricing.get_instance_rate(instance_type, is_spot) + if not rate: + vcpus = data.get('instance_vcpus', 192) + rate = vcpus * ec2_pricing.get_fallback_vcpu_rate(is_spot) + return round(hours * rate, 4) + + +def extract_pr_number(name: str) -> int | None: + m = _PR_RE.search(name) + if m: + return int(m.group(1)) + # Try matching GitHub PR URL in ANSI-encoded strings + m = _URL_PR_RE.search(name) + if m: + return int(m.group(1)) + # Strip ANSI codes and retry + clean = _ANSI_RE.sub('', name) + m = _PR_RE.search(clean) + return int(m.group(1)) if m else None + + +def _get_ci_runs_from_redis(redis_conn, date_from_ms=None, date_to_ms=None): + """Read CI runs from Redis sorted sets.""" + branch_pr_map = github_data.get_branch_pr_map() + + runs = [] + for section in SECTIONS: + key = f'ci-run-{section}' + try: + if date_from_ms is not None or date_to_ms is not None: + lo = date_from_ms if date_from_ms is not None else '-inf' + hi = date_to_ms if date_to_ms is not None else '+inf' + entries = redis_conn.zrangebyscore(key, lo, hi, withscores=True) + else: + entries = redis_conn.zrange(key, 0, -1, withscores=True) + for entry_bytes, score in entries: + try: + raw = entry_bytes.decode() if isinstance(entry_bytes, bytes) else entry_bytes + data = json.loads(raw) + data.setdefault('dashboard', section) + data['cost_usd'] = compute_run_cost(data) + data['pr_number'] = ( + extract_pr_number(data.get('name', '')) + or extract_pr_number(data.get('msg', '')) + or (int(data['pr_number']) if data.get('pr_number') else None) + or branch_pr_map.get(data.get('name')) + ) + runs.append(data) + except Exception: + continue + except Exception as e: + print(f"[rk_metrics] Error reading {key}: {e}") + return runs + + +def _get_ci_runs_from_sqlite(date_from_ms=None, date_to_ms=None): + """Read CI runs from SQLite (persistent store).""" + conditions = [] + params = [] + if date_from_ms is not None: + conditions.append('timestamp_ms >= ?') + params.append(date_from_ms) + if date_to_ms is not None: + conditions.append('timestamp_ms <= ?') + params.append(date_to_ms) + where = ('WHERE ' + ' AND '.join(conditions)) if conditions else '' + rows = db.query(f'SELECT * FROM ci_runs {where} ORDER BY timestamp_ms', params) + runs = [] + for row in rows: + runs.append({ + 'dashboard': row['dashboard'], + 'name': row['name'], + 'timestamp': row['timestamp_ms'], + 'complete': row['complete_ms'], + 'status': row['status'], + 'author': row['author'], + 'pr_number': row['pr_number'], + 'instance_type': row['instance_type'], + 'instance_vcpus': row.get('instance_vcpus'), + 'spot': bool(row['spot']), + 'cost_usd': row['cost_usd'], + 'job_id': row.get('job_id', ''), + 'arch': row.get('arch', ''), + }) + return runs + + +def get_ci_runs(redis_conn, date_from_ms=None, date_to_ms=None): + """Read CI runs from Redis, backfilled with SQLite for data that Redis has flushed.""" + redis_runs = _get_ci_runs_from_redis(redis_conn, date_from_ms, date_to_ms) + + # Find the earliest timestamp in Redis to know what SQLite needs to fill + redis_keys = set() + redis_min_ts = float('inf') + for run in redis_runs: + ts = run.get('timestamp', 0) + redis_keys.add((run.get('dashboard', ''), ts, run.get('name', ''))) + if ts < redis_min_ts: + redis_min_ts = ts + + # If requesting data older than what Redis has, backfill from SQLite + sqlite_runs = [] + need_sqlite = (date_from_ms is not None and date_from_ms < redis_min_ts) or not redis_runs + if need_sqlite: + sqlite_to = int(redis_min_ts) if redis_runs else date_to_ms + sqlite_runs = _get_ci_runs_from_sqlite(date_from_ms, sqlite_to) + # Deduplicate: only include SQLite runs not already in Redis + sqlite_runs = [r for r in sqlite_runs + if (r.get('dashboard', ''), r.get('timestamp', 0), r.get('name', '')) + not in redis_keys] + + return sqlite_runs + redis_runs + + +def _ts_to_date(ts_ms): + return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc).strftime('%Y-%m-%d') + + +# ---- Test event handling (only thing needing SQLite) ---- + +def _handle_test_event(channel: str, data: dict): + status = channel.split(':')[-1] + # Handle field name mismatches: run_test_cmd publishes 'cmd' for failed/flaked + # but 'test_cmd' for started events. Same for 'log_key' vs 'log_url'. + test_cmd = data.get('test_cmd') or data.get('cmd', '') + log_url = data.get('log_url') or data.get('log_key') + if log_url and not log_url.startswith('http'): + log_url = f'http://ci.aztec-labs.com/{log_url}' + db.execute(''' + INSERT INTO test_events + (status, test_cmd, log_url, ref_name, commit_hash, commit_author, + commit_msg, exit_code, duration_secs, is_scenario, owners, + flake_group_id, dashboard, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + status, + test_cmd, + log_url, + data.get('ref_name', ''), + data.get('commit_hash'), + data.get('commit_author'), + data.get('commit_msg'), + data.get('exit_code'), + data.get('duration_seconds'), + 1 if data.get('is_scenario_test') else 0, + json.dumps(data['owners']) if data.get('owners') else None, + data.get('flake_group_id'), + data.get('dashboard', ''), + data.get('timestamp', datetime.now(timezone.utc).isoformat()), + )) + + +def start_test_listener(redis_conn): + """Subscribe to test event channels only. Reconnects on failure.""" + channels = [b'ci:test:started', b'ci:test:passed', b'ci:test:failed', b'ci:test:flaked'] + + def listener(): + backoff = 1 + while True: + try: + pubsub = redis_conn.pubsub() + pubsub.subscribe(*channels) + backoff = 1 # reset on successful connection + for message in pubsub.listen(): + if message['type'] != 'message': + continue + channel = message['channel'] + if isinstance(channel, bytes): + channel = channel.decode() + try: + payload = message['data'] + if isinstance(payload, bytes): + payload = payload.decode() + _handle_test_event(channel, json.loads(payload)) + except Exception as e: + print(f"[rk_metrics] Error parsing test event: {e}") + except Exception as e: + print(f"[rk_metrics] Test listener error (reconnecting in {backoff}s): {e}") + time.sleep(backoff) + backoff = min(backoff * 2, 60) + + t = threading.Thread(target=listener, daemon=True, name='test-listener') + t.start() + return t + + +# ---- Sync failed_tests_{section} lists from Redis into SQLite ---- + +_ANSI_STRIP = re.compile(r'\x1b\[[^m]*m|\x1b\]8;;[^\x07]*\x07') +_GRIND_CMD_RE = re.compile(r'/grind\?cmd=([^&\x07"]+)') +_LOG_KEY_RE = re.compile(r'ci\.aztec-labs\.com/([a-f0-9]{16})') +_INLINE_CMD_RE = re.compile(r'(?:grind\)|[0-9a-f]{16}\)):?\s+(.+?)\s+\(\d+s\)') +_DURATION_RE = re.compile(r'\((\d+)s\)') +_AUTHOR_MSG_RE = re.compile(r'\(code: \d+\)\s+\((.+?): (.+?)\)\s*$') +_FLAKE_GROUP_RE = re.compile(r'group:(\S+)') + +_failed_tests_sync_ts = 0 +_FAILED_TESTS_SYNC_TTL = 3600 # 1 hour + + +def _parse_failed_test_entry(raw: str, section: str) -> dict | None: + """Parse an ANSI-formatted failed_tests_{section} entry into structured data.""" + from urllib.parse import unquote + clean = _ANSI_STRIP.sub('', raw) + + # Status + if 'FLAKED' in clean: + status = 'flaked' + elif 'FAILED' in clean: + status = 'failed' + else: + return None + + # Timestamp: "02-11 15:11:00: ..." + ts_match = re.match(r'(\d{2}-\d{2} \d{2}:\d{2}:\d{2})', clean) + if not ts_match: + return None + # Assume current year for MM-DD HH:MM:SS; handle year rollover + now = datetime.now(timezone.utc) + year = now.year + ts_str = f'{year}-{ts_match.group(1)}' + try: + parsed_dt = datetime.strptime(ts_str, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc) + # If parsed date is in the future, it's from the previous year + if parsed_dt > now + timedelta(days=1): + parsed_dt = parsed_dt.replace(year=year - 1) + timestamp = parsed_dt.isoformat() + except ValueError: + return None + + # Log key + log_key = None + m = _LOG_KEY_RE.search(raw) + if m: + log_key = m.group(1) + + # Test command: try grind link first, then inline text + test_cmd = '' + m = _GRIND_CMD_RE.search(raw) + if m: + cmd_raw = unquote(m.group(1)) + # Format: "hash:KEY=VAL:KEY=VAL actual_command" + # Strip the hash:KEY=VAL prefix to get the actual test command + parts = cmd_raw.split(' ', 1) + if len(parts) == 2 and ':' in parts[0]: + test_cmd = parts[1].strip() + else: + test_cmd = cmd_raw + else: + # Fallback: extract from inline text after log key + m = _INLINE_CMD_RE.search(clean) + if m: + test_cmd = m.group(1).strip() + + # Duration + duration = None + m = _DURATION_RE.search(clean) + if m: + duration = float(m.group(1)) + + # Author and commit message + author, msg = None, None + m = _AUTHOR_MSG_RE.search(clean) + if m: + author = m.group(1) + msg = m.group(2) + + # Flake group + flake_group = None + m = _FLAKE_GROUP_RE.search(clean) + if m: + flake_group = m.group(1) + + return { + 'status': status, + 'test_cmd': test_cmd, + 'log_url': f'http://ci.aztec-labs.com/{log_key}' if log_key else None, + 'log_key': log_key, + 'ref_name': section, # section is the best ref we have from these lists + 'commit_author': author, + 'commit_msg': msg, + 'duration_secs': duration, + 'flake_group_id': flake_group, + 'timestamp': timestamp, + 'dashboard': section, + } + + +def sync_failed_tests_to_sqlite(redis_conn): + """Read failed_tests_{section} lists from Redis and insert into test_events.""" + global _failed_tests_sync_ts + now = time.time() + if now - _failed_tests_sync_ts < _FAILED_TESTS_SYNC_TTL: + return + _failed_tests_sync_ts = now + + conn = db.get_db() + # Track existing entries to avoid duplicates: log_url for entries that have one, + # (test_cmd, timestamp, dashboard) composite key for entries without log_url + existing_urls = {row['log_url'] for row in conn.execute( + "SELECT DISTINCT log_url FROM test_events WHERE log_url IS NOT NULL" + ).fetchall()} + existing_keys = {(row['test_cmd'], row['timestamp'], row['dashboard']) for row in conn.execute( + "SELECT test_cmd, timestamp, dashboard FROM test_events WHERE log_url IS NULL" + ).fetchall()} + + total = 0 + for section in SECTIONS: + key = f'failed_tests_{section}' + try: + entries = redis_conn.lrange(key, 0, -1) + except Exception as e: + print(f"[rk_metrics] Error reading {key}: {e}") + continue + + for entry_bytes in entries: + raw = entry_bytes.decode() if isinstance(entry_bytes, bytes) else entry_bytes + parsed = _parse_failed_test_entry(raw, section) + if not parsed: + continue + if parsed['log_url']: + if parsed['log_url'] in existing_urls: + continue + existing_urls.add(parsed['log_url']) + else: + composite = (parsed['test_cmd'], parsed['timestamp'], parsed['dashboard']) + if composite in existing_keys: + continue + existing_keys.add(composite) + try: + conn.execute(''' + INSERT INTO test_events + (status, test_cmd, log_url, ref_name, commit_author, + commit_msg, duration_secs, flake_group_id, dashboard, + timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + parsed['status'], parsed['test_cmd'], parsed['log_url'], + parsed['ref_name'], parsed['commit_author'], + parsed['commit_msg'], parsed['duration_secs'], + parsed['flake_group_id'], parsed['dashboard'], + parsed['timestamp'], + )) + total += 1 + except Exception as e: + print(f"[rk_metrics] Error inserting test event: {e}") + conn.commit() + if total: + print(f"[rk_metrics] Synced {total} test events from Redis lists") + + +# ---- Seed loading ---- + +def _load_seed_data(): + """Load CI runs and test events from ci-run-seed.json.gz if SQLite is empty.""" + import gzip + from pathlib import Path + + conn = db.get_db() + ci_count = conn.execute('SELECT COUNT(*) as c FROM ci_runs').fetchone()['c'] + te_count = conn.execute('SELECT COUNT(*) as c FROM test_events').fetchone()['c'] + if ci_count > 0 and te_count > 0: + return + + seed = Path(__file__).parent / 'ci-run-seed.json.gz' + if not seed.exists(): + return + + with gzip.open(seed, 'rt') as f: + data = json.load(f) + + now_iso = datetime.now(timezone.utc).isoformat() + + if ci_count == 0 and data.get('ci_runs'): + runs = data['ci_runs'] + for run in runs: + try: + conn.execute(''' + INSERT OR IGNORE INTO ci_runs + (dashboard, name, timestamp_ms, complete_ms, status, author, + pr_number, instance_type, instance_vcpus, spot, cost_usd, + job_id, arch, synced_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + run.get('dashboard', ''), + run.get('name', ''), + run.get('timestamp', 0), + run.get('complete'), + run.get('status'), + run.get('author'), + run.get('pr_number'), + run.get('instance_type'), + run.get('instance_vcpus'), + 1 if run.get('spot') else 0, + run.get('cost_usd'), + run.get('job_id', ''), + run.get('arch', ''), + now_iso, + )) + except Exception: + continue + conn.commit() + print(f"[rk_metrics] Loaded {len(runs)} CI runs from seed") + + if te_count == 0 and data.get('test_events'): + events = data['test_events'] + for ev in events: + try: + conn.execute(''' + INSERT OR IGNORE INTO test_events + (status, test_cmd, log_url, ref_name, commit_hash, commit_author, + commit_msg, exit_code, duration_secs, is_scenario, owners, + flake_group_id, dashboard, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + ev.get('status', ''), + ev.get('test_cmd', ''), + ev.get('log_url'), + ev.get('ref_name', ''), + ev.get('commit_hash'), + ev.get('commit_author'), + ev.get('commit_msg'), + ev.get('exit_code'), + ev.get('duration_secs'), + ev.get('is_scenario', 0), + ev.get('owners'), + ev.get('flake_group_id'), + ev.get('dashboard', ''), + ev.get('timestamp', ''), + )) + except Exception: + continue + conn.commit() + print(f"[rk_metrics] Loaded {len(events)} test events from seed") + + +# ---- CI run sync (Redis → SQLite) for flake correlation ---- + +_ci_sync_ts = 0 +_CI_SYNC_TTL = 3600 # 1 hour + + +def sync_ci_runs_to_sqlite(redis_conn): + """Sync all CI runs from Redis into SQLite for persistence.""" + global _ci_sync_ts + now = time.time() + if now - _ci_sync_ts < _CI_SYNC_TTL: + return + _ci_sync_ts = now + + # Sync everything Redis has (not just 30 days) + runs = _get_ci_runs_from_redis(redis_conn) + + now_iso = datetime.now(timezone.utc).isoformat() + conn = db.get_db() + count = 0 + for run in runs: + try: + conn.execute(''' + INSERT OR REPLACE INTO ci_runs + (dashboard, name, timestamp_ms, complete_ms, status, author, + pr_number, instance_type, instance_vcpus, spot, cost_usd, + job_id, arch, synced_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + run.get('dashboard', ''), + run.get('name', ''), + run.get('timestamp', 0), + run.get('complete'), + run.get('status'), + run.get('author'), + run.get('pr_number'), + run.get('instance_type'), + run.get('instance_vcpus'), + 1 if run.get('spot') else 0, + run.get('cost_usd'), + run.get('job_id', ''), + run.get('arch', ''), + now_iso, + )) + count += 1 + except Exception as e: + print(f"[rk_metrics] Error syncing run: {e}") + conn.commit() + print(f"[rk_metrics] Synced {count} CI runs to SQLite") + + +def start_ci_run_sync(redis_conn): + """Start periodic CI run + test event sync thread.""" + _load_seed_data() + + def loop(): + while True: + try: + sync_ci_runs_to_sqlite(redis_conn) + sync_failed_tests_to_sqlite(redis_conn) + except Exception as e: + print(f"[rk_metrics] sync error: {e}") + time.sleep(600) # check every 10 min (TTL gates actual work) + + t = threading.Thread(target=loop, daemon=True, name='ci-run-sync') + t.start() + return t + + +def get_flakes_by_command(date_from, date_to, dashboard=''): + """Get flake stats grouped by CI command type (dashboard/section).""" + if dashboard: + rows = db.query(''' + SELECT dashboard, test_cmd, COUNT(*) as count + FROM test_events + WHERE status = 'flaked' AND dashboard = ? + AND timestamp >= ? AND timestamp < ? + GROUP BY dashboard, test_cmd + ORDER BY count DESC + ''', (dashboard, date_from, date_to + 'T23:59:59')) + else: + rows = db.query(''' + SELECT dashboard, test_cmd, COUNT(*) as count + FROM test_events + WHERE status = 'flaked' AND dashboard != '' + AND timestamp >= ? AND timestamp < ? + GROUP BY dashboard, test_cmd + ORDER BY count DESC + ''', (date_from, date_to + 'T23:59:59')) + + by_command = {} + total_flakes = 0 + for row in rows: + cmd = row['dashboard'] + if cmd not in by_command: + by_command[cmd] = {'total': 0, 'tests': {}} + by_command[cmd]['total'] += row['count'] + by_command[cmd]['tests'][row['test_cmd']] = row['count'] + total_flakes += row['count'] + + if dashboard: + failure_rows = db.query(''' + SELECT dashboard, COUNT(*) as count + FROM test_events + WHERE status = 'failed' AND dashboard = ? + AND timestamp >= ? AND timestamp < ? + GROUP BY dashboard + ''', (dashboard, date_from, date_to + 'T23:59:59')) + else: + failure_rows = db.query(''' + SELECT dashboard, COUNT(*) as count + FROM test_events + WHERE status = 'failed' AND dashboard != '' + AND timestamp >= ? AND timestamp < ? + GROUP BY dashboard + ''', (date_from, date_to + 'T23:59:59')) + failures_by_command = {r['dashboard']: r['count'] for r in failure_rows} + + result_list = [] + for cmd, data in sorted(by_command.items(), key=lambda x: -x[1]['total']): + top_tests = sorted(data['tests'].items(), key=lambda x: -x[1])[:10] + result_list.append({ + 'command': cmd, + 'total_flakes': data['total'], + 'total_failures': failures_by_command.get(cmd, 0), + 'top_tests': [{'test_cmd': t, 'count': c} for t, c in top_tests], + }) + + return { + 'by_command': result_list, + 'summary': { + 'total_flakes': total_flakes, + 'total_failures': sum(failures_by_command.values()), + }, + } diff --git a/ci3/ci-metrics/requirements.txt b/ci3/ci-metrics/requirements.txt new file mode 100644 index 000000000000..d6516263133f --- /dev/null +++ b/ci3/ci-metrics/requirements.txt @@ -0,0 +1,8 @@ +flask +gunicorn +redis +Flask-Compress +Flask-HTTPAuth +requests +google-cloud-bigquery +boto3 diff --git a/ci3/ci-metrics/sync_to_sqlite.py b/ci3/ci-metrics/sync_to_sqlite.py new file mode 100755 index 000000000000..5dd6faae6172 --- /dev/null +++ b/ci3/ci-metrics/sync_to_sqlite.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Sync ephemeral Redis CI data to persistent SQLite. + +Normally run automatically by the ci-metrics server's background sync thread. +Can also be run standalone for a one-off manual sync: + + cd ci3/ci-metrics && python3 sync_to_sqlite.py + +Connects to Redis, reads all CI runs and failed test lists, writes to SQLite. +""" +import os +import sys +import time + +# Ensure this script can import sibling modules +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import redis as redis_lib +import db +import metrics + +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) + + +def main(): + start = time.time() + r = redis_lib.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=False) + + try: + r.ping() + except Exception as e: + print(f"[sync] Cannot connect to Redis at {REDIS_HOST}:{REDIS_PORT}: {e}") + sys.exit(1) + + # Ensure DB schema is up to date + db.get_db() + + # Force sync by resetting the TTL gates + metrics._ci_sync_ts = 0 + metrics._failed_tests_sync_ts = 0 + + # Sync CI runs + print("[sync] Syncing CI runs from Redis to SQLite...") + metrics.sync_ci_runs_to_sqlite(r) + + # Sync failed/flaked test events from Redis lists + print("[sync] Syncing test events from Redis to SQLite...") + metrics.sync_failed_tests_to_sqlite(r) + + # Report + conn = db.get_db() + ci_count = conn.execute('SELECT COUNT(*) as c FROM ci_runs').fetchone()['c'] + te_count = conn.execute('SELECT COUNT(*) as c FROM test_events').fetchone()['c'] + elapsed = time.time() - start + print(f"[sync] Done in {elapsed:.1f}s. SQLite: {ci_count} CI runs, {te_count} test events.") + + +if __name__ == '__main__': + main() diff --git a/ci3/ci-metrics/views/ci-insights.html b/ci3/ci-metrics/views/ci-insights.html new file mode 100644 index 000000000000..533b6bfb62cd --- /dev/null +++ b/ci3/ci-metrics/views/ci-insights.html @@ -0,0 +1,658 @@ + + + + + ACI - CI Insights + + + + + +

ci insights

+ +
+ + + + | + + + | + + + + | + +
+ +
+ + + +
+
daily ci spend
--
+
cost / merge
--
+
mq success rate
--
+
flakes / day
--
+
prs merged / day
--
+
+ + +
+
+

daily ci cost + 7-day rolling cost per merge

+
+
+
+

merge queue: daily outcomes + success rate

+
+
+
+

flakes + test failures per day

+
+
+
+ + +
flakes by pipeline
+
+ + + +
+
+ + +
author ci profile
+
+ + + +
+
+ + + + + diff --git a/ci3/ci-metrics/views/cost-overview.html b/ci3/ci-metrics/views/cost-overview.html new file mode 100644 index 000000000000..53424a2d2d70 --- /dev/null +++ b/ci3/ci-metrics/views/cost-overview.html @@ -0,0 +1,905 @@ + + + + + ACI - Cost Overview + + + + + +

cost overview

+ +
+ + + + | + + + | + + + + | + +
+ +
+ +
+
Overview
+
Resource Details
+
CI Attribution
+
+ +
+
+ +
+
+

combined daily spend

+
+
+
+

service category breakdown

+
+
+
+

aws vs gcp split

+
+
+
+ + + + +
+
+ +
+
+
+
+ + + +
+
+
+ +
+
+ +
+
+
+

ci cost by run type (time series)

+
+
+
+

cost by user (AWS + GCP)

+
+
+
+

cost by run type

+
+
+
+

instances

+
+ + + +
+
+
+ + + + + diff --git a/ci3/ci-metrics/views/test-timings.html b/ci3/ci-metrics/views/test-timings.html new file mode 100644 index 000000000000..0bf6c7213bd6 --- /dev/null +++ b/ci3/ci-metrics/views/test-timings.html @@ -0,0 +1,289 @@ + + + + + ACI - Test Timings + + + +
+ < CI | + cost overview + namespace billing + ci insights + test timings +
+ +

test timings

+ +
+ + + + + | + + + | + + + | + + +
+ +
loading...
+ +
+ +
+
+

avg duration by day

+
+
+
+

test run count by day

+
+
+
+ +

tests by duration

+
+ + + + + + + + + + + + + + + + +
test commandrunsavg (s)min (s)max (s)total (h)pass %passedfailedflaked
+
+ +

slowest individual runs

+
+ + + + + + + + + + + + + +
test commandduration (s)statusdateauthorpipelinelog
+
+ + + + + diff --git a/ci3/dashboard/Dockerfile b/ci3/dashboard/Dockerfile index 2ca190fd9753..2da7805ffa83 100644 --- a/ci3/dashboard/Dockerfile +++ b/ci3/dashboard/Dockerfile @@ -16,7 +16,12 @@ RUN apt update && apt install -y \ WORKDIR /app COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt gunicorn + +# Install ci-metrics dependencies (ci-metrics runs as subprocess) +COPY ci-metrics/requirements.txt ci-metrics/requirements.txt +RUN pip install --no-cache-dir -r ci-metrics/requirements.txt + RUN git config --global --add safe.directory /aztec-packages COPY . . -EXPOSE 8080 +EXPOSE 8080 8081 CMD ["gunicorn", "-w", "100", "-b", "0.0.0.0:8080", "rk:app"] diff --git a/ci3/dashboard/deploy.sh b/ci3/dashboard/deploy.sh index cc417006d072..1d9e930e95a1 100755 --- a/ci3/dashboard/deploy.sh +++ b/ci3/dashboard/deploy.sh @@ -1,7 +1,13 @@ #!/bin/bash set -euo pipefail -rsync -avz --exclude='deploy.sh' -e "ssh -i ~/.ssh/build_instance_key" * ubuntu@ci.aztec-labs.com:rk +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Sync dashboard (rkapp) files +rsync -avz --exclude='deploy.sh' -e "ssh -i ~/.ssh/build_instance_key" "$SCRIPT_DIR"/* ubuntu@ci.aztec-labs.com:rk + +# Sync ci-metrics server (started as subprocess by rkapp) +rsync -avz -e "ssh -i ~/.ssh/build_instance_key" "$SCRIPT_DIR/../ci-metrics/" ubuntu@ci.aztec-labs.com:rk/ci-metrics/ ssh -i ~/.ssh/build_instance_key ubuntu@ci.aztec-labs.com " cd rk diff --git a/ci3/dashboard/rk.py b/ci3/dashboard/rk.py index 4e194cbc3a10..aedf35a824e2 100644 --- a/ci3/dashboard/rk.py +++ b/ci3/dashboard/rk.py @@ -18,13 +18,40 @@ YELLOW, BLUE, GREEN, RED, PURPLE, BOLD, RESET, hyperlink, r, get_section_data, get_list_as_string ) - LOGS_DISK_PATH = os.getenv('LOGS_DISK_PATH', '/logs-disk') DASHBOARD_PASSWORD = os.getenv('DASHBOARD_PASSWORD', 'password') +CI_METRICS_PORT = int(os.getenv('CI_METRICS_PORT', '8081')) +CI_METRICS_URL = os.getenv('CI_METRICS_URL', f'http://localhost:{CI_METRICS_PORT}') + app = Flask(__name__) Compress(app) auth = HTTPBasicAuth() +# Start the ci-metrics server as a subprocess +# Check sibling dir (repo layout) then subdirectory (Docker layout) +_ci_metrics_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'ci-metrics') +if not os.path.isdir(_ci_metrics_dir): + _ci_metrics_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ci-metrics') +if os.path.isdir(_ci_metrics_dir): + # Kill any stale process on the port (e.g. leftover from previous reload) + import signal + try: + out = subprocess.check_output( + ['lsof', '-ti', f':{CI_METRICS_PORT}'], stderr=subprocess.DEVNULL, text=True) + for pid in out.strip().split('\n'): + if pid: + os.kill(int(pid), signal.SIGTERM) + import time; time.sleep(0.5) + except (subprocess.CalledProcessError, OSError): + pass + _ci_metrics_env = {**os.environ, 'CI_METRICS_PORT': str(CI_METRICS_PORT)} + subprocess.Popen( + ['gunicorn', '-w', '4', '-b', f'0.0.0.0:{CI_METRICS_PORT}', '--timeout', '120', 'app:app'], + cwd=_ci_metrics_dir, + env=_ci_metrics_env, + ) + print(f"[rk.py] ci-metrics server started on port {CI_METRICS_PORT}") + def read_from_disk(key): """Read log from disk as fallback when Redis key not found.""" try: @@ -145,6 +172,14 @@ def root() -> str: f"{hyperlink('https://aztecprotocol.github.io/benchmark-page-data/bench?branch=next', 'next')}\n" f"{hyperlink('/chonk-breakdowns', 'chonk breakdowns')}\n" f"{RESET}" + f"\n" + f"CI Metrics:\n" + f"\n{YELLOW}" + f"{hyperlink('/cost-overview', 'cost overview (AWS + GCP)')}\n" + f"{hyperlink('/namespace-billing', 'namespace billing')}\n" + f"{hyperlink('/ci-insights', 'ci insights')}\n" + f"{hyperlink('/test-timings', 'test timings')}\n" + f"{RESET}" ) def section_view(section: str) -> str: @@ -487,6 +522,57 @@ def make_options(param_name, options, current_value, suffix=''): # Redirect to log view. return redirect(f'/{run_id}') + +# ---- Reverse proxy to ci-metrics server ---- + +_proxy_session = requests.Session() +_HOP_BY_HOP = frozenset([ + 'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', + 'te', 'trailers', 'transfer-encoding', 'upgrade', 'content-length', + # `requests` auto-decompresses gzip responses, so Content-Encoding is + # stale — strip it so the browser doesn't try to decompress plain content. + # Flask-Compress on rkapp handles browser compression. + 'content-encoding', +]) +# Don't forward Accept-Encoding — let `requests` negotiate with ci-metrics +# (it adds its own and auto-decompresses). +_STRIP_REQUEST_HEADERS = frozenset(['host', 'accept-encoding']) + +def _proxy(path): + """Forward request to ci-metrics, streaming the response back.""" + url = f'{CI_METRICS_URL}/{path.lstrip("/")}' + try: + resp = _proxy_session.request( + method=request.method, + url=url, + params=request.args, + data=request.get_data(), + headers={k: v for k, v in request.headers if k.lower() not in _STRIP_REQUEST_HEADERS}, + stream=True, + timeout=60, + ) + # Strip hop-by-hop headers + headers = {k: v for k, v in resp.headers.items() if k.lower() not in _HOP_BY_HOP} + return Response(resp.iter_content(chunk_size=8192), + status=resp.status_code, headers=headers) + except Exception as e: + return Response(json.dumps({'error': f'ci-metrics unavailable: {e}'}), + mimetype='application/json', status=502) + +@app.route('/namespace-billing') +@app.route('/ci-health') +@app.route('/ci-insights') +@app.route('/cost-overview') +@app.route('/test-timings') +@auth.login_required +def proxy_dashboard(): + return _proxy(request.path) + +@app.route('/api/', methods=['GET', 'POST', 'PUT', 'DELETE']) +@auth.login_required +def proxy_api(path): + return _proxy(f'/api/{path}') + @app.route('/') @auth.login_required def get_value(key): diff --git a/ci3/find_ports b/ci3/find_ports deleted file mode 100755 index d7da0afe53bf..000000000000 --- a/ci3/find_ports +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -source $(git rev-parse --show-toplevel)/ci3/source -# Find 'num_ports' free ports between 9000 and 10000 -# Read first arg, default to 1 port -num_ports="${1:-1}" -echo $(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n "$num_ports") diff --git a/ci3/log_ci_run b/ci3/log_ci_run index 5c9567ae91dd..b52b93256edc 100755 --- a/ci3/log_ci_run +++ b/ci3/log_ci_run @@ -35,6 +35,14 @@ if [ -z "$key" ]; then author="$(git log -1 --pretty=format:"%an")" name=$REF_NAME [ "$(aws_get_meta_data instance-life-cycle)" == "spot" ] && spot=true || spot=false + instance_type=$(aws_get_meta_data instance-type 2>/dev/null || echo "unknown") + instance_vcpus=$(nproc 2>/dev/null || echo 0) + + # Extract PR number from branch name or merge queue ref + pr_number="" + if [[ "$REF_NAME" =~ [Pp][Rr]-?([0-9]+) ]]; then + pr_number="${BASH_REMATCH[1]}" + fi # If this is github merge queue, just keep the queue name. if [[ "$name" =~ ^gh-readonly-queue/([^/]+)/ ]]; then @@ -42,6 +50,7 @@ if [ -z "$key" ]; then fi msg=$(pr_link "$msg") + dashboard="${range_key#ci-run-}" json=$(jq -c -j -n \ --argjson timestamp "$key" \ @@ -53,7 +62,12 @@ if [ -z "$key" ]; then --arg author "$author" \ --arg arch "$(arch)" \ --argjson spot "$spot" \ - '{timestamp: $timestamp, run_id: $run_id, job_id: $job_id, status: $status, msg: $msg, name: $name, author: $author, arch: $arch, spot: $spot}') + --arg instance_type "$instance_type" \ + --argjson instance_vcpus "$instance_vcpus" \ + --arg pr_number "$pr_number" \ + --arg dashboard "$dashboard" \ + --arg github_actor "${GITHUB_ACTOR:-}" \ + '{timestamp: $timestamp, run_id: $run_id, job_id: $job_id, status: $status, msg: $msg, name: $name, author: $author, github_actor: $github_actor, arch: $arch, spot: $spot, instance_type: $instance_type, instance_vcpus: $instance_vcpus, pr_number: $pr_number, dashboard: $dashboard}') # echo "$json" >&2 redis_cli ZADD $range_key $key "$json" &>/dev/null redis_cli SETEX hb-$key 60 1 &>/dev/null diff --git a/ci3/run_test_cmd b/ci3/run_test_cmd index 66334e535f27..a8fd10836497 100755 --- a/ci3/run_test_cmd +++ b/ci3/run_test_cmd @@ -119,6 +119,21 @@ function publish_log_final { cat $tmp_file 2>/dev/null | cache_persistent $log_key $expire } +# Finalize the current log and start a fresh one with a new unique key. +function rotate_log { + if [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then + publish_log_final "$@" + fi + log_key=$(uuid) + log_info=" ($(ci_term_link $log_key))" + > $tmp_file + if [ "$live_logging" -eq 1 ]; then + kill ${publish_pid:-} &>/dev/null + live_publish_log & + publish_pid=$! + fi +} + function live_publish_log { # Not replacing previous trap as we run this function in the background. trap 'kill $sleep_pid &>/dev/null; exit' SIGTERM SIGINT @@ -160,7 +175,8 @@ if [ "$publish" -eq 1 ]; then --arg commit_hash "$COMMIT_HASH" \ --arg commit_author "$COMMIT_AUTHOR" \ --arg commit_msg "$COMMIT_MSG" \ - '{status: $status, test_cmd: $test_cmd, log_id: $log_id, log_url: $log_url, ref_name: $ref_name, commit_hash: $commit_hash, commit_author: $commit_author, commit_msg: $commit_msg, timestamp: now | todate}') + --arg dashboard "${CI_DASHBOARD:-}" \ + '{status: $status, test_cmd: $test_cmd, log_id: $log_id, log_url: $log_url, ref_name: $ref_name, commit_hash: $commit_hash, commit_author: $commit_author, commit_msg: $commit_msg, dashboard: $dashboard, timestamp: now | todate}') redis_publish "ci:test:started" "$start_redis_data" fi @@ -228,15 +244,16 @@ function track_test_failed { function publish_redis { local redis_data=$(jq -n \ --arg status "$1" \ - --arg cmd "$cmd" \ - --arg log_key "$log_key" \ - --arg ref_name "$REF_NAME" \ + --arg test_cmd "$cmd" \ + --arg log_url "http://ci.aztec-labs.com/$log_key" \ + --arg ref_name "${TARGET_BRANCH:-$REF_NAME}" \ --arg commit_hash "$COMMIT_HASH" \ --arg commit_author "$COMMIT_AUTHOR" \ --arg commit_msg "$COMMIT_MSG" \ --argjson code "$code" \ --argjson duration "$SECONDS" \ - '{status: $status, cmd: $cmd, log_key: $log_key, ref_name: $ref_name, commit_hash: $commit_hash, commit_author: $commit_author, commit_msg: $commit_msg, exit_code: $code, duration_seconds: $duration, timestamp: now | todate}') + --arg dashboard "${CI_DASHBOARD:-}" \ + '{status: $status, test_cmd: $test_cmd, log_url: $log_url, ref_name: $ref_name, commit_hash: $commit_hash, commit_author: $commit_author, commit_msg: $commit_msg, exit_code: $code, duration_seconds: $duration, dashboard: $dashboard, timestamp: now | todate}') redis_publish "ci:test:$1" "$redis_data" } @@ -247,6 +264,8 @@ function pass { local line="${green}PASSED${reset}${log_info:-}: $test_cmd (${SECONDS}s)" echo -e "$line" + [ "$publish" -eq 1 ] && publish_redis "passed" + if [ "$track_test_history" -eq 1 ]; then local track_line="${green}PASSED${reset}${log_info:-} ${fail_links}: $test_cmd (${SECONDS}s) (${purple}$COMMIT_AUTHOR${reset}: $COMMIT_MSG)" track_test_history "$track_line" @@ -332,12 +351,23 @@ flake_group_id=$(echo "$test_entries" | jq -r '.flake_group_id // empty' | head if [ -z "$owners" ]; then fail else - echo -e "${yellow}RETRYING${reset}${log_info:-}: $test_cmd" + failure_log_key=$log_key + failure_log_info=$log_info + rotate_log $((60 * 60 * 24 * 7 * 12)) + + echo -e "${yellow}RETRYING${reset}${log_info}: $test_cmd" run_test - # Test passed. Signal it as a flake, but pass. - [ $code -eq 0 ] && flake + if [ $code -eq 0 ]; then + # Publish the retry's log, then point back at the failure for flake reporting. + if [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then + publish_log_final + fi + log_key=$failure_log_key + log_info=$failure_log_info + flake + fi # Otherwise we failed twice in a row, so hard fail. fail diff --git a/docs/docs-operate/operators/reference/changelog/v4.md b/docs/docs-operate/operators/reference/changelog/v4.md index 10bfef79cc4d..369e3fec643a 100644 --- a/docs/docs-operate/operators/reference/changelog/v4.md +++ b/docs/docs-operate/operators/reference/changelog/v4.md @@ -88,6 +88,55 @@ A new environment variable `AZTEC_INITIAL_ETH_PER_FEE_ASSET` has been added to c This replaces the previous hardcoded default and allows network operators to set the starting price point for the fee asset. +### `reloadKeystore` admin RPC endpoint + +Node operators can now update validator attester keys, coinbase, and fee recipient without restarting the node by calling the new `reloadKeystore` admin RPC endpoint. + +What is updated on reload: +- Validator attester keys (add, remove, or replace) +- Coinbase and fee recipient per validator +- Publisher-to-validator mapping + +What is NOT updated (requires restart): +- L1 publisher signers +- Prover keys +- HA signer connections + +New validators must use a publisher key already initialized at startup. Reload is rejected with a clear error if validation fails. + +### Admin API key authentication + +The admin JSON-RPC endpoint now supports auto-generated API key authentication. + +**Behavior:** +- A cryptographically secure API key is auto-generated at first startup and displayed once via stdout +- Only the SHA-256 hash is persisted to `/admin/api_key_hash` +- The key is reused across restarts when `--data-directory` is set +- Supports both `x-api-key` and `Authorization: Bearer ` headers +- Health check endpoint (`GET /status`) is excluded from auth (for k8s probes) + +**Configuration:** + +```bash +--admin-api-key-hash ($AZTEC_ADMIN_API_KEY_HASH) # Use a pre-generated SHA-256 key hash +--no-admin-api-key ($AZTEC_NO_ADMIN_API_KEY) # Disable auth entirely +--reset-admin-api-key ($AZTEC_RESET_ADMIN_API_KEY) # Force key regeneration +``` + +**Helm charts**: Admin API key auth is disabled by default (`noAdminApiKey: true`). Set to `false` in production values to enable. + +**Migration**: No action required — auth is opt-out. To enable, ensure `--no-admin-api-key` is not set and note the key printed at startup. + +### Transaction pool error codes for RPC callers + +Transaction submission via RPC now returns structured rejection codes when a transaction is rejected by the mempool: + +- `LOW_PRIORITY_FEE` — tx priority fee is too low +- `INSUFFICIENT_FEE_PAYER_BALANCE` — fee payer doesn't have enough balance +- `NULLIFIER_CONFLICT` — conflicting nullifier already in pool + +**Impact**: Improved developer experience — callers can now programmatically handle specific rejection reasons. + ## Changed defaults ## Troubleshooting diff --git a/noir-projects/aztec-nr/bootstrap.sh b/noir-projects/aztec-nr/bootstrap.sh index ed07b0eb933c..2181000ce913 100755 --- a/noir-projects/aztec-nr/bootstrap.sh +++ b/noir-projects/aztec-nr/bootstrap.sh @@ -20,17 +20,29 @@ function test_cmds { i=0 $NARGO test --list-tests --silence-warnings | sort | while read -r package test; do # We assume there are 8 txe's running. - port=$((45730 + (i++ % ${NUM_TXES:-1}))) + port=$((14730 + (i++ % ${NUM_TXES:-1}))) echo "$hash noir-projects/scripts/run_test.sh aztec-nr $package $test $port" done } function test { # Start txe server. + # Port is below the Linux ephemeral range (32768-60999) to avoid conflicts. + local txe_base_port=14730 trap 'kill $(jobs -p)' EXIT - (cd $root/yarn-project/txe && LOG_LEVEL=error TXE_PORT=45730 yarn start) & + check_port $txe_base_port || echo "WARNING: port $txe_base_port is in use, TXE may fail to start" + (cd $root/yarn-project/txe && LOG_LEVEL=error TXE_PORT=$txe_base_port yarn start) & echo "Waiting for TXE to start..." - while ! nc -z 127.0.0.1 45730 &>/dev/null; do sleep 1; done + local j=0 + while ! nc -z 127.0.0.1 $txe_base_port &>/dev/null; do + if [ $j == 60 ]; then + echo "TXE failed to start on port $txe_base_port after 60s." >&2 + check_port $txe_base_port + exit 1 + fi + sleep 1 + j=$((j+1)) + done export NARGO_FOREIGN_CALL_TIMEOUT=300000 test_cmds | filter_test_cmds | parallelize diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 12c6a06c4ee3..fe6b671029c3 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -237,7 +237,7 @@ function test_cmds { i=0 $NARGO test --list-tests --silence-warnings | sort | while read -r package test; do - port=$((45730 + (i++ % ${NUM_TXES:-1}))) + port=$((14730 + (i++ % ${NUM_TXES:-1}))) [ -z "${cache[$package]:-}" ] && cache[$package]=$(get_contract_hash $package $folder_name) echo "${cache[$package]} noir-projects/scripts/run_test.sh noir-contracts $package $test $port" done @@ -245,14 +245,27 @@ function test_cmds { function test { # Starting txe servers with incrementing port numbers. + # Base port is below the Linux ephemeral range (32768-60999) to avoid conflicts. + local txe_base_port=14730 export NUM_TXES=8 trap 'kill $(jobs -p) &>/dev/null || true' EXIT for i in $(seq 0 $((NUM_TXES-1))); do - (cd $root/yarn-project/txe && LOG_LEVEL=silent TXE_PORT=$((45730 + i)) yarn start) >/dev/null & + check_port $((txe_base_port + i)) || echo "WARNING: port $((txe_base_port + i)) is in use, TXE $i may fail to start" + (cd $root/yarn-project/txe && LOG_LEVEL=silent TXE_PORT=$((txe_base_port + i)) yarn start) >/dev/null & done echo "Waiting for TXE's to start..." for i in $(seq 0 $((NUM_TXES-1))); do - while ! nc -z 127.0.0.1 $((45730 + i)) &>/dev/null; do sleep 1; done + local j=0 + local port=$((txe_base_port + i)) + while ! nc -z 127.0.0.1 $port &>/dev/null; do + if [ $j == 60 ]; then + echo "TXE $i failed to start on port $port after 60s." >&2 + check_port $port + exit 1 + fi + sleep 1 + j=$((j+1)) + done done export NARGO_FOREIGN_CALL_TIMEOUT=300000 diff --git a/spartan/.gitignore b/spartan/.gitignore index b47543f24982..792fa0ebb8b7 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -1,4 +1,5 @@ *.tgz +!terraform/modules/web3signer/web3signer-1.0.6.tgz scripts/logs scripts/LICENSE tfplan diff --git a/spartan/aztec-chaos-scenarios/values/network-requirements.yaml b/spartan/aztec-chaos-scenarios/values/network-requirements.yaml index a56dbe192d68..f300e19f1880 100644 --- a/spartan/aztec-chaos-scenarios/values/network-requirements.yaml +++ b/spartan/aztec-chaos-scenarios/values/network-requirements.yaml @@ -9,7 +9,7 @@ networkShaping: correlation: "75" bandwidth: enabled: true - rate: 25mbps + rate: 200mbps packetLoss: enabled: true loss: diff --git a/spartan/aztec-node/templates/_pod-template.yaml b/spartan/aztec-node/templates/_pod-template.yaml index 8d433f5577a3..709a71b3c9cf 100644 --- a/spartan/aztec-node/templates/_pod-template.yaml +++ b/spartan/aztec-node/templates/_pod-template.yaml @@ -190,6 +190,13 @@ spec: value: "{{ .Values.service.rpc.port }}" - name: AZTEC_ADMIN_PORT value: "{{ .Values.service.admin.port }}" + {{- if .Values.node.adminApiKeyHash }} + - name: AZTEC_ADMIN_API_KEY_HASH + value: {{ .Values.node.adminApiKeyHash | quote }} + {{- else if .Values.node.noAdminApiKey }} + - name: AZTEC_NO_ADMIN_API_KEY + value: "true" + {{- end }} - name: LOG_LEVEL value: "{{ .Values.node.logLevel }}" - name: LOG_JSON diff --git a/spartan/aztec-node/values.yaml b/spartan/aztec-node/values.yaml index 735097b03781..97d0661e1c3b 100644 --- a/spartan/aztec-node/values.yaml +++ b/spartan/aztec-node/values.yaml @@ -99,6 +99,15 @@ node: envEnabled: false filesEnabled: false + # -- SHA-256 hex hash of a pre-generated admin API key. + # When set, the node uses this hash for authentication instead of auto-generating a key. + # Generate with: echo -n "your-api-key" | sha256sum | cut -d' ' -f1 + adminApiKeyHash: "" + + # -- Disable admin API key authentication. + # Set to false in production to enable API key auth. + noAdminApiKey: true + # the address that will receive block or proof rewards coinbase: diff --git a/spartan/aztec-validator/values.yaml b/spartan/aztec-validator/values.yaml index 9868263c5baa..b5112c561d8a 100644 --- a/spartan/aztec-validator/values.yaml +++ b/spartan/aztec-validator/values.yaml @@ -25,6 +25,8 @@ validator: replicaCount: 1 node: + # Set to false in production to enable API key auth. + noAdminApiKey: true configMap: envEnabled: true secret: diff --git a/spartan/environments/devnet.env b/spartan/environments/devnet.env index 6fef140f7011..07479bd312b9 100644 --- a/spartan/environments/devnet.env +++ b/spartan/environments/devnet.env @@ -74,4 +74,4 @@ RPC_INGRESS_HOSTS="[\"$NAMESPACE.aztec-labs.com\"]" RPC_INGRESS_STATIC_IP_NAME=$NAMESPACE-rpc-ip RPC_INGRESS_SSL_CERT_NAMES="[\"$NAMESPACE-rpc-cert\"]" -WS_NUM_HISTORIC_BLOCKS=300 +WS_NUM_HISTORIC_CHECKPOINTS=300 diff --git a/spartan/environments/five-tps-long-epoch.env b/spartan/environments/five-tps-long-epoch.env index 84bd8ee6e591..ff87b7ef63a2 100644 --- a/spartan/environments/five-tps-long-epoch.env +++ b/spartan/environments/five-tps-long-epoch.env @@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} AZTEC_EPOCH_DURATION=32 AZTEC_SLOT_DURATION=36 -AZTEC_PROOF_SUBMISSION_WINDOW=64 +AZTEC_PROOF_SUBMISSION_EPOCHS=2 ETHEREUM_CHAIN_ID=1337 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/spartan/environments/five-tps-short-epoch.env b/spartan/environments/five-tps-short-epoch.env index 56141ee724c4..85f36344fc19 100644 --- a/spartan/environments/five-tps-short-epoch.env +++ b/spartan/environments/five-tps-short-epoch.env @@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} AZTEC_EPOCH_DURATION=8 AZTEC_SLOT_DURATION=36 -AZTEC_PROOF_SUBMISSION_WINDOW=16 +AZTEC_PROOF_SUBMISSION_EPOCHS=10 ETHEREUM_CHAIN_ID=1337 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/spartan/environments/prove-n-tps-real.env b/spartan/environments/prove-n-tps-real.env index 9b8989671922..129abf2e7750 100644 --- a/spartan/environments/prove-n-tps-real.env +++ b/spartan/environments/prove-n-tps-real.env @@ -7,6 +7,7 @@ AZTEC_SLOT_DURATION=72 AZTEC_PROOF_SUBMISSION_EPOCHS=1 AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 +AZTEC_MANA_TARGET=1000000000 # 1B mana CREATE_ETH_DEVNET=true DESTROY_NAMESPACE=true @@ -30,8 +31,8 @@ REAL_VERIFIER=true RPC_REPLICAS=1 RPC_INGRESS_ENABLED=false -PROVER_REPLICAS=200 -PROVER_RESOURCE_PROFILE="prod" +PROVER_REPLICAS=4 +PROVER_RESOURCE_PROFILE="prod-hi-tps" PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 PROVER_AGENT_POLL_INTERVAL_MS=10000 PUBLISHERS_PER_PROVER=1 diff --git a/spartan/environments/ten-tps-long-epoch.env b/spartan/environments/ten-tps-long-epoch.env index 39ea3d75e197..e3fefc644364 100644 --- a/spartan/environments/ten-tps-long-epoch.env +++ b/spartan/environments/ten-tps-long-epoch.env @@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} AZTEC_EPOCH_DURATION=32 AZTEC_SLOT_DURATION=36 -AZTEC_PROOF_SUBMISSION_WINDOW=64 +AZTEC_PROOF_SUBMISSION_EPOCHS=2 ETHEREUM_CHAIN_ID=1337 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/spartan/environments/ten-tps-short-epoch.env b/spartan/environments/ten-tps-short-epoch.env index 35868695e0f6..90f16277c385 100644 --- a/spartan/environments/ten-tps-short-epoch.env +++ b/spartan/environments/ten-tps-short-epoch.env @@ -6,7 +6,7 @@ DESTROY_ETH_DEVNET=true CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} AZTEC_EPOCH_DURATION=8 AZTEC_SLOT_DURATION=36 -AZTEC_PROOF_SUBMISSION_WINDOW=16 +AZTEC_PROOF_SUBMISSION_EPOCHS=2 ETHEREUM_CHAIN_ID=1337 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/spartan/environments/tps-scenario.env b/spartan/environments/tps-scenario.env index 18ecd87bc070..abdaf40948b0 100644 --- a/spartan/environments/tps-scenario.env +++ b/spartan/environments/tps-scenario.env @@ -4,7 +4,7 @@ GCP_REGION=us-west1-a AZTEC_EPOCH_DURATION=8 AZTEC_SLOT_DURATION=72 -AZTEC_PROOF_SUBMISSION_WINDOW=16 +AZTEC_PROOF_SUBMISSION_EPOCHS=2 AZTEC_LAG_IN_EPOCHS=1 CREATE_ETH_DEVNET=false @@ -54,6 +54,8 @@ PROVER_RESOURCE_PROFILE="hi-tps" PROVER_AGENT_POLL_INTERVAL_MS=10000 WAIT_FOR_PROVER_DEPLOY=false +P2P_PUBLIC_IP=false + RUN_TESTS=false P2P_MAX_TX_POOL_SIZE=1000000000 diff --git a/spartan/metrics/grafana/alerts/rules.yaml b/spartan/metrics/grafana/alerts/rules.yaml index cbea5beb931a..bfe88fa23d11 100644 --- a/spartan/metrics/grafana/alerts/rules.yaml +++ b/spartan/metrics/grafana/alerts/rules.yaml @@ -273,7 +273,7 @@ groups: datasourceUid: spartan-metrics-prometheus model: editorMode: code - expr: sum by (k8s_namespace_name, aztec_error_type) (increase(aztec_sequencer_block_proposal_precheck_failed_count{k8s_namespace_name=~".*(fisherman|mainnet).*"}[$__rate_interval])) + expr: sum by (k8s_namespace_name, aztec_error_type) (increase(aztec_sequencer_checkpoint_precheck_failed_count{k8s_namespace_name=~".*(fisherman|mainnet).*"}[$__rate_interval])) instant: true intervalMs: 60000 legendFormat: __auto diff --git a/spartan/metrics/grafana/dashboards/aztec_network.json b/spartan/metrics/grafana/dashboards/aztec_network.json index fd9041a559a4..7b037701c3a8 100644 --- a/spartan/metrics/grafana/dashboards/aztec_network.json +++ b/spartan/metrics/grafana/dashboards/aztec_network.json @@ -372,14 +372,14 @@ "uid": "${data_source}" }, "editorMode": "code", - "expr": "avg_over_time((clamp(increase(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_status=\"\"}[$__range]) / on(service_name) (increase(aztec_p2p_gossip_message_validation_count{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_gossip_topic_name=\"block_proposal\"}[$__range])), 0, 1))[15m:1m])", + "expr": "(avg_over_time((clamp(increase(aztec_archiver_checkpoint_height{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_status=\"\"}[$__range]) / on(service_name) (increase(aztec_p2p_gossip_message_validation_count{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_gossip_topic_name=\"checkpoint_proposal\", aztec_ok=\"true\"}[$__range])), 0, 1))[15m:1m]))\nor (avg_over_time((clamp(increase(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_status=\"\"}[$__range]) / on(service_name) (increase(aztec_p2p_gossip_message_validation_count{k8s_namespace_name=\"$namespace\", service_name=\"prover-node\", aztec_gossip_topic_name=\"block_proposal\", aztec_ok=\"true\"}[$__range])), 0, 1))[15m:1m]))", "hide": false, - "legendFormat": "Mined block proposals", + "legendFormat": "Mined proposals", "range": true, "refId": "A" } ], - "title": "Mined block proposals", + "title": "Mined proposals", "type": "gauge" }, { diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index 3ca8f9ee9e90..ad40060dee94 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -97,6 +97,7 @@ SEQ_MIN_TX_PER_BLOCK=${SEQ_MIN_TX_PER_BLOCK:-0} SEQ_MAX_TX_PER_BLOCK=${SEQ_MAX_TX_PER_BLOCK:-8} SEQ_BLOCK_DURATION_MS=${SEQ_BLOCK_DURATION_MS:-} SEQ_BUILD_CHECKPOINT_IF_EMPTY=${SEQ_BUILD_CHECKPOINT_IF_EMPTY:-} +SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT=${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT:-0} PROVER_REPLICAS=${PROVER_REPLICAS:-4} PROVER_AGENTS_PER_PROVER=${PROVER_AGENTS_PER_PROVER:-1} R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID:-} @@ -466,7 +467,7 @@ if [[ "${CLUSTER}" == "kind" ]]; then P2P_PUBLIC_IP=false else P2P_NODEPORT_ENABLED=false - P2P_PUBLIC_IP=true + P2P_PUBLIC_IP=${P2P_PUBLIC_IP:-true} fi cat > "${DEPLOY_AZTEC_INFRA_DIR}/terraform.tfvars" << EOF @@ -507,6 +508,7 @@ SEQ_MIN_TX_PER_BLOCK = ${SEQ_MIN_TX_PER_BLOCK} SEQ_MAX_TX_PER_BLOCK = ${SEQ_MAX_TX_PER_BLOCK} SEQ_BLOCK_DURATION_MS = ${SEQ_BLOCK_DURATION_MS:-null} SEQ_BUILD_CHECKPOINT_IF_EMPTY = ${SEQ_BUILD_CHECKPOINT_IF_EMPTY:-null} +SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT = ${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT} PROVER_MNEMONIC = "${LABS_INFRA_MNEMONIC}" PROVER_PUBLISHER_MNEMONIC_START_INDEX = ${PROVER_PUBLISHER_MNEMONIC_START_INDEX} PROVER_PUBLISHERS_PER_PROVER = ${PUBLISHERS_PER_PROVER} @@ -595,7 +597,7 @@ FULL_NODE_INCLUDE_METRICS = "${FULL_NODE_INCLUDE_METRICS-null}" LOG_LEVEL = "${LOG_LEVEL}" FISHERMAN_LOG_LEVEL = "${FISHERMAN_LOG_LEVEL}" -WS_NUM_HISTORIC_BLOCKS = ${WS_NUM_HISTORIC_BLOCKS:-null} +WS_NUM_HISTORIC_CHECKPOINTS = ${WS_NUM_HISTORIC_CHECKPOINTS:-null} P2P_PUBLIC_IP = ${P2P_PUBLIC_IP} P2P_NODEPORT_ENABLED = ${P2P_NODEPORT_ENABLED} diff --git a/spartan/scripts/setup_gcp_secrets.sh b/spartan/scripts/setup_gcp_secrets.sh index 2bde3c4e4b15..362544669e36 100755 --- a/spartan/scripts/setup_gcp_secrets.sh +++ b/spartan/scripts/setup_gcp_secrets.sh @@ -62,9 +62,11 @@ mask_secret_value() { if [[ "$is_json_secret" == "true" ]]; then jq -r '.[]' "$secret_file" | while IFS= read -r element; do - echo "::add-mask::$element" + if [[ -n "$element" ]]; then + echo "::add-mask::$element" + fi done - else + elif [[ -n "$secret_value" ]]; then echo "::add-mask::$secret_value" fi } diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index dccea9a87427..338cef6dffeb 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -218,8 +218,9 @@ locals { "validator.node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "validator.node.env.P2P_DROP_TX" = var.P2P_DROP_TX "validator.node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "validator.node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "validator.node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS "validator.node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS + "validator.node.env.SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT" = var.SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT } # Note: nonsensitive() is required here because helm_releases is used in for_each, @@ -305,7 +306,7 @@ locals { p2p = { publicIP = var.P2P_PUBLIC_IP } } } - })], local.is_kind ? [yamlencode({ + })], local.is_kind ? [yamlencode({ agent = { nodeSelector = null affinity = null @@ -334,9 +335,9 @@ locals { "broker.node.logLevel" = var.LOG_LEVEL "broker.node.env.BOOTSTRAP_NODES" = "asdf" "broker.node.env.PROVER_BROKER_DEBUG_REPLAY_ENABLED" = var.PROVER_BROKER_DEBUG_REPLAY_ENABLED - "agent.node.image.repository" = local.prover_agent_image.repository - "agent.node.image.tag" = local.prover_agent_image.tag - "agent.node.env.CRS_PATH" = "/usr/src/crs" + "agent.node.image.repository" = local.prover_agent_image.repository + "agent.node.image.tag" = local.prover_agent_image.tag + "agent.node.env.CRS_PATH" = "/usr/src/crs" "agent.node.proverRealProofs" = var.PROVER_REAL_PROOFS "agent.node.env.PROVER_AGENT_POLL_INTERVAL_MS" = var.PROVER_AGENT_POLL_INTERVAL_MS "agent.replicaCount" = var.PROVER_REPLICAS @@ -357,7 +358,7 @@ locals { "node.node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "node.node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS "node.node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS "node.service.p2p.nodePortEnabled" = var.P2P_NODEPORT_ENABLED "node.service.p2p.announcePort" = local.p2p_port_prover @@ -415,7 +416,6 @@ locals { })] custom_settings = merge({ - "nodeType" = "rpc" "replicaCount" = var.RPC_REPLICAS "service.p2p.nodePortEnabled" = var.P2P_NODEPORT_ENABLED "service.p2p.announcePort" = local.p2p_port_rpc @@ -437,7 +437,7 @@ locals { "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS "node.env.TX_FILE_STORE_ENABLED" = var.TX_FILE_STORE_ENABLED "node.env.TX_FILE_STORE_URL" = var.TX_FILE_STORE_URL "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS @@ -494,7 +494,7 @@ locals { "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS } boot_node_host_path = "node.env.BOOT_NODE_HOST" @@ -534,7 +534,7 @@ locals { "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS } boot_node_host_path = "node.env.BOOT_NODE_HOST" @@ -571,7 +571,7 @@ locals { "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS } boot_node_host_path = "node.env.BOOT_NODE_HOST" bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES" diff --git a/spartan/terraform/deploy-aztec-infra/values/full-node.yaml b/spartan/terraform/deploy-aztec-infra/values/full-node.yaml index ac81a957591f..44db0c6f9e87 100644 --- a/spartan/terraform/deploy-aztec-infra/values/full-node.yaml +++ b/spartan/terraform/deploy-aztec-infra/values/full-node.yaml @@ -3,6 +3,17 @@ node: env: OTEL_SERVICE_NAME: "full-node" + preStartScript: | + if [ -n "${BOOT_NODE_HOST:-}" ]; then + until curl --silent --head --fail "${BOOT_NODE_HOST}/status" > /dev/null; do + echo "Waiting for boot node..." + sleep 1 + done + echo "Boot node is ready!" + + export BOOTSTRAP_NODES=$(curl -X POST -H "content-type: application/json" --data '{"method": "bootstrap_getEncodedEnr"}' $BOOT_NODE_HOST | jq -r .result) + fi + startCmd: - --node - --archiver diff --git a/spartan/terraform/deploy-aztec-infra/values/prover-resources-hi-tps.yaml b/spartan/terraform/deploy-aztec-infra/values/prover-resources-hi-tps.yaml index bdaee2f34f3c..586e22a37d7b 100644 --- a/spartan/terraform/deploy-aztec-infra/values/prover-resources-hi-tps.yaml +++ b/spartan/terraform/deploy-aztec-infra/values/prover-resources-hi-tps.yaml @@ -1,5 +1,4 @@ node: - hostNetwork: true node: enableInspector: true nodeJsOptions: diff --git a/spartan/terraform/deploy-aztec-infra/values/prover-resources-prod-hi-tps.yaml b/spartan/terraform/deploy-aztec-infra/values/prover-resources-prod-hi-tps.yaml new file mode 100644 index 000000000000..ffd0347086cc --- /dev/null +++ b/spartan/terraform/deploy-aztec-infra/values/prover-resources-prod-hi-tps.yaml @@ -0,0 +1,82 @@ +node: + node: + resources: + requests: + cpu: "7.5" + memory: "55Gi" + + nodeJsOptions: + - "--max-old-space-size=61440" + + nodeSelector: + local-ssd: "false" + node-type: "network" + cores: "8" + hi-mem: "true" + + persistence: + enabled: true + statefulSet: + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi + +broker: + replicaCount: 1 + + node: + resources: + requests: + cpu: "7.5" + memory: "55Gi" + + nodeJsOptions: + - "--max-old-space-size=61440" + + nodeSelector: + local-ssd: "false" + node-type: "network" + cores: "8" + hi-mem: "true" + + persistence: + enabled: true + statefulSet: + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 64Gi +agent: + replicaCount: 4 + + node: + env: + # the pod will be scheduled on a 32-core VM + HARDWARE_CONCURRENCY: "32" + resources: + requests: + memory: "115Gi" + cpu: "31" + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-spot + operator: Exists + + tolerations: + - key: "cloud.google.com/gke-spot" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/spartan/terraform/deploy-aztec-infra/values/rpc.yaml b/spartan/terraform/deploy-aztec-infra/values/rpc.yaml index 55cab255e6b8..a79515b9cd5a 100644 --- a/spartan/terraform/deploy-aztec-infra/values/rpc.yaml +++ b/spartan/terraform/deploy-aztec-infra/values/rpc.yaml @@ -1,5 +1,5 @@ +nodeType: "rpc-node" node: - nodeType: "rpc-node" env: OTEL_SERVICE_NAME: "node" AWS_ACCESS_KEY_ID: "" diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index f3f27dc0cde8..358e2b0ec335 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -343,6 +343,12 @@ variable "SEQ_MAX_TX_PER_BLOCK" { default = "8" } +variable "SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT" { + description = "Percentage probability of skipping checkpoint publishing" + type = string + default = "0" +} + variable "SEQ_BLOCK_DURATION_MS" { description = "Duration per block in milliseconds when building multiple blocks per slot" type = string @@ -766,8 +772,8 @@ variable "P2P_DROP_TX_CHANCE" { default = 0 } -variable "WS_NUM_HISTORIC_BLOCKS" { - description = "Number of historic blocks for world state" +variable "WS_NUM_HISTORIC_CHECKPOINTS" { + description = "Number of historic checkpoints for world state" type = string nullable = true default = null diff --git a/spartan/terraform/modules/web3signer/main.tf b/spartan/terraform/modules/web3signer/main.tf index d882c73d0d83..b51b0f9f773a 100644 --- a/spartan/terraform/modules/web3signer/main.tf +++ b/spartan/terraform/modules/web3signer/main.tf @@ -62,13 +62,13 @@ resource "helm_release" "keystore_setup" { resource "helm_release" "web3signer" { name = "${var.RELEASE_NAME}-signer" - repository = "https://ethpandaops.github.io/ethereum-helm-charts" - chart = "web3signer" - version = "1.0.6" + chart = "${path.module}/web3signer-1.0.6.tgz" namespace = var.NAMESPACE create_namespace = true upgrade_install = true + depends_on = [helm_release.keystore_setup] + values = [ file("${path.module}/values/web3signer.yaml"), yamlencode({ diff --git a/spartan/terraform/modules/web3signer/web3signer-1.0.6.tgz b/spartan/terraform/modules/web3signer/web3signer-1.0.6.tgz new file mode 100644 index 000000000000..6e66659c1215 Binary files /dev/null and b/spartan/terraform/modules/web3signer/web3signer-1.0.6.tgz differ diff --git a/yarn-project/archiver/src/archiver-sync.test.ts b/yarn-project/archiver/src/archiver-sync.test.ts index ab5ae6f9d2bd..bbe5f3aa236e 100644 --- a/yarn-project/archiver/src/archiver-sync.test.ts +++ b/yarn-project/archiver/src/archiver-sync.test.ts @@ -30,6 +30,7 @@ import { Archiver, type ArchiverEmitter } from './archiver.js'; import type { ArchiverInstrumentation } from './modules/instrumentation.js'; import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js'; import { KVArchiverDataStore } from './store/kv_archiver_store.js'; +import { L2TipsCache } from './store/l2_tips_cache.js'; import { FakeL1State } from './test/fake_l1_state.js'; describe('Archiver Sync', () => { @@ -116,6 +117,9 @@ describe('Archiver Sync', () => { // Create event emitter shared by archiver and synchronizer const events = new EventEmitter() as ArchiverEmitter; + // Create L2 tips cache shared by archiver and synchronizer + const l2TipsCache = new L2TipsCache(archiverStore.blockStore); + // Create the L1 synchronizer synchronizer = new ArchiverL1Synchronizer( publicClient, @@ -132,6 +136,7 @@ describe('Archiver Sync', () => { l1Constants, events, instrumentation.tracer, + l2TipsCache, syncLogger, ); @@ -147,6 +152,7 @@ describe('Archiver Sync', () => { l1Constants, synchronizer, events, + l2TipsCache, ); }); @@ -374,7 +380,7 @@ describe('Archiver Sync', () => { }); // Create a random blob that doesn't match the checkpoint - const randomBlob = makeRandomBlob(3); + const randomBlob = await makeRandomBlob(3); // Override blob client to return the random blob instead of the correct one blobClient.getBlobSidecar.mockResolvedValue([randomBlob]); @@ -970,9 +976,253 @@ describe('Archiver Sync', () => { expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(3)); }); - xit('handles an upcoming L2 prune', () => {}); + it('handles an upcoming L2 prune', async () => { + const pruneSpy = jest.fn(); + archiver.events.on(L2BlockSourceEvents.L2PruneUnproven, pruneSpy); + + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Add and sync checkpoints 1, 2, 3 + const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: 3, + }); + + await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 80n, + messagesL1BlockNumber: 60n, + numL1ToL2Messages: 3, + }); + + await fake.addCheckpoint(CheckpointNumber(3), { + l1BlockNumber: 90n, + messagesL1BlockNumber: 66n, + numL1ToL2Messages: 3, + }); + + fake.setL1BlockNumber(100n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(3)); + + // Mark checkpoint 1 as proven + fake.markCheckpointAsProven(CheckpointNumber(1)); + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Enable pruning (simulate proof window about to expire) + fake.setCanPrune(true); + + // Sync again — handleEpochPrune should remove checkpoints 2 and 3 + fake.setL1BlockNumber(101n); + await archiver.syncImmediate(); + + // Proven checkpoint should advance to 1 since we synced it + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // Checkpoints 2 and 3 should be removed, archiver at checkpoint 1 + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // L2PruneUnproven event should have been emitted with the correct epoch + // CP2 is at L1 block 80 → slot = (80 * 12) / 24 = 40 → epoch = 40 / 4 = 10 + expect(pruneSpy).toHaveBeenCalledWith( + expect.objectContaining({ + type: L2BlockSourceEvents.L2PruneUnproven, + epochNumber: EpochNumber(10), + }), + ); + + // L2Tips should reflect rollback to checkpoint 1 + const lastBlockInCheckpoint1 = cp1.blocks[cp1.blocks.length - 1].number; + const tips = await archiver.getL2Tips(); + expect(tips.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + + // Data from checkpoints 2 and 3 should be removed + expect(await archiver.getCheckpoints(CheckpointNumber(2), 1)).toEqual([]); + expect(await archiver.getCheckpoints(CheckpointNumber(3), 1)).toEqual([]); + + archiver.events.off(L2BlockSourceEvents.L2PruneUnproven, pruneSpy); + }, 15_000); + + it('lost a proof (proven checkpoint rolls back to zero)', async () => { + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Add and sync checkpoints 1 and 2 + await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: 3, + }); + + await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 80n, + messagesL1BlockNumber: 60n, + numL1ToL2Messages: 3, + }); + + fake.setL1BlockNumber(90n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + + // Mark checkpoint 1 as proven, sync + fake.markCheckpointAsProven(CheckpointNumber(1)); + fake.setL1BlockNumber(91n); + await archiver.syncImmediate(); + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // Reset proven to 0 (simulate lost proof due to L1 reorg) + fake.markCheckpointAsProven(CheckpointNumber(0)); + fake.setL1BlockNumber(92n); + await archiver.syncImmediate(); + + // Proven checkpoint should be back at 0 + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Pending/checkpointed chain should still be at checkpoint 2 + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + + // L2Tips proven tip should reflect rollback + const tips = await archiver.getL2Tips(); + expect(tips.proven.block.number).toEqual(0); + }, 10_000); + + it('new proof appeared for previously pruned blocks', async () => { + const provenSpy = jest.fn(); + archiver.events.on(L2BlockSourceEvents.L2BlockProven, provenSpy); + + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Add and sync checkpoints 1, 2, 3 + const cp1NumMessages = 3; + const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: cp1NumMessages, + }); + const cp1Archive = cp1.blocks[cp1.blocks.length - 1].archive; + + await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 80n, + messagesL1BlockNumber: 60n, + numL1ToL2Messages: 3, + }); + + await fake.addCheckpoint(CheckpointNumber(3), { + l1BlockNumber: 90n, + messagesL1BlockNumber: 66n, + numL1ToL2Messages: 3, + }); + + fake.setL1BlockNumber(100n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(3)); + + // Mark checkpoint 1 as proven so epoch prune only removes 2 and 3 + fake.markCheckpointAsProven(CheckpointNumber(1)); + + // Enable pruning to trigger epoch prune (unwind checkpoints 2 and 3) + fake.setCanPrune(true); + fake.setL1BlockNumber(101n); + await archiver.syncImmediate(); + + // Verify checkpoints 2 and 3 are pruned (only proven checkpoint 1 remains) + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // Disable pruning + fake.setCanPrune(false); + + // Re-add checkpoints 2 and 3 on L1 (new epoch proposal). + // Remove old checkpoint events and their messages from the fake. + // The message removal triggers rolling hash recalculation, and on next sync + // handleL1ToL2Messages detects the mismatch and clears the archiver's message store. + fake.removeCheckpoint(CheckpointNumber(2)); + fake.removeCheckpoint(CheckpointNumber(3)); + fake.removeMessagesAfter(cp1NumMessages); + + await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 110n, + numL1ToL2Messages: 0, + previousArchive: cp1Archive, + }); + + await fake.addCheckpoint(CheckpointNumber(3), { + l1BlockNumber: 120n, + numL1ToL2Messages: 0, + }); + + // Mark checkpoint 2 as proven + fake.markCheckpointAsProven(CheckpointNumber(2)); - xit('does not attempt to download data for a checkpoint that has been pruned', () => {}); + // Sync + fake.setL1BlockNumber(130n); + await archiver.syncImmediate(); + + // Archiver should re-sync checkpoints 2 and 3 + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(3)); + + // Proven checkpoint should advance to 2 + expect(await archiver.getProvenCheckpointNumber()).toEqual(CheckpointNumber(2)); + + // L2BlockProven event should have been emitted + expect(provenSpy).toHaveBeenCalledWith( + expect.objectContaining({ + type: L2BlockSourceEvents.L2BlockProven, + }), + ); + + archiver.events.off(L2BlockSourceEvents.L2BlockProven, provenSpy); + }, 15_000); + + it('detects new checkpoint behind L1 syncpoint due to L1 reorg', async () => { + const loggerSpy = jest.spyOn(syncLogger, 'warn'); + + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Sync checkpoint 1 from L1 to establish baseline (sync point = 70) + await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: 3, + }); + + fake.setL1BlockNumber(100n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // Manually advance the sync point past where the new checkpoint will appear. + // This simulates a scenario where the sync point was advanced (e.g., via invalid + // attestation handling at line 204), placing it ahead of a new checkpoint. + await archiverStore.setCheckpointSynchedL1BlockNumber(200n); + // checkForNewCheckpointsBeforeL1SyncPoint requires validationResult?.valid to be true + await archiverStore.setPendingChainValidationStatus({ valid: true }); + + // Add checkpoint 2 at L1 block 150 (behind the manual sync point of 200). + // This simulates an L1 reorg that added a new checkpoint in a range already scanned. + await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 150n, + messagesL1BlockNumber: 130n, + numL1ToL2Messages: 3, + }); + + // Sync: searches from 201 onward, doesn't find CP2 at 150. + // checkForNewCheckpointsBeforeL1SyncPoint detects latestLocal(1) < pending(2) + // and rolls back the sync point to CP1's L1 block (70). + // The rollback does NOT re-fetch in the same iteration. + fake.setL1BlockNumber(201n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + expect(loggerSpy).toHaveBeenCalledWith( + expect.stringMatching(/Failed to reach checkpoint 2.*Rolling back/), + expect.anything(), + ); + + // Second sync: fetches from the rolled-back sync point (70) and finds CP2 at L1 block 150 + fake.setL1BlockNumber(202n); + await archiver.syncImmediate(); + + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + }, 15_000); }); describe('checkpointing local proposed blocks', () => { diff --git a/yarn-project/archiver/src/archiver.ts b/yarn-project/archiver/src/archiver.ts index 9eefe8d2d66c..4aec6f3c9e69 100644 --- a/yarn-project/archiver/src/archiver.ts +++ b/yarn-project/archiver/src/archiver.ts @@ -1,5 +1,4 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; -import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; import { EpochCache } from '@aztec/epoch-cache'; import { BlockTagTooOldError, RollupContract } from '@aztec/ethereum/contracts'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; @@ -15,8 +14,6 @@ import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/runni import { DateProvider } from '@aztec/foundation/timer'; import { type ArchiverEmitter, - type CheckpointId, - GENESIS_CHECKPOINT_HEADER_HASH, L2Block, type L2BlockSink, type L2Tips, @@ -41,6 +38,7 @@ import { ArchiverDataStoreUpdater } from './modules/data_store_updater.js'; import type { ArchiverInstrumentation } from './modules/instrumentation.js'; import type { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js'; import type { KVArchiverDataStore } from './store/kv_archiver_store.js'; +import { L2TipsCache } from './store/l2_tips_cache.js'; /** Export ArchiverEmitter for use in factory and tests. */ export type { ArchiverEmitter }; @@ -83,6 +81,9 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra /** Helper to handle updates to the store */ private readonly updater: ArchiverDataStoreUpdater; + /** In-memory cache for L2 chain tips. */ + private readonly l2TipsCache: L2TipsCache; + public readonly tracer: Tracer; /** @@ -122,6 +123,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra protected override readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr }, synchronizer: ArchiverL1Synchronizer, events: ArchiverEmitter, + l2TipsCache?: L2TipsCache, private readonly log: Logger = createLogger('archiver'), ) { super(dataStore, l1Constants); @@ -130,7 +132,8 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra this.initialSyncPromise = promiseWithResolvers(); this.synchronizer = synchronizer; this.events = events; - this.updater = new ArchiverDataStoreUpdater(this.dataStore); + this.l2TipsCache = l2TipsCache ?? new L2TipsCache(this.dataStore.blockStore); + this.updater = new ArchiverDataStoreUpdater(this.dataStore, this.l2TipsCache); // Running promise starts with a small interval inbetween runs, so all iterations needed for the initial sync // are done as fast as possible. This then gets updated once the initial sync completes. @@ -391,111 +394,8 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra return true; } - public async getL2Tips(): Promise { - const [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber, finalizedBlockNumber] = await Promise.all([ - this.getBlockNumber(), - this.getProvenBlockNumber(), - this.getCheckpointedL2BlockNumber(), - this.getFinalizedL2BlockNumber(), - ] as const); - - const beforeInitialblockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1); - - // Get the latest block header and checkpointed blocks for proven, finalised and checkpointed blocks - const [latestBlockHeader, provenCheckpointedBlock, finalizedCheckpointedBlock, checkpointedBlock] = - await Promise.all([ - latestBlockNumber > beforeInitialblockNumber ? this.getBlockHeader(latestBlockNumber) : undefined, - provenBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(provenBlockNumber) : undefined, - finalizedBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(finalizedBlockNumber) : undefined, - checkpointedBlockNumber > beforeInitialblockNumber - ? this.getCheckpointedBlock(checkpointedBlockNumber) - : undefined, - ] as const); - - if (latestBlockNumber > beforeInitialblockNumber && !latestBlockHeader) { - throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`); - } - - // Checkpointed blocks must exist for proven, finalized and checkpointed tips if they are beyond the initial block number. - if (checkpointedBlockNumber > beforeInitialblockNumber && !checkpointedBlock?.block.header) { - throw new Error( - `Failed to retrieve checkpointed block header for block ${checkpointedBlockNumber} (latest block is ${latestBlockNumber})`, - ); - } - - if (provenBlockNumber > beforeInitialblockNumber && !provenCheckpointedBlock?.block.header) { - throw new Error( - `Failed to retrieve proven checkpointed for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`, - ); - } - - if (finalizedBlockNumber > beforeInitialblockNumber && !finalizedCheckpointedBlock?.block.header) { - throw new Error( - `Failed to retrieve finalized block header for block ${finalizedBlockNumber} (latest block is ${latestBlockNumber})`, - ); - } - - const latestBlockHeaderHash = (await latestBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - const provenBlockHeaderHash = (await provenCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - const finalizedBlockHeaderHash = - (await finalizedCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - const checkpointedBlockHeaderHash = (await checkpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - - // Now attempt to retrieve checkpoints for proven, finalised and checkpointed blocks - const [[provenBlockCheckpoint], [finalizedBlockCheckpoint], [checkpointedBlockCheckpoint]] = await Promise.all([ - provenCheckpointedBlock !== undefined - ? await this.getCheckpoints(provenCheckpointedBlock?.checkpointNumber, 1) - : [undefined], - finalizedCheckpointedBlock !== undefined - ? await this.getCheckpoints(finalizedCheckpointedBlock?.checkpointNumber, 1) - : [undefined], - checkpointedBlock !== undefined ? await this.getCheckpoints(checkpointedBlock?.checkpointNumber, 1) : [undefined], - ]); - - const initialcheckpointId: CheckpointId = { - number: CheckpointNumber.ZERO, - hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(), - }; - - const makeCheckpointId = (checkpoint: PublishedCheckpoint | undefined) => { - if (checkpoint === undefined) { - return initialcheckpointId; - } - return { - number: checkpoint.checkpoint.number, - hash: checkpoint.checkpoint.hash().toString(), - }; - }; - - const l2Tips: L2Tips = { - proposed: { - number: latestBlockNumber, - hash: latestBlockHeaderHash.toString(), - }, - proven: { - block: { - number: provenBlockNumber, - hash: provenBlockHeaderHash.toString(), - }, - checkpoint: makeCheckpointId(provenBlockCheckpoint), - }, - finalized: { - block: { - number: finalizedBlockNumber, - hash: finalizedBlockHeaderHash.toString(), - }, - checkpoint: makeCheckpointId(finalizedBlockCheckpoint), - }, - checkpointed: { - block: { - number: checkpointedBlockNumber, - hash: checkpointedBlockHeaderHash.toString(), - }, - checkpoint: makeCheckpointId(checkpointedBlockCheckpoint), - }, - }; - - return l2Tips; + public getL2Tips(): Promise { + return this.l2TipsCache.getL2Tips(); } public async rollbackTo(targetL2BlockNumber: BlockNumber): Promise { @@ -532,7 +432,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra await this.store.setMessageSynchedL1Block({ l1BlockNumber: targetL1BlockNumber, l1BlockHash: targetL1BlockHash }); if (targetL2BlockNumber < currentProvenBlock) { this.log.info(`Clearing proven L2 block number`); - await this.store.setProvenCheckpointNumber(CheckpointNumber.ZERO); + await this.updater.setProvenCheckpointNumber(CheckpointNumber.ZERO); } // TODO(palla/reorg): Set the finalized block when we add support for it. // if (targetL2BlockNumber < currentFinalizedBlock) { diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index eeb5090c406e..dc0ca5552d85 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -25,6 +25,7 @@ import { type ArchiverConfig, mapArchiverConfig } from './config.js'; import { ArchiverInstrumentation } from './modules/instrumentation.js'; import { ArchiverL1Synchronizer } from './modules/l1_synchronizer.js'; import { ARCHIVER_DB_VERSION, KVArchiverDataStore } from './store/kv_archiver_store.js'; +import { L2TipsCache } from './store/l2_tips_cache.js'; export const ARCHIVER_STORE_NAME = 'archiver'; @@ -128,6 +129,9 @@ export async function createArchiver( // Create the event emitter that will be shared by archiver and synchronizer const events = new EventEmitter() as ArchiverEmitter; + // Create L2 tips cache shared by archiver and synchronizer + const l2TipsCache = new L2TipsCache(archiverStore.blockStore); + // Create the L1 synchronizer const synchronizer = new ArchiverL1Synchronizer( publicClient, @@ -144,6 +148,8 @@ export async function createArchiver( l1Constants, events, instrumentation.tracer, + l2TipsCache, + undefined, // log (use default) ); const archiver = new Archiver( @@ -158,6 +164,7 @@ export async function createArchiver( l1Constants, synchronizer, events, + l2TipsCache, ); await archiver.start(opts.blockUntilSync); diff --git a/yarn-project/archiver/src/index.ts b/yarn-project/archiver/src/index.ts index 224884764f17..51aa5f45706c 100644 --- a/yarn-project/archiver/src/index.ts +++ b/yarn-project/archiver/src/index.ts @@ -8,5 +8,6 @@ export * from './config.js'; export { type L1PublishedData } from './structs/published.js'; export { KVArchiverDataStore, ARCHIVER_DB_VERSION } from './store/kv_archiver_store.js'; export { ContractInstanceStore } from './store/contract_instance_store.js'; +export { L2TipsCache } from './store/l2_tips_cache.js'; export { retrieveCheckpointsFromRollup, retrieveL2ProofVerifiedEvents } from './l1/data_retrieval.js'; diff --git a/yarn-project/archiver/src/l1/calldata_retriever.test.ts b/yarn-project/archiver/src/l1/calldata_retriever.test.ts index dd4239ff6fd2..45f0a81d9db1 100644 --- a/yarn-project/archiver/src/l1/calldata_retriever.test.ts +++ b/yarn-project/archiver/src/l1/calldata_retriever.test.ts @@ -14,6 +14,7 @@ import { GasFees } from '@aztec/stdlib/gas'; import { ConsensusPayload, SignatureDomainSeparator } from '@aztec/stdlib/p2p'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; import { type Hex, @@ -1018,6 +1019,32 @@ describe('CalldataRetriever', () => { expect(debugClient.request).toHaveBeenCalledTimes(2); }); + it('should log trace+debug failure warn only once per tx hash', async () => { + CalldataRetriever.resetTraceFailureWarnedForTesting(); + const warnSpy = jest.spyOn(logger, 'warn'); + + // First attempt: both trace and debug fail + debugClient.request.mockRejectedValueOnce(new Error('trace_transaction not supported')); + debugClient.request.mockRejectedValueOnce(new Error('debug_traceTransaction not supported')); + + await expect(retriever.extractCalldataViaTrace(txHash)).rejects.toThrow( + 'Failed to trace transaction ' + txHash + ' to extract propose calldata', + ); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Cannot decode L1 tx')); + + // Second attempt: same tx, both fail again - should not log warn again + debugClient.request.mockRejectedValueOnce(new Error('trace_transaction not supported')); + debugClient.request.mockRejectedValueOnce(new Error('debug_traceTransaction not supported')); + + await expect(retriever.extractCalldataViaTrace(txHash)).rejects.toThrow( + 'Failed to trace transaction ' + txHash + ' to extract propose calldata', + ); + expect(warnSpy).toHaveBeenCalledTimes(1); + + warnSpy.mockRestore(); + }); + it('should throw when no propose calls found', async () => { // Mock debug client to return empty trace debugClient.request.mockResolvedValueOnce([]); diff --git a/yarn-project/archiver/src/l1/calldata_retriever.ts b/yarn-project/archiver/src/l1/calldata_retriever.ts index b6225aba7d1d..f023bfbac865 100644 --- a/yarn-project/archiver/src/l1/calldata_retriever.ts +++ b/yarn-project/archiver/src/l1/calldata_retriever.ts @@ -39,6 +39,14 @@ import type { CallInfo } from './types.js'; * in order to reconstruct an L2 block header. */ export class CalldataRetriever { + /** Tx hashes we've already logged for trace+debug failure (log once per tx per process). */ + private static readonly traceFailureWarnedTxHashes = new Set(); + + /** Clears the trace-failure warned set. For testing only. */ + static resetTraceFailureWarnedForTesting(): void { + CalldataRetriever.traceFailureWarnedTxHashes.clear(); + } + /** Pre-computed valid contract calls for validation */ private readonly validContractCalls: ValidContractCall[]; @@ -314,7 +322,8 @@ export class CalldataRetriever { this.logger.debug(`Successfully traced using trace_transaction, found ${calls.length} calls`); } catch (err) { const traceError = err instanceof Error ? err : new Error(String(err)); - this.logger.verbose(`Failed trace_transaction for ${txHash}`, { traceError }); + this.logger.verbose(`Failed trace_transaction for ${txHash}: ${traceError.message}`); + this.logger.debug(`Trace failure details for ${txHash}`, { traceError }); try { // Fall back to debug_traceTransaction (Geth RPC) @@ -323,7 +332,16 @@ export class CalldataRetriever { this.logger.debug(`Successfully traced using debug_traceTransaction, found ${calls.length} calls`); } catch (debugErr) { const debugError = debugErr instanceof Error ? debugErr : new Error(String(debugErr)); - this.logger.warn(`All tracing methods failed for tx ${txHash}`, { + // Log once per tx so we don't spam on every sync cycle when sync point doesn't advance + if (!CalldataRetriever.traceFailureWarnedTxHashes.has(txHash)) { + CalldataRetriever.traceFailureWarnedTxHashes.add(txHash); + this.logger.warn( + `Cannot decode L1 tx ${txHash}: trace and debug RPC failed or unavailable. ` + + `trace_transaction: ${traceError.message}; debug_traceTransaction: ${debugError.message}`, + ); + } + // Full error objects can be very long; keep at debug only + this.logger.debug(`Trace/debug failure details for tx ${txHash}`, { traceError, debugError, txHash, diff --git a/yarn-project/archiver/src/modules/data_source_base.ts b/yarn-project/archiver/src/modules/data_source_base.ts index 7be5a1b801d8..7a8cfc85f238 100644 --- a/yarn-project/archiver/src/modules/data_source_base.ts +++ b/yarn-project/archiver/src/modules/data_source_base.ts @@ -1,11 +1,12 @@ +import { range } from '@aztec/foundation/array'; import { BlockNumber, CheckpointNumber, type EpochNumber, type SlotNumber } from '@aztec/foundation/branded-types'; import type { Fr } from '@aztec/foundation/curves/bn254'; import type { EthAddress } from '@aztec/foundation/eth-address'; import { isDefined } from '@aztec/foundation/types'; import type { FunctionSelector } from '@aztec/stdlib/abi'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { type BlockHash, CheckpointedL2Block, CommitteeAttestation, L2Block, type L2Tips } from '@aztec/stdlib/block'; -import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { type BlockData, type BlockHash, CheckpointedL2Block, L2Block, type L2Tips } from '@aztec/stdlib/block'; +import { Checkpoint, type CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import { type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client'; @@ -17,7 +18,6 @@ import type { BlockHeader, IndexedTxEffect, TxHash, TxReceipt } from '@aztec/std import type { UInt64 } from '@aztec/stdlib/types'; import type { ArchiverDataSource } from '../interfaces.js'; -import type { CheckpointData } from '../store/block_store.js'; import type { KVArchiverDataStore } from '../store/kv_archiver_store.js'; import type { ValidateCheckpointResult } from './validation.js'; @@ -114,7 +114,7 @@ export abstract class ArchiverDataSourceBase if (!checkpointData) { return undefined; } - return BlockNumber(checkpointData.startBlock + checkpointData.numBlocks - 1); + return BlockNumber(checkpointData.startBlock + checkpointData.blockCount - 1); } public getCheckpointedBlocks(from: BlockNumber, limit: number): Promise { @@ -129,6 +129,14 @@ export abstract class ArchiverDataSourceBase return this.store.getBlockHeaderByArchive(archive); } + public getBlockData(number: BlockNumber): Promise { + return this.store.getBlockData(number); + } + + public getBlockDataByArchive(archive: Fr): Promise { + return this.store.getBlockDataByArchive(archive); + } + public async getL2Block(number: BlockNumber): Promise { // If the number provided is -ve, then return the latest block. if (number < 0) { @@ -223,28 +231,21 @@ export abstract class ArchiverDataSourceBase public async getCheckpoints(checkpointNumber: CheckpointNumber, limit: number): Promise { const checkpoints = await this.store.getRangeOfCheckpoints(checkpointNumber, limit); - const blocks = ( - await Promise.all(checkpoints.map(ch => this.store.getBlocksForCheckpoint(ch.checkpointNumber))) - ).filter(isDefined); - - const fullCheckpoints: PublishedCheckpoint[] = []; - for (let i = 0; i < checkpoints.length; i++) { - const blocksForCheckpoint = blocks[i]; - const checkpoint = checkpoints[i]; - const fullCheckpoint = new Checkpoint( - checkpoint.archive, - checkpoint.header, - blocksForCheckpoint, - checkpoint.checkpointNumber, - ); - const publishedCheckpoint = new PublishedCheckpoint( - fullCheckpoint, - checkpoint.l1, - checkpoint.attestations.map(x => CommitteeAttestation.fromBuffer(x)), - ); - fullCheckpoints.push(publishedCheckpoint); + return Promise.all(checkpoints.map(ch => this.getPublishedCheckpointFromCheckpointData(ch))); + } + + private async getPublishedCheckpointFromCheckpointData(checkpoint: CheckpointData): Promise { + const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpoint.checkpointNumber); + if (!blocksForCheckpoint) { + throw new Error(`Blocks for checkpoint ${checkpoint.checkpointNumber} not found`); } - return fullCheckpoints; + const fullCheckpoint = new Checkpoint( + checkpoint.archive, + checkpoint.header, + blocksForCheckpoint, + checkpoint.checkpointNumber, + ); + return new PublishedCheckpoint(fullCheckpoint, checkpoint.l1, checkpoint.attestations); } public getBlocksForSlot(slotNumber: SlotNumber): Promise { @@ -252,84 +253,44 @@ export abstract class ArchiverDataSourceBase } public async getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { - if (!this.l1Constants) { - throw new Error('L1 constants not set'); - } - - const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const blocks: CheckpointedL2Block[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpoint && slot(checkpoint) >= start) { - if (slot(checkpoint) <= end) { - // push the blocks on backwards - const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1; - for (let i = endBlock; i >= checkpoint.startBlock; i--) { - const checkpointedBlock = await this.getCheckpointedBlock(BlockNumber(i)); - if (checkpointedBlock) { - blocks.push(checkpointedBlock); - } - } - } - checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1)); - } - - return blocks.reverse(); + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + const blocks = await Promise.all( + checkpointsData.flatMap(checkpoint => + range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber => + this.getCheckpointedBlock(BlockNumber(blockNumber)), + ), + ), + ); + return blocks.filter(isDefined); } public async getCheckpointedBlockHeadersForEpoch(epochNumber: EpochNumber): Promise { - if (!this.l1Constants) { - throw new Error('L1 constants not set'); - } - - const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const blocks: BlockHeader[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpoint = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpoint && slot(checkpoint) >= start) { - if (slot(checkpoint) <= end) { - // push the blocks on backwards - const endBlock = checkpoint.startBlock + checkpoint.numBlocks - 1; - for (let i = endBlock; i >= checkpoint.startBlock; i--) { - const block = await this.getBlockHeader(BlockNumber(i)); - if (block) { - blocks.push(block); - } - } - } - checkpoint = await this.store.getCheckpointData(CheckpointNumber(checkpoint.checkpointNumber - 1)); - } - return blocks.reverse(); + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + const blocks = await Promise.all( + checkpointsData.flatMap(checkpoint => + range(checkpoint.blockCount, checkpoint.startBlock).map(blockNumber => + this.getBlockHeader(BlockNumber(blockNumber)), + ), + ), + ); + return blocks.filter(isDefined); } public async getCheckpointsForEpoch(epochNumber: EpochNumber): Promise { + const checkpointsData = await this.getCheckpointsDataForEpoch(epochNumber); + return Promise.all( + checkpointsData.map(data => this.getPublishedCheckpointFromCheckpointData(data).then(p => p.checkpoint)), + ); + } + + /** Returns checkpoint data for all checkpoints whose slot falls within the given epoch. */ + public getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { if (!this.l1Constants) { throw new Error('L1 constants not set'); } const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1Constants); - const checkpoints: Checkpoint[] = []; - - // Walk the list of checkpoints backwards and filter by slots matching the requested epoch. - // We'll typically ask for checkpoints for a very recent epoch, so we shouldn't need an index here. - let checkpointData = await this.store.getCheckpointData(await this.store.getSynchedCheckpointNumber()); - const slot = (b: CheckpointData) => b.header.slotNumber; - while (checkpointData && slot(checkpointData) >= start) { - if (slot(checkpointData) <= end) { - // push the checkpoints on backwards - const [checkpoint] = await this.getCheckpoints(checkpointData.checkpointNumber, 1); - checkpoints.push(checkpoint.checkpoint); - } - checkpointData = await this.store.getCheckpointData(CheckpointNumber(checkpointData.checkpointNumber - 1)); - } - - return checkpoints.reverse(); + return this.store.getCheckpointDataForSlotRange(start, end); } public async getBlock(number: BlockNumber): Promise { diff --git a/yarn-project/archiver/src/modules/data_store_updater.ts b/yarn-project/archiver/src/modules/data_store_updater.ts index 1df274146880..dd2e6becd57a 100644 --- a/yarn-project/archiver/src/modules/data_store_updater.ts +++ b/yarn-project/archiver/src/modules/data_store_updater.ts @@ -25,6 +25,7 @@ import type { UInt64 } from '@aztec/stdlib/types'; import groupBy from 'lodash.groupby'; import type { KVArchiverDataStore } from '../store/kv_archiver_store.js'; +import type { L2TipsCache } from '../store/l2_tips_cache.js'; /** Operation type for contract data updates. */ enum Operation { @@ -44,7 +45,10 @@ type ReconcileCheckpointsResult = { export class ArchiverDataStoreUpdater { private readonly log = createLogger('archiver:store_updater'); - constructor(private store: KVArchiverDataStore) {} + constructor( + private store: KVArchiverDataStore, + private l2TipsCache?: L2TipsCache, + ) {} /** * Adds proposed blocks to the store with contract class/instance extraction from logs. @@ -56,11 +60,11 @@ export class ArchiverDataStoreUpdater { * @param pendingChainValidationStatus - Optional validation status to set. * @returns True if the operation is successful. */ - public addProposedBlocks( + public async addProposedBlocks( blocks: L2Block[], pendingChainValidationStatus?: ValidateCheckpointResult, ): Promise { - return this.store.transactionAsync(async () => { + const result = await this.store.transactionAsync(async () => { await this.store.addProposedBlocks(blocks); const opResults = await Promise.all([ @@ -72,8 +76,10 @@ export class ArchiverDataStoreUpdater { ...blocks.map(block => this.addContractDataToDb(block)), ]); + await this.l2TipsCache?.refresh(); return opResults.every(Boolean); }); + return result; } /** @@ -87,11 +93,11 @@ export class ArchiverDataStoreUpdater { * @param pendingChainValidationStatus - Optional validation status to set. * @returns Result with information about any pruned blocks. */ - public addCheckpoints( + public async addCheckpoints( checkpoints: PublishedCheckpoint[], pendingChainValidationStatus?: ValidateCheckpointResult, ): Promise { - return this.store.transactionAsync(async () => { + const result = await this.store.transactionAsync(async () => { // Before adding checkpoints, check for conflicts with local blocks if any const { prunedBlocks, lastAlreadyInsertedBlockNumber } = await this.pruneMismatchingLocalBlocks(checkpoints); @@ -111,8 +117,10 @@ export class ArchiverDataStoreUpdater { ...newBlocks.map(block => this.addContractDataToDb(block)), ]); + await this.l2TipsCache?.refresh(); return { prunedBlocks, lastAlreadyInsertedBlockNumber }; }); + return result; } /** @@ -197,8 +205,8 @@ export class ArchiverDataStoreUpdater { * @returns The removed blocks. * @throws Error if any block to be removed is checkpointed. */ - public removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise { - return this.store.transactionAsync(async () => { + public async removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise { + const result = await this.store.transactionAsync(async () => { // Verify we're only removing uncheckpointed blocks const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber(); if (blockNumber < lastCheckpointedBlockNumber) { @@ -207,8 +215,11 @@ export class ArchiverDataStoreUpdater { ); } - return await this.removeBlocksAfter(blockNumber); + const result = await this.removeBlocksAfter(blockNumber); + await this.l2TipsCache?.refresh(); + return result; }); + return result; } /** @@ -238,17 +249,31 @@ export class ArchiverDataStoreUpdater { * @returns True if the operation is successful. */ public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise { - const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber); - - const opResults = await Promise.all([ - // Prune rolls back to the last proven block, which is by definition valid - this.store.setPendingChainValidationStatus({ valid: true }), - // Remove contract data for all blocks being removed - ...blocksRemoved.map(block => this.removeContractDataFromDb(block)), - this.store.deleteLogs(blocksRemoved), - ]); + return await this.store.transactionAsync(async () => { + const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber); + + const opResults = await Promise.all([ + // Prune rolls back to the last proven block, which is by definition valid + this.store.setPendingChainValidationStatus({ valid: true }), + // Remove contract data for all blocks being removed + ...blocksRemoved.map(block => this.removeContractDataFromDb(block)), + this.store.deleteLogs(blocksRemoved), + ]); - return opResults.every(Boolean); + await this.l2TipsCache?.refresh(); + return opResults.every(Boolean); + }); + } + + /** + * Updates the proven checkpoint number and refreshes the L2 tips cache. + * @param checkpointNumber - The checkpoint number to set as proven. + */ + public async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise { + await this.store.transactionAsync(async () => { + await this.store.setProvenCheckpointNumber(checkpointNumber); + await this.l2TipsCache?.refresh(); + }); } /** Extracts and stores contract data from a single block. */ diff --git a/yarn-project/archiver/src/modules/instrumentation.ts b/yarn-project/archiver/src/modules/instrumentation.ts index 57f7c6413f75..fbf91cf16a1a 100644 --- a/yarn-project/archiver/src/modules/instrumentation.ts +++ b/yarn-project/archiver/src/modules/instrumentation.ts @@ -1,5 +1,6 @@ import { createLogger } from '@aztec/foundation/log'; import type { L2Block } from '@aztec/stdlib/block'; +import type { CheckpointData } from '@aztec/stdlib/checkpoint'; import { Attributes, type Gauge, @@ -17,6 +18,7 @@ export class ArchiverInstrumentation { public readonly tracer: Tracer; private blockHeight: Gauge; + private checkpointHeight: Gauge; private txCount: UpDownCounter; private l1BlockHeight: Gauge; private proofsSubmittedDelay: Histogram; @@ -47,6 +49,8 @@ export class ArchiverInstrumentation { this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT); + this.checkpointHeight = meter.createGauge(Metrics.ARCHIVER_CHECKPOINT_HEIGHT); + this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT); this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS); @@ -105,6 +109,7 @@ export class ArchiverInstrumentation { public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) { this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock)); this.blockHeight.record(Math.max(...blocks.map(b => b.number))); + this.checkpointHeight.record(Math.max(...blocks.map(b => b.checkpointNumber))); this.syncBlockCount.add(blocks.length); for (const block of blocks) { @@ -127,8 +132,10 @@ export class ArchiverInstrumentation { this.pruneDuration.record(Math.ceil(duration)); } - public updateLastProvenBlock(blockNumber: number) { - this.blockHeight.record(blockNumber, { [Attributes.STATUS]: 'proven' }); + public updateLastProvenCheckpoint(checkpoint: CheckpointData) { + const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1; + this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' }); + this.checkpointHeight.record(checkpoint.checkpointNumber, { [Attributes.STATUS]: 'proven' }); } public processProofsVerified(logs: { proverId: string; l2BlockNumber: bigint; delay: bigint }[]) { diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index 7d8992c09616..22b1ed5aba29 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -28,6 +28,7 @@ import { retrievedToPublishedCheckpoint, } from '../l1/data_retrieval.js'; import type { KVArchiverDataStore } from '../store/kv_archiver_store.js'; +import type { L2TipsCache } from '../store/l2_tips_cache.js'; import type { InboxMessage } from '../structs/inbox_message.js'; import { ArchiverDataStoreUpdater } from './data_store_updater.js'; import type { ArchiverInstrumentation } from './instrumentation.js'; @@ -77,9 +78,10 @@ export class ArchiverL1Synchronizer implements Traceable { private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr }, private readonly events: ArchiverEmitter, tracer: Tracer, + l2TipsCache?: L2TipsCache, private readonly log: Logger = createLogger('archiver:l1-sync'), ) { - this.updater = new ArchiverDataStoreUpdater(this.store); + this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache); this.tracer = tracer; } @@ -550,7 +552,7 @@ export class ArchiverL1Synchronizer implements Traceable { if (provenCheckpointNumber === 0) { const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber(); if (localProvenCheckpointNumber !== provenCheckpointNumber) { - await this.store.setProvenCheckpointNumber(provenCheckpointNumber); + await this.updater.setProvenCheckpointNumber(provenCheckpointNumber); this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber }); } } @@ -582,13 +584,13 @@ export class ArchiverL1Synchronizer implements Traceable { ) { const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber(); if (localProvenCheckpointNumber !== provenCheckpointNumber) { - await this.store.setProvenCheckpointNumber(provenCheckpointNumber); + await this.updater.setProvenCheckpointNumber(provenCheckpointNumber); this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber }); const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber; const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants); const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + - localCheckpointForDestinationProvenCheckpointNumber.numBlocks - + localCheckpointForDestinationProvenCheckpointNumber.blockCount - 1; this.events.emit(L2BlockSourceEvents.L2BlockProven, { @@ -597,7 +599,7 @@ export class ArchiverL1Synchronizer implements Traceable { slotNumber: provenSlotNumber, epochNumber: provenEpochNumber, }); - this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint); + this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber); } else { this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`); } diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 732fa7e13c3b..a9ec9a501c85 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -9,6 +9,7 @@ import { isDefined } from '@aztec/foundation/types'; import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncSingleton, Range } from '@aztec/kv-store'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { + type BlockData, BlockHash, Body, CheckpointedL2Block, @@ -18,7 +19,7 @@ import { deserializeValidateCheckpointResult, serializeValidateCheckpointResult, } from '@aztec/stdlib/block'; -import { L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { type CheckpointData, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; @@ -61,23 +62,14 @@ type BlockStorage = { type CheckpointStorage = { header: Buffer; archive: Buffer; + checkpointOutHash: Buffer; checkpointNumber: number; startBlock: number; - numBlocks: number; + blockCount: number; l1: Buffer; attestations: Buffer[]; }; -export type CheckpointData = { - checkpointNumber: CheckpointNumber; - header: CheckpointHeader; - archive: AppendOnlyTreeSnapshot; - startBlock: number; - numBlocks: number; - l1: L1PublishedData; - attestations: Buffer[]; -}; - export type RemoveCheckpointsResult = { blocksRemoved: L2Block[] | undefined }; /** @@ -90,6 +82,9 @@ export class BlockStore { /** Map checkpoint number to checkpoint data */ #checkpoints: AztecAsyncMap; + /** Map slot number to checkpoint number, for looking up checkpoints by slot range. */ + #slotToCheckpoint: AztecAsyncMap; + /** Map block hash to list of tx hashes */ #blockTxs: AztecAsyncMap; @@ -130,6 +125,7 @@ export class BlockStore { this.#lastProvenCheckpoint = db.openSingleton('archiver_last_proven_l2_checkpoint'); this.#pendingChainValidationStatus = db.openSingleton('archiver_pending_chain_validation_status'); this.#checkpoints = db.openMap('archiver_checkpoints'); + this.#slotToCheckpoint = db.openMap('archiver_slot_to_checkpoint'); } /** @@ -273,7 +269,7 @@ export class BlockStore { // If we have a previous checkpoint then we need to get the previous block number if (previousCheckpointData !== undefined) { - previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.numBlocks - 1); + previousBlockNumber = BlockNumber(previousCheckpointData.startBlock + previousCheckpointData.blockCount - 1); previousBlock = await this.getBlock(previousBlockNumber); if (previousBlock === undefined) { // We should be able to get the required previous block @@ -337,12 +333,16 @@ export class BlockStore { await this.#checkpoints.set(checkpoint.checkpoint.number, { header: checkpoint.checkpoint.header.toBuffer(), archive: checkpoint.checkpoint.archive.toBuffer(), + checkpointOutHash: checkpoint.checkpoint.getCheckpointOutHash().toBuffer(), l1: checkpoint.l1.toBuffer(), attestations: checkpoint.attestations.map(attestation => attestation.toBuffer()), checkpointNumber: checkpoint.checkpoint.number, startBlock: checkpoint.checkpoint.blocks[0].number, - numBlocks: checkpoint.checkpoint.blocks.length, + blockCount: checkpoint.checkpoint.blocks.length, }); + + // Update slot-to-checkpoint index + await this.#slotToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, checkpoint.checkpoint.number); } await this.#lastSynchedL1Block.set(checkpoints[checkpoints.length - 1].l1.blockNumber); @@ -425,7 +425,7 @@ export class BlockStore { if (!targetCheckpoint) { throw new Error(`Target checkpoint ${checkpointNumber} not found in store`); } - lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.numBlocks - 1); + lastBlockToKeep = BlockNumber(targetCheckpoint.startBlock + targetCheckpoint.blockCount - 1); } // Remove all blocks after lastBlockToKeep (both checkpointed and uncheckpointed) @@ -433,6 +433,11 @@ export class BlockStore { // Remove all checkpoints after the target for (let c = latestCheckpointNumber; c > checkpointNumber; c = CheckpointNumber(c - 1)) { + const checkpointStorage = await this.#checkpoints.getAsync(c); + if (checkpointStorage) { + const slotNumber = CheckpointHeader.fromBuffer(checkpointStorage.header).slotNumber; + await this.#slotToCheckpoint.delete(slotNumber); + } await this.#checkpoints.delete(c); this.#log.debug(`Removed checkpoint ${c}`); } @@ -461,17 +466,32 @@ export class BlockStore { return checkpoints; } - private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage) { - const data: CheckpointData = { + /** Returns checkpoint data for all checkpoints whose slot falls within the given range (inclusive). */ + async getCheckpointDataForSlotRange(startSlot: SlotNumber, endSlot: SlotNumber): Promise { + const result: CheckpointData[] = []; + for await (const [, checkpointNumber] of this.#slotToCheckpoint.entriesAsync({ + start: startSlot, + end: endSlot + 1, + })) { + const checkpointStorage = await this.#checkpoints.getAsync(checkpointNumber); + if (checkpointStorage) { + result.push(this.checkpointDataFromCheckpointStorage(checkpointStorage)); + } + } + return result; + } + + private checkpointDataFromCheckpointStorage(checkpointStorage: CheckpointStorage): CheckpointData { + return { header: CheckpointHeader.fromBuffer(checkpointStorage.header), archive: AppendOnlyTreeSnapshot.fromBuffer(checkpointStorage.archive), + checkpointOutHash: Fr.fromBuffer(checkpointStorage.checkpointOutHash), checkpointNumber: CheckpointNumber(checkpointStorage.checkpointNumber), - startBlock: checkpointStorage.startBlock, - numBlocks: checkpointStorage.numBlocks, + startBlock: BlockNumber(checkpointStorage.startBlock), + blockCount: checkpointStorage.blockCount, l1: L1PublishedData.fromBuffer(checkpointStorage.l1), - attestations: checkpointStorage.attestations, + attestations: checkpointStorage.attestations.map(buf => CommitteeAttestation.fromBuffer(buf)), }; - return data; } async getBlocksForCheckpoint(checkpointNumber: CheckpointNumber): Promise { @@ -483,7 +503,7 @@ export class BlockStore { const blocksForCheckpoint = await toArray( this.#blocks.entriesAsync({ start: checkpoint.startBlock, - end: checkpoint.startBlock + checkpoint.numBlocks, + end: checkpoint.startBlock + checkpoint.blockCount, }), ); @@ -556,7 +576,7 @@ export class BlockStore { if (!checkpointStorage) { throw new CheckpointNotFoundError(provenCheckpointNumber); } else { - return BlockNumber(checkpointStorage.startBlock + checkpointStorage.numBlocks - 1); + return BlockNumber(checkpointStorage.startBlock + checkpointStorage.blockCount - 1); } } @@ -655,6 +675,32 @@ export class BlockStore { } } + /** + * Gets block metadata (without tx data) by block number. + * @param blockNumber - The number of the block to return. + * @returns The requested block data. + */ + async getBlockData(blockNumber: BlockNumber): Promise { + const blockStorage = await this.#blocks.getAsync(blockNumber); + if (!blockStorage || !blockStorage.header) { + return undefined; + } + return this.getBlockDataFromBlockStorage(blockStorage); + } + + /** + * Gets block metadata (without tx data) by archive root. + * @param archive - The archive root of the block to return. + * @returns The requested block data. + */ + async getBlockDataByArchive(archive: Fr): Promise { + const blockNumber = await this.#blockArchiveIndex.getAsync(archive.toString()); + if (blockNumber === undefined) { + return undefined; + } + return this.getBlockData(BlockNumber(blockNumber)); + } + /** * Gets an L2 block. * @param blockNumber - The number of the block to return. @@ -759,15 +805,24 @@ export class BlockStore { } } + private getBlockDataFromBlockStorage(blockStorage: BlockStorage): BlockData { + return { + header: BlockHeader.fromBuffer(blockStorage.header), + archive: AppendOnlyTreeSnapshot.fromBuffer(blockStorage.archive), + blockHash: Fr.fromBuffer(blockStorage.blockHash), + checkpointNumber: CheckpointNumber(blockStorage.checkpointNumber), + indexWithinCheckpoint: IndexWithinCheckpoint(blockStorage.indexWithinCheckpoint), + }; + } + private async getBlockFromBlockStorage( blockNumber: number, blockStorage: BlockStorage, ): Promise { - const header = BlockHeader.fromBuffer(blockStorage.header); - const archive = AppendOnlyTreeSnapshot.fromBuffer(blockStorage.archive); - const blockHash = blockStorage.blockHash; - header.setHash(Fr.fromBuffer(blockHash)); - const blockHashString = bufferToHex(blockHash); + const { header, archive, blockHash, checkpointNumber, indexWithinCheckpoint } = + this.getBlockDataFromBlockStorage(blockStorage); + header.setHash(blockHash); + const blockHashString = bufferToHex(blockStorage.blockHash); const blockTxsBuffer = await this.#blockTxs.getAsync(blockHashString); if (blockTxsBuffer === undefined) { this.#log.warn(`Could not find body for block ${header.globalVariables.blockNumber} ${blockHash}`); @@ -786,13 +841,7 @@ export class BlockStore { txEffects.push(deserializeIndexedTxEffect(txEffect).data); } const body = new Body(txEffects); - const block = new L2Block( - archive, - header, - body, - CheckpointNumber(blockStorage.checkpointNumber!), - IndexWithinCheckpoint(blockStorage.indexWithinCheckpoint), - ); + const block = new L2Block(archive, header, body, checkpointNumber, indexWithinCheckpoint); if (block.number !== blockNumber) { throw new Error( @@ -892,7 +941,7 @@ export class BlockStore { if (!checkpoint) { return BlockNumber(INITIAL_L2_BLOCK_NUM - 1); } - return BlockNumber(checkpoint.startBlock + checkpoint.numBlocks - 1); + return BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1); } async getLatestL2BlockNumber(): Promise { diff --git a/yarn-project/archiver/src/store/kv_archiver_store.test.ts b/yarn-project/archiver/src/store/kv_archiver_store.test.ts index f0100cc2cce0..d05044ded8d2 100644 --- a/yarn-project/archiver/src/store/kv_archiver_store.test.ts +++ b/yarn-project/archiver/src/store/kv_archiver_store.test.ts @@ -1474,7 +1474,7 @@ describe('KVArchiverDataStore', () => { expect(checkpoints.length).toBe(1); expect(checkpoints[0].checkpointNumber).toBe(1); expect(checkpoints[0].startBlock).toBe(1); - expect(checkpoints[0].numBlocks).toBe(2); + expect(checkpoints[0].blockCount).toBe(2); }); it('returns multiple checkpoints in order', async () => { @@ -1504,7 +1504,7 @@ describe('KVArchiverDataStore', () => { expect(checkpoints.length).toBe(3); expect(checkpoints.map(c => c.checkpointNumber)).toEqual([1, 2, 3]); expect(checkpoints.map(c => c.startBlock)).toEqual([1, 3, 6]); - expect(checkpoints.map(c => c.numBlocks)).toEqual([2, 3, 1]); + expect(checkpoints.map(c => c.blockCount)).toEqual([2, 3, 1]); }); it('respects the from parameter', async () => { @@ -1586,7 +1586,7 @@ describe('KVArchiverDataStore', () => { const data = checkpoints[0]; expect(data.checkpointNumber).toBe(1); expect(data.startBlock).toBe(1); - expect(data.numBlocks).toBe(3); + expect(data.blockCount).toBe(3); expect(data.l1.blockNumber).toBe(42n); expect(data.header.equals(checkpoint.checkpoint.header)).toBe(true); expect(data.archive.equals(checkpoint.checkpoint.archive)).toBe(true); diff --git a/yarn-project/archiver/src/store/kv_archiver_store.ts b/yarn-project/archiver/src/store/kv_archiver_store.ts index 2be54f985f2d..d46075e2a588 100644 --- a/yarn-project/archiver/src/store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/store/kv_archiver_store.ts @@ -6,8 +6,14 @@ import { createLogger } from '@aztec/foundation/log'; import type { AztecAsyncKVStore, CustomRange, StoreSize } from '@aztec/kv-store'; import { FunctionSelector } from '@aztec/stdlib/abi'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { BlockHash, CheckpointedL2Block, L2Block, type ValidateCheckpointResult } from '@aztec/stdlib/block'; -import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { + type BlockData, + BlockHash, + CheckpointedL2Block, + L2Block, + type ValidateCheckpointResult, +} from '@aztec/stdlib/block'; +import type { CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, @@ -25,7 +31,7 @@ import type { UInt64 } from '@aztec/stdlib/types'; import { join } from 'path'; import type { InboxMessage } from '../structs/inbox_message.js'; -import { BlockStore, type CheckpointData, type RemoveCheckpointsResult } from './block_store.js'; +import { BlockStore, type RemoveCheckpointsResult } from './block_store.js'; import { ContractClassStore } from './contract_class_store.js'; import { ContractInstanceStore } from './contract_instance_store.js'; import { LogStore } from './log_store.js'; @@ -74,6 +80,11 @@ export class KVArchiverDataStore implements ContractDataSource { this.#contractInstanceStore = new ContractInstanceStore(db); } + /** Returns the underlying block store. Used by L2TipsCache. */ + get blockStore(): BlockStore { + return this.#blockStore; + } + /** Opens a new transaction to the underlying store and runs all operations within it. */ public transactionAsync(callback: () => Promise): Promise { return this.db.transactionAsync(callback); @@ -369,6 +380,22 @@ export class KVArchiverDataStore implements ContractDataSource { return this.#blockStore.getBlockHeaderByArchive(archive); } + /** + * Gets block metadata (without tx data) by block number. + * @param blockNumber - The block number to return. + */ + getBlockData(blockNumber: BlockNumber): Promise { + return this.#blockStore.getBlockData(blockNumber); + } + + /** + * Gets block metadata (without tx data) by archive root. + * @param archive - The archive root to return. + */ + getBlockDataByArchive(archive: Fr): Promise { + return this.#blockStore.getBlockDataByArchive(archive); + } + /** * Gets a tx effect. * @param txHash - The hash of the tx corresponding to the tx effect. @@ -618,6 +645,11 @@ export class KVArchiverDataStore implements ContractDataSource { return this.#blockStore.getCheckpointData(checkpointNumber); } + /** Returns checkpoint data for all checkpoints whose slot falls within the given range (inclusive). */ + getCheckpointDataForSlotRange(startSlot: SlotNumber, endSlot: SlotNumber): Promise { + return this.#blockStore.getCheckpointDataForSlotRange(startSlot, endSlot); + } + /** * Gets all blocks that have the given slot number. * @param slotNumber - The slot number to search for. diff --git a/yarn-project/archiver/src/store/l2_tips_cache.ts b/yarn-project/archiver/src/store/l2_tips_cache.ts new file mode 100644 index 000000000000..64a0192e7624 --- /dev/null +++ b/yarn-project/archiver/src/store/l2_tips_cache.ts @@ -0,0 +1,89 @@ +import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; +import { type BlockData, type CheckpointId, GENESIS_CHECKPOINT_HEADER_HASH, type L2Tips } from '@aztec/stdlib/block'; + +import type { BlockStore } from './block_store.js'; + +/** + * In-memory cache for L2 chain tips (proposed, checkpointed, proven, finalized). + * Populated from the BlockStore on first access, then kept up-to-date by the ArchiverDataStoreUpdater. + * Refresh calls should happen within the store transaction that mutates block data to ensure consistency. + */ +export class L2TipsCache { + #tipsPromise: Promise | undefined; + + constructor(private blockStore: BlockStore) {} + + /** Returns the cached L2 tips. Loads from the block store on first call. */ + public getL2Tips(): Promise { + return (this.#tipsPromise ??= this.loadFromStore()); + } + + /** Reloads the L2 tips from the block store. Should be called within the store transaction that mutates data. */ + public async refresh(): Promise { + this.#tipsPromise = this.loadFromStore(); + await this.#tipsPromise; + } + + private async loadFromStore(): Promise { + const [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber, finalizedBlockNumber] = await Promise.all([ + this.blockStore.getLatestBlockNumber(), + this.blockStore.getProvenBlockNumber(), + this.blockStore.getCheckpointedL2BlockNumber(), + this.blockStore.getFinalizedL2BlockNumber(), + ]); + + const genesisBlockHeader = { + blockHash: GENESIS_BLOCK_HEADER_HASH, + checkpointNumber: CheckpointNumber.ZERO, + } as const; + const beforeInitialBlockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1); + + const getBlockData = (blockNumber: BlockNumber) => + blockNumber > beforeInitialBlockNumber ? this.blockStore.getBlockData(blockNumber) : genesisBlockHeader; + + const [latestBlockData, provenBlockData, checkpointedBlockData, finalizedBlockData] = await Promise.all( + [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber, finalizedBlockNumber].map(getBlockData), + ); + + if (!latestBlockData || !provenBlockData || !finalizedBlockData || !checkpointedBlockData) { + throw new Error('Failed to load block data for L2 tips'); + } + + const [provenCheckpointId, finalizedCheckpointId, checkpointedCheckpointId] = await Promise.all([ + this.getCheckpointIdForBlock(provenBlockData), + this.getCheckpointIdForBlock(finalizedBlockData), + this.getCheckpointIdForBlock(checkpointedBlockData), + ]); + + return { + proposed: { number: latestBlockNumber, hash: latestBlockData.blockHash.toString() }, + proven: { + block: { number: provenBlockNumber, hash: provenBlockData.blockHash.toString() }, + checkpoint: provenCheckpointId, + }, + finalized: { + block: { number: finalizedBlockNumber, hash: finalizedBlockData.blockHash.toString() }, + checkpoint: finalizedCheckpointId, + }, + checkpointed: { + block: { number: checkpointedBlockNumber, hash: checkpointedBlockData.blockHash.toString() }, + checkpoint: checkpointedCheckpointId, + }, + }; + } + + private async getCheckpointIdForBlock(blockData: Pick): Promise { + const checkpointData = await this.blockStore.getCheckpointData(blockData.checkpointNumber); + if (!checkpointData) { + return { + number: CheckpointNumber.ZERO, + hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(), + }; + } + return { + number: checkpointData.checkpointNumber, + hash: checkpointData.header.hash().toString(), + }; + } +} diff --git a/yarn-project/archiver/src/test/fake_l1_state.ts b/yarn-project/archiver/src/test/fake_l1_state.ts index 6ff0a0d3dcb9..e55a234b544b 100644 --- a/yarn-project/archiver/src/test/fake_l1_state.ts +++ b/yarn-project/archiver/src/test/fake_l1_state.ts @@ -131,6 +131,7 @@ export class FakeL1State { private provenCheckpointNumber: CheckpointNumber = CheckpointNumber(0); private targetCommitteeSize: number = 0; private version: bigint = 1n; + private canPruneResult: boolean = false; // Computed from checkpoints based on L1 block visibility private pendingCheckpointNumber: CheckpointNumber = CheckpointNumber(0); @@ -194,9 +195,9 @@ export class FakeL1State { this.addMessages(checkpointNumber, messagesL1BlockNumber, messages); // Create the transaction and blobs - const tx = this.makeRollupTx(checkpoint, signers); - const blobHashes = this.makeVersionedBlobHashes(checkpoint); - const blobs = this.makeBlobsFromCheckpoint(checkpoint); + const tx = await this.makeRollupTx(checkpoint, signers); + const blobHashes = await this.makeVersionedBlobHashes(checkpoint); + const blobs = await this.makeBlobsFromCheckpoint(checkpoint); // Store the checkpoint data this.checkpoints.push({ @@ -276,6 +277,11 @@ export class FakeL1State { this.targetCommitteeSize = size; } + /** Sets whether the rollup contract would allow pruning at the next block. */ + setCanPrune(value: boolean): void { + this.canPruneResult = value; + } + /** * Removes all entries for a checkpoint number (simulates L1 reorg or prune). * Note: Does NOT remove messages for this checkpoint (use numL1ToL2Messages: 0 when re-adding). @@ -384,6 +390,8 @@ export class FakeL1State { }); }); + mockRollup.canPruneAtTime.mockImplementation(() => Promise.resolve(this.canPruneResult)); + // Mock the wrapper method for fetching checkpoint events mockRollup.getCheckpointProposedEvents.mockImplementation((fromBlock: bigint, toBlock: bigint) => Promise.resolve(this.getCheckpointProposedLogs(fromBlock, toBlock)), @@ -531,14 +539,14 @@ export class FakeL1State { })); } - private makeRollupTx(checkpoint: Checkpoint, signers: Secp256k1Signer[]): Transaction { + private async makeRollupTx(checkpoint: Checkpoint, signers: Secp256k1Signer[]): Promise { const attestations = signers .map(signer => makeCheckpointAttestationFromCheckpoint(checkpoint, signer)) .map(attestation => CommitteeAttestation.fromSignature(attestation.signature)) .map(committeeAttestation => committeeAttestation.toViem()); const header = checkpoint.header.toViem(); - const blobInput = getPrefixedEthBlobCommitments(getBlobsPerL1Block(checkpoint.toBlobFields())); + const blobInput = getPrefixedEthBlobCommitments(await getBlobsPerL1Block(checkpoint.toBlobFields())); const archive = toHex(checkpoint.archive.root.toBuffer()); const attestationsAndSigners = new CommitteeAttestationsAndSigners( attestations.map(attestation => CommitteeAttestation.fromViem(attestation)), @@ -587,13 +595,13 @@ export class FakeL1State { } as Transaction; } - private makeVersionedBlobHashes(checkpoint: Checkpoint): `0x${string}`[] { - return getBlobsPerL1Block(checkpoint.toBlobFields()).map( + private async makeVersionedBlobHashes(checkpoint: Checkpoint): Promise<`0x${string}`[]> { + return (await getBlobsPerL1Block(checkpoint.toBlobFields())).map( b => `0x${b.getEthVersionedBlobHash().toString('hex')}` as `0x${string}`, ); } - private makeBlobsFromCheckpoint(checkpoint: Checkpoint): Blob[] { - return getBlobsPerL1Block(checkpoint.toBlobFields()); + private async makeBlobsFromCheckpoint(checkpoint: Checkpoint): Promise { + return await getBlobsPerL1Block(checkpoint.toBlobFields()); } } diff --git a/yarn-project/archiver/src/test/mock_archiver.ts b/yarn-project/archiver/src/test/mock_archiver.ts index a613dabf011b..bcdcc3928d96 100644 --- a/yarn-project/archiver/src/test/mock_archiver.ts +++ b/yarn-project/archiver/src/test/mock_archiver.ts @@ -56,8 +56,9 @@ export class MockPrefilledArchiver extends MockArchiver { } const fromBlock = this.l2Blocks.length; - // TODO: Add L2 blocks and checkpoints separately once archiver has the apis for that. - this.addProposedBlocks(this.prefilled.slice(fromBlock, fromBlock + numBlocks).flatMap(c => c.blocks)); + const checkpointsToAdd = this.prefilled.slice(fromBlock, fromBlock + numBlocks); + this.addProposedBlocks(checkpointsToAdd.flatMap(c => c.blocks)); + this.checkpointList.push(...checkpointsToAdd); return Promise.resolve(); } } diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 3d06e42e7391..ff4a0fe4af52 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -8,6 +8,7 @@ import { createLogger } from '@aztec/foundation/log'; import type { FunctionSelector } from '@aztec/stdlib/abi'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { + type BlockData, BlockHash, CheckpointedL2Block, L2Block, @@ -15,9 +16,11 @@ import { type L2Tips, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; -import { Checkpoint, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { Checkpoint, type CheckpointData, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import { EmptyL1RollupConstants, type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; +import { computeCheckpointOutHash } from '@aztec/stdlib/messaging'; +import { CheckpointHeader } from '@aztec/stdlib/rollup'; import { type BlockHeader, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; import type { UInt64 } from '@aztec/stdlib/types'; @@ -26,6 +29,7 @@ import type { UInt64 } from '@aztec/stdlib/types'; */ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { protected l2Blocks: L2Block[] = []; + protected checkpointList: Checkpoint[] = []; private provenBlockNumber: number = 0; private finalizedBlockNumber: number = 0; @@ -33,14 +37,30 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { private log = createLogger('archiver:mock_l2_block_source'); + /** Creates blocks grouped into single-block checkpoints. */ public async createBlocks(numBlocks: number) { - for (let i = 0; i < numBlocks; i++) { - const blockNum = this.l2Blocks.length + 1; - const block = await L2Block.random(BlockNumber(blockNum), { slotNumber: SlotNumber(blockNum) }); - this.l2Blocks.push(block); + await this.createCheckpoints(numBlocks, 1); + } + + /** Creates checkpoints, each containing `blocksPerCheckpoint` blocks. */ + public async createCheckpoints(numCheckpoints: number, blocksPerCheckpoint: number = 1) { + for (let c = 0; c < numCheckpoints; c++) { + const checkpointNum = CheckpointNumber(this.checkpointList.length + 1); + const startBlockNum = this.l2Blocks.length + 1; + const slotNumber = SlotNumber(Number(checkpointNum)); + const checkpoint = await Checkpoint.random(checkpointNum, { + numBlocks: blocksPerCheckpoint, + startBlockNumber: startBlockNum, + slotNumber, + checkpointNumber: checkpointNum, + }); + this.checkpointList.push(checkpoint); + this.l2Blocks.push(...checkpoint.blocks); } - this.log.verbose(`Created ${numBlocks} blocks in the mock L2 block source`); + this.log.verbose( + `Created ${numCheckpoints} checkpoints with ${blocksPerCheckpoint} blocks each in the mock L2 block source`, + ); } public addProposedBlocks(blocks: L2Block[]) { @@ -50,6 +70,16 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { public removeBlocks(numBlocks: number) { this.l2Blocks = this.l2Blocks.slice(0, -numBlocks); + const maxBlockNum = this.l2Blocks.length; + // Remove any checkpoint whose last block is beyond the remaining blocks. + this.checkpointList = this.checkpointList.filter(c => { + const lastBlockNum = c.blocks[0].number + c.blocks.length - 1; + return lastBlockNum <= maxBlockNum; + }); + // Keep tip numbers consistent with remaining blocks. + this.checkpointedBlockNumber = Math.min(this.checkpointedBlockNumber, maxBlockNum); + this.provenBlockNumber = Math.min(this.provenBlockNumber, maxBlockNum); + this.finalizedBlockNumber = Math.min(this.finalizedBlockNumber, maxBlockNum); this.log.verbose(`Removed ${numBlocks} blocks from the mock L2 block source`); } @@ -65,7 +95,33 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } public setCheckpointedBlockNumber(checkpointedBlockNumber: number) { + const prevCheckpointed = this.checkpointedBlockNumber; this.checkpointedBlockNumber = checkpointedBlockNumber; + // Auto-create single-block checkpoints for newly checkpointed blocks that don't have one yet. + // This handles blocks added via addProposedBlocks that are now being marked as checkpointed. + const newCheckpoints: Checkpoint[] = []; + for (let blockNum = prevCheckpointed + 1; blockNum <= checkpointedBlockNumber; blockNum++) { + const block = this.l2Blocks[blockNum - 1]; + if (!block) { + continue; + } + if (this.checkpointList.some(c => c.blocks.some(b => b.number === block.number))) { + continue; + } + const checkpointNum = CheckpointNumber(this.checkpointList.length + newCheckpoints.length + 1); + const checkpoint = new Checkpoint( + block.archive, + CheckpointHeader.random({ slotNumber: block.header.globalVariables.slotNumber }), + [block], + checkpointNum, + ); + newCheckpoints.push(checkpoint); + } + // Insert new checkpoints in order by number. + if (newCheckpoints.length > 0) { + this.checkpointList.push(...newCheckpoints); + this.checkpointList.sort((a, b) => a.number - b.number); + } } /** @@ -112,13 +168,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { if (!block) { return Promise.resolve(undefined); } - const checkpointedBlock = new CheckpointedL2Block( - CheckpointNumber.fromBlockNumber(number), - block, - new L1PublishedData(BigInt(number), BigInt(number), `0x${number.toString(16).padStart(64, '0')}`), - [], - ); - return Promise.resolve(checkpointedBlock); + return Promise.resolve(this.toCheckpointedBlock(block)); } public async getCheckpointedBlocks(from: BlockNumber, limit: number): Promise { @@ -167,44 +217,22 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } public getCheckpoints(from: CheckpointNumber, limit: number) { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const blocks = this.l2Blocks.slice(from - 1, from - 1 + limit); - return Promise.all( - blocks.map(async block => { - // Create a checkpoint from the block - manually construct since L2Block doesn't have toCheckpoint() - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return new PublishedCheckpoint( - checkpoint, - new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - [], - ); - }), + const checkpoints = this.checkpointList.slice(from - 1, from - 1 + limit); + return Promise.resolve( + checkpoints.map(checkpoint => new PublishedCheckpoint(checkpoint, this.mockL1DataForCheckpoint(checkpoint), [])), ); } - public async getCheckpointByArchive(archive: Fr): Promise { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const block = this.l2Blocks.find(b => b.archive.root.equals(archive)); - if (!block) { - return undefined; - } - // Create a checkpoint from the block - manually construct since L2Block doesn't have toCheckpoint() - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return checkpoint; + public getCheckpointByArchive(archive: Fr): Promise { + const checkpoint = this.checkpointList.find(c => c.archive.root.equals(archive)); + return Promise.resolve(checkpoint); } public async getCheckpointedBlockByHash(blockHash: BlockHash): Promise { for (const block of this.l2Blocks) { const hash = await block.hash(); if (hash.equals(blockHash)) { - return CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - attestations: [], - }); + return this.toCheckpointedBlock(block); } } return undefined; @@ -215,14 +243,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { if (!block) { return Promise.resolve(undefined); } - return Promise.resolve( - CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), - attestations: [], - }), - ); + return Promise.resolve(this.toCheckpointedBlock(block)); } public async getL2BlockByHash(blockHash: BlockHash): Promise { @@ -255,47 +276,69 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { return Promise.resolve(block?.header); } + public async getBlockData(number: BlockNumber): Promise { + const block = this.l2Blocks[number - 1]; + if (!block) { + return undefined; + } + return { + header: block.header, + archive: block.archive, + blockHash: await block.hash(), + checkpointNumber: block.checkpointNumber, + indexWithinCheckpoint: block.indexWithinCheckpoint, + }; + } + + public async getBlockDataByArchive(archive: Fr): Promise { + const block = this.l2Blocks.find(b => b.archive.root.equals(archive)); + if (!block) { + return undefined; + } + return { + header: block.header, + archive: block.archive, + blockHash: await block.hash(), + checkpointNumber: block.checkpointNumber, + indexWithinCheckpoint: block.indexWithinCheckpoint, + }; + } + getBlockHeader(number: number | 'latest'): Promise { return Promise.resolve(this.l2Blocks.at(typeof number === 'number' ? number - 1 : -1)?.header); } getCheckpointsForEpoch(epochNumber: EpochNumber): Promise { - // TODO(mbps): Implement this properly. This only works when we have one block per checkpoint. - const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; - const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); - const blocks = this.l2Blocks.filter(b => { - const slot = b.header.globalVariables.slotNumber; - return slot >= start && slot <= end; - }); - // Create checkpoints from blocks - manually construct since L2Block doesn't have toCheckpoint() - return Promise.all( - blocks.map(async block => { - const checkpoint = await Checkpoint.random(block.checkpointNumber, { numBlocks: 1 }); - checkpoint.blocks = [block]; - return checkpoint; - }), - ); + return Promise.resolve(this.getCheckpointsInEpoch(epochNumber)); } - getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { - const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; - const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); - const blocks = this.l2Blocks.filter(b => { - const slot = b.header.globalVariables.slotNumber; - return slot >= start && slot <= end; - }); + getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { + const checkpoints = this.getCheckpointsInEpoch(epochNumber); return Promise.resolve( - blocks.map(block => - CheckpointedL2Block.fromFields({ - checkpointNumber: CheckpointNumber.fromBlockNumber(block.number), - block, - l1: new L1PublishedData(BigInt(block.number), BigInt(block.number), Buffer32.random().toString()), + checkpoints.map( + (checkpoint): CheckpointData => ({ + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: computeCheckpointOutHash( + checkpoint.blocks.map(b => b.body.txEffects.map(tx => tx.l2ToL1Msgs)), + ), + startBlock: checkpoint.blocks[0].number, + blockCount: checkpoint.blocks.length, attestations: [], + l1: this.mockL1DataForCheckpoint(checkpoint), }), ), ); } + getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { + const checkpoints = this.getCheckpointsInEpoch(epochNumber); + return Promise.resolve( + checkpoints.flatMap(checkpoint => checkpoint.blocks.map(block => this.toCheckpointedBlock(block))), + ); + } + getBlocksForSlot(slotNumber: SlotNumber): Promise { const blocks = this.l2Blocks.filter(b => b.header.globalVariables.slotNumber === slotNumber); return Promise.resolve(blocks); @@ -384,7 +427,10 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { const makeTipId = (blockId: typeof latestBlockId) => ({ block: blockId, - checkpoint: { number: CheckpointNumber.fromBlockNumber(blockId.number), hash: blockId.hash }, + checkpoint: { + number: this.findCheckpointNumberForBlock(blockId.number) ?? CheckpointNumber(0), + hash: blockId.hash, + }, }); return { @@ -472,4 +518,38 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { getPendingChainValidationStatus(): Promise { return Promise.resolve({ valid: true }); } + + /** Returns checkpoints whose slot falls within the given epoch. */ + private getCheckpointsInEpoch(epochNumber: EpochNumber): Checkpoint[] { + const epochDuration = DefaultL1ContractsConfig.aztecEpochDuration; + const [start, end] = getSlotRangeForEpoch(epochNumber, { epochDuration }); + return this.checkpointList.filter(c => c.header.slotNumber >= start && c.header.slotNumber <= end); + } + + /** Creates a mock L1PublishedData for a checkpoint. */ + private mockL1DataForCheckpoint(checkpoint: Checkpoint): L1PublishedData { + return new L1PublishedData(BigInt(checkpoint.number), BigInt(checkpoint.number), Buffer32.random().toString()); + } + + /** Creates a CheckpointedL2Block from a block using stored checkpoint info. */ + private toCheckpointedBlock(block: L2Block): CheckpointedL2Block { + const checkpoint = this.checkpointList.find(c => c.blocks.some(b => b.number === block.number)); + const checkpointNumber = checkpoint?.number ?? block.checkpointNumber; + return new CheckpointedL2Block( + checkpointNumber, + block, + new L1PublishedData( + BigInt(block.number), + BigInt(block.number), + `0x${block.number.toString(16).padStart(64, '0')}`, + ), + [], + ); + } + + /** Finds the checkpoint number for a block, or undefined if the block is not in any checkpoint. */ + private findCheckpointNumberForBlock(blockNumber: BlockNumber): CheckpointNumber | undefined { + const checkpoint = this.checkpointList.find(c => c.blocks.some(b => b.number === blockNumber)); + return checkpoint?.number; + } } diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index 646153664c8c..be268d1753df 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -68,6 +68,7 @@ "@aztec/archiver": "workspace:^", "@aztec/bb-prover": "workspace:^", "@aztec/blob-client": "workspace:^", + "@aztec/blob-lib": "workspace:^", "@aztec/constants": "workspace:^", "@aztec/epoch-cache": "workspace:^", "@aztec/ethereum": "workspace:^", @@ -81,6 +82,7 @@ "@aztec/p2p": "workspace:^", "@aztec/protocol-contracts": "workspace:^", "@aztec/prover-client": "workspace:^", + "@aztec/prover-node": "workspace:^", "@aztec/sequencer-client": "workspace:^", "@aztec/simulator": "workspace:^", "@aztec/slasher": "workspace:^", diff --git a/yarn-project/aztec-node/src/aztec-node/config.test.ts b/yarn-project/aztec-node/src/aztec-node/config.test.ts index 29de6785796a..0cbd120fba45 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.test.ts @@ -1,7 +1,7 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import type { EthPrivateKey } from '@aztec/node-keystore'; import type { SharedNodeConfig } from '@aztec/node-lib/config'; -import type { SequencerClientConfig, TxSenderConfig } from '@aztec/sequencer-client/config'; +import type { SequencerClientConfig, SequencerTxSenderConfig } from '@aztec/sequencer-client/config'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { ValidatorClientConfig } from '@aztec/validator-client/config'; @@ -33,7 +33,7 @@ describe('createKeyStoreForValidator', () => { web3SignerUrl?: string, validatorAddresses: EthAddress[] = [], publisherAddresses: EthAddress[] = [], - ): TxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig => { + ): SequencerTxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig => { const mockValidatorPrivateKeys = validatorKeys.length > 0 ? { @@ -46,14 +46,14 @@ describe('createKeyStoreForValidator', () => { return { validatorPrivateKeys: mockValidatorPrivateKeys, - publisherPrivateKeys: mockPublisherPrivateKeys, + sequencerPublisherPrivateKeys: mockPublisherPrivateKeys, coinbase: coinbase, feeRecipient: feeRecipient, web3SignerUrl, validatorAddresses: validatorAddresses.map(addr => addr), - publisherAddresses: publisherAddresses.map(addr => addr), + sequencerPublisherAddresses: publisherAddresses.map(addr => addr), l1Contracts: { rollupAddress: EthAddress.random() }, - } as TxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig; + } as SequencerTxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig; }; beforeAll(async () => { @@ -69,11 +69,11 @@ describe('createKeyStoreForValidator', () => { it('should return undefined when validatorPrivateKeys is undefined', () => { const config = { validatorPrivateKeys: undefined, - publisherPrivateKeys: undefined, + sequencerPublisherPrivateKeys: undefined, coinbase: undefined, feeRecipient: undefined, l1Contracts: { rollupAddress: EthAddress.random() }, - } as unknown as TxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig; + } as unknown as SequencerTxSenderConfig & ValidatorClientConfig & SequencerClientConfig & SharedNodeConfig; const result = createKeyStoreForValidator(config); expect(result).toBeUndefined(); }); diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 6b89c57a52ac..27d433cc6db4 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -13,9 +13,14 @@ import { import { type SharedNodeConfig, sharedNodeConfigMappings } from '@aztec/node-lib/config'; import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p/config'; import { type ProverClientUserConfig, proverClientConfigMappings } from '@aztec/prover-client/config'; +import { + type ProverNodeConfig, + proverNodeConfigMappings, + specificProverNodeConfigMappings, +} from '@aztec/prover-node/config'; import { type SequencerClientConfig, - type TxSenderConfig, + type SequencerTxSenderConfig, sequencerClientConfigMappings, } from '@aztec/sequencer-client/config'; import { slasherConfigMappings } from '@aztec/slasher'; @@ -46,16 +51,18 @@ export type AztecNodeConfig = ArchiverConfig & SharedNodeConfig & GenesisStateConfig & NodeRPCConfig & - SlasherConfig & { + SlasherConfig & + ProverNodeConfig & { /** L1 contracts addresses */ l1Contracts: L1ContractAddresses; /** Whether the validator is disabled for this node */ disableValidator: boolean; /** Whether to skip waiting for the archiver to be fully synced before starting other services */ skipArchiverInitialSync: boolean; - /** A flag to force verification of tx Chonk proofs. Only used for testnet */ debugForceTxProofVerification: boolean; + /** Whether to enable the prover node as a subsystem. */ + enableProverNode: boolean; }; export const aztecNodeConfigMappings: ConfigMappingsType = { @@ -63,6 +70,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType = { ...keyStoreConfigMappings, ...archiverConfigMappings, ...sequencerClientConfigMappings, + ...proverNodeConfigMappings, ...validatorClientConfigMappings, ...proverClientConfigMappings, ...worldStateConfigMappings, @@ -72,6 +80,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType = { ...genesisStateConfigMappings, ...nodeRpcConfigMappings, ...slasherConfigMappings, + ...specificProverNodeConfigMappings, l1Contracts: { description: 'The deployed L1 contract addresses', nested: l1ContractAddressesMapping, @@ -91,6 +100,11 @@ export const aztecNodeConfigMappings: ConfigMappingsType = { description: 'Whether to skip waiting for the archiver to be fully synced before starting other services.', ...booleanConfigHelper(false), }, + enableProverNode: { + env: 'ENABLE_PROVER_NODE', + description: 'Whether to enable the prover node as a subsystem.', + ...booleanConfigHelper(false), + }, }; /** @@ -101,7 +115,7 @@ export function getConfigEnvVars(): AztecNodeConfig { return getConfigFromMappings(aztecNodeConfigMappings); } -type ConfigRequiredToBuildKeyStore = TxSenderConfig & SequencerClientConfig & SharedNodeConfig & ValidatorClientConfig; +type ConfigRequiredToBuildKeyStore = SequencerClientConfig & SharedNodeConfig & ValidatorClientConfig; function createKeyStoreFromWeb3Signer(config: ConfigRequiredToBuildKeyStore): KeyStore | undefined { const validatorKeyStores: ValidatorKeyStore[] = []; @@ -120,7 +134,7 @@ function createKeyStoreFromWeb3Signer(config: ConfigRequiredToBuildKeyStore): Ke feeRecipient: config.feeRecipient ?? AztecAddress.ZERO, coinbase: config.coinbase ?? config.validatorAddresses[0], remoteSigner: config.web3SignerUrl, - publisher: config.publisherAddresses ?? [], + publisher: config.sequencerPublisherAddresses ?? [], }); const keyStore: KeyStore = { @@ -145,8 +159,10 @@ function createKeyStoreFromPrivateKeys(config: ConfigRequiredToBuildKeyStore): K const coinbase = config.coinbase ?? EthAddress.fromString(privateKeyToAddress(ethPrivateKeys[0])); const feeRecipient = config.feeRecipient ?? AztecAddress.ZERO; - const publisherKeys = config.publisherPrivateKeys - ? config.publisherPrivateKeys.map((k: { getValue: () => string }) => ethPrivateKeySchema.parse(k.getValue())) + const publisherKeys = config.sequencerPublisherPrivateKeys + ? config.sequencerPublisherPrivateKeys.map((k: { getValue: () => string }) => + ethPrivateKeySchema.parse(k.getValue()), + ) : []; validatorKeyStores.push({ @@ -168,7 +184,7 @@ function createKeyStoreFromPrivateKeys(config: ConfigRequiredToBuildKeyStore): K } export function createKeyStoreForValidator( - config: TxSenderConfig & SequencerClientConfig & SharedNodeConfig, + config: SequencerTxSenderConfig & SequencerClientConfig & SharedNodeConfig, ): KeyStore | undefined { if (config.web3SignerUrl !== undefined && config.web3SignerUrl.length > 0) { return createKeyStoreFromWeb3Signer(config); diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index bb88a0873375..17d6f3f51928 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -4,13 +4,17 @@ import type { RollupContract } from '@aztec/ethereum/contracts'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { BadRequestError } from '@aztec/foundation/json-rpc'; +import type { Hex } from '@aztec/foundation/string'; import { DateProvider } from '@aztec/foundation/timer'; import { unfreeze } from '@aztec/foundation/types'; +import { type KeyStore, KeystoreManager, RemoteSigner, type ValidatorKeyStore } from '@aztec/node-keystore'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import type { P2P } from '@aztec/p2p'; import { protocolContractsHash } from '@aztec/protocol-contracts'; import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice'; -import type { GlobalVariableBuilder } from '@aztec/sequencer-client'; +import type { GlobalVariableBuilder, SequencerClient } from '@aztec/sequencer-client'; +import type { SlasherClientInterface } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { L2Block, type L2BlockSource } from '@aztec/stdlib/block'; import type { ContractDataSource } from '@aztec/stdlib/contract'; @@ -33,11 +37,15 @@ import { Tx, } from '@aztec/stdlib/tx'; import { getPackageVersion } from '@aztec/stdlib/update-checker'; +import type { ValidatorClient } from '@aztec/validator-client'; -import { readFileSync } from 'fs'; +import { jest } from '@jest/globals'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; import { type MockProxy, mock } from 'jest-mock-extended'; -import { dirname, resolve } from 'path'; +import { tmpdir } from 'os'; +import { dirname, join, resolve } from 'path'; import { fileURLToPath } from 'url'; +import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts'; import { type AztecNodeConfig, getConfigEnvVars } from './config.js'; import { AztecNodeService } from './server.js'; @@ -173,6 +181,7 @@ describe('aztec node', () => { undefined, undefined, undefined, + undefined, 12345, rollupVersion.toNumber(), globalVariablesBuilder, @@ -390,4 +399,284 @@ describe('aztec node', () => { await expect(node.simulatePublicCalls(tx)).rejects.toThrow(/gas/i); }); }); + + describe('reloadKeystore', () => { + it('throws BadRequestError if no file-based keystore directory is configured', async () => { + // Default node has no keyStoreDirectory set + await expect(node.reloadKeystore()).rejects.toThrow(BadRequestError); + }); + + it('throws BadRequestError if keystore directory is set but validator client is not configured', async () => { + // Satisfies the first check (directory exists) but validatorClient is undefined + nodeConfig.keyStoreDirectory = '/tmp/fake-keystore-dir'; + await expect(node.reloadKeystore()).rejects.toThrow(BadRequestError); + }); + + describe('with file-based keystore', () => { + let keyStoreDir: string; + let validatorClient: MockProxy; + let slasherClient: MockProxy; + let validatorPrivateKey: string; + let nodeWithValidator: AztecNodeService; + + // Helper to build a KeyStore with default coinbase/feeRecipient/remoteSigner. + // Each entry needs only `attester` (required) and optionally `publisher`. + const makeKeyStore = ( + ...validators: Array & Pick, 'publisher'>> + ): KeyStore => ({ + schemaVersion: 1, + validators: validators.map(v => ({ + attester: v.attester, + coinbase: undefined, + feeRecipient: AztecAddress.ZERO, + remoteSigner: undefined, + ...(v.publisher !== undefined ? { publisher: v.publisher } : {}), + })), + }); + + beforeEach(() => { + // Create a temp directory with a keystore file + keyStoreDir = mkdtempSync(join(tmpdir(), 'keystore-test-')); + validatorPrivateKey = generatePrivateKey(); + const keyStore = makeKeyStore({ attester: [validatorPrivateKey as Hex<32>] }); + writeFileSync(join(keyStoreDir, 'keystore.json'), JSON.stringify(keyStore)); + + validatorClient = mock(); + slasherClient = mock(); + + const validatorNodeConfig = { ...nodeConfig, keyStoreDirectory: keyStoreDir }; + + nodeWithValidator = new AztecNodeService( + validatorNodeConfig, + p2p, + l2BlockSource, + mock(), + mock(), + mock(), + mock({ getCommitted: () => merkleTreeOps }), + undefined, + undefined, + slasherClient, + undefined, + undefined, + 12345, + rollupVersion.toNumber(), + globalVariablesBuilder, + epochCache, + getPackageVersion() ?? '', + new TestCircuitVerifier(), + undefined, + undefined, + undefined, + validatorClient as unknown as ValidatorClient, + new KeystoreManager(keyStore), + ); + }); + + afterEach(() => { + rmSync(keyStoreDir, { recursive: true, force: true }); + }); + + it('reloads keystore from disk and calls validatorClient.reloadKeystore', async () => { + await nodeWithValidator.reloadKeystore(); + expect(validatorClient.reloadKeystore).toHaveBeenCalledTimes(1); + }); + + it('adds new validators to slasher dont-slash-self list on reload', async () => { + // Write a new keystore file with an additional validator + const newPrivateKey = generatePrivateKey(); + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify(makeKeyStore({ attester: [validatorPrivateKey as Hex<32>, newPrivateKey as Hex<32>] })), + ); + + await nodeWithValidator.reloadKeystore(); + + const updateArg = slasherClient.updateConfig.mock.calls[0][0]; + const neverSlashList = updateArg.slashValidatorsNever!; + + const originalAddress = EthAddress.fromString( + privateKeyToAccount(validatorPrivateKey as `0x${string}`).address, + ); + const newAddress = EthAddress.fromString(privateKeyToAccount(newPrivateKey as `0x${string}`).address); + + expect(neverSlashList.some(a => a.equals(originalAddress))).toBe(true); + expect(neverSlashList.some(a => a.equals(newAddress))).toBe(true); + }); + + it('removes validators from slasher dont-slash-self list when removed from keystore', async () => { + // First add two validators + const secondPrivateKey = generatePrivateKey(); + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify(makeKeyStore({ attester: [validatorPrivateKey as Hex<32>, secondPrivateKey as Hex<32>] })), + ); + await nodeWithValidator.reloadKeystore(); + + // Now remove the second validator, keeping only the original + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify(makeKeyStore({ attester: [validatorPrivateKey as Hex<32>] })), + ); + await nodeWithValidator.reloadKeystore(); + + // The second call to updateConfig should only contain the remaining validator + const updateArg = slasherClient.updateConfig.mock.calls[1][0]; + const neverSlashList = updateArg.slashValidatorsNever!; + + const originalAddress = EthAddress.fromString( + privateKeyToAccount(validatorPrivateKey as `0x${string}`).address, + ); + const removedAddress = EthAddress.fromString(privateKeyToAccount(secondPrivateKey as `0x${string}`).address); + + expect(neverSlashList.some(a => a.equals(originalAddress))).toBe(true); + expect(neverSlashList.some(a => a.equals(removedAddress))).toBe(false); + }); + + it('does not update slasher if slashSelfAllowed is true', async () => { + (nodeWithValidator as any).config.slashSelfAllowed = true; + await nodeWithValidator.reloadKeystore(); + + expect(validatorClient.reloadKeystore).toHaveBeenCalledTimes(1); + expect(slasherClient.updateConfig).not.toHaveBeenCalled(); + }); + + it('reloads keystore with remote signer validators from disk', async () => { + // Update keystore file to add a remote signer validator alongside the local key validator. + // This verifies the full reload path supports mixed local + remote signer keystores: + // file-on-disk -> loadKeystores -> KeystoreManager -> validateSigners (mocked) -> + // ValidatorClient.reloadKeystore -> NodeKeystoreAdapter (creates RemoteSigner instances) + const remoteSignerUrl = 'https://web3signer.example.com:9000'; + const remoteAttesterAddress = EthAddress.random(); + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify( + makeKeyStore( + { attester: [validatorPrivateKey as Hex<32>] }, + { attester: { address: remoteAttesterAddress, remoteSignerUrl } }, + ), + ), + ); + + // Mock RemoteSigner.validateAccess to avoid a real HTTP call to web3signer. + // validateSigners() calls this to verify each remote signer URL is reachable + // and that the requested addresses are available. + const validateSpy = jest.spyOn(RemoteSigner, 'validateAccess').mockImplementation(() => Promise.resolve()); + + try { + await nodeWithValidator.reloadKeystore(); + + // Verify RemoteSigner.validateAccess was called with the correct URL and address + expect(validateSpy).toHaveBeenCalledTimes(1); + expect(validateSpy).toHaveBeenCalledWith( + remoteSignerUrl, + expect.arrayContaining([remoteAttesterAddress.toString().toLowerCase()]), + ); + + // Verify validatorClient.reloadKeystore was called (reload succeeded) + expect(validatorClient.reloadKeystore).toHaveBeenCalledTimes(1); + + // Verify the new KeystoreManager was passed through with both validators + const passedManager = validatorClient.reloadKeystore.mock.calls[0][0] as KeystoreManager; + expect(passedManager.getValidatorCount()).toBe(2); + + // Verify slasher list includes both the local and remote validator addresses + const updateArg = slasherClient.updateConfig.mock.calls[0][0]; + const neverSlashList = updateArg.slashValidatorsNever!; + expect(neverSlashList.some(a => a.equals(remoteAttesterAddress))).toBe(true); + } finally { + validateSpy.mockRestore(); + } + }); + + it('rejects reload when remote signer validation fails', async () => { + // If RemoteSigner.validateAccess fails (e.g. web3signer unreachable or address not found), + // the reload should be rejected and the old keystore should remain intact. + const remoteSignerUrl = 'https://web3signer.example.com:9000'; + const remoteAttesterAddress = EthAddress.random(); + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify( + makeKeyStore( + { attester: [validatorPrivateKey as Hex<32>] }, + // EthAddress has toJSON() so JSON.stringify serializes it as a hex string. + { attester: { address: remoteAttesterAddress, remoteSignerUrl } }, + ), + ), + ); + + // Mock RemoteSigner.validateAccess to reject — simulates unreachable web3signer + const validateSpy = jest + .spyOn(RemoteSigner, 'validateAccess') + .mockRejectedValue(new Error('Unable to connect to web3signer')); + + try { + await expect(nodeWithValidator.reloadKeystore()).rejects.toThrow(/Unable to connect to web3signer/); + + // Validator client should NOT have been called (reload rejected before mutation) + expect(validatorClient.reloadKeystore).not.toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + } + }); + + it('rejects reload when new validator has a publisher key not in the L1 signers', async () => { + // Initial keystore has validator with publisherKeyA + const publisherKeyA = generatePrivateKey(); + const publisherKeyB = generatePrivateKey(); // different, not in L1 signers + + const initialKeyStore = makeKeyStore({ + attester: [validatorPrivateKey as Hex<32>], + publisher: [publisherKeyA as Hex<32>], + }); + + // Recreate node with a truthy sequencer so the publisher validation path runs. + // Only truthiness matters: the code checks `if (this.keyStoreManager && this.sequencer)` + // and the validation logic uses keyStoreManager, not sequencer methods. + // The test expects rejection before sequencer.updatePublisherNodeKeyStore() is reached. + const nodeWithSequencer = new AztecNodeService( + { ...nodeConfig, keyStoreDirectory: keyStoreDir }, + p2p, + l2BlockSource, + mock(), + mock(), + mock(), + mock({ getCommitted: () => merkleTreeOps }), + {} as SequencerClient, + undefined, + slasherClient, + undefined, + undefined, + 12345, + rollupVersion.toNumber(), + globalVariablesBuilder, + epochCache, + getPackageVersion() ?? '', + new TestCircuitVerifier(), + undefined, + undefined, + undefined, + validatorClient as unknown as ValidatorClient, + new KeystoreManager(initialKeyStore), + ); + + // Write new keystore: new validator uses publisherKeyB (not in the L1 signers) + const newValidatorKey = generatePrivateKey(); + writeFileSync( + join(keyStoreDir, 'keystore.json'), + JSON.stringify( + makeKeyStore( + { attester: [validatorPrivateKey as Hex<32>], publisher: [publisherKeyA as Hex<32>] }, + { attester: [newValidatorKey as Hex<32>], publisher: [publisherKeyB as Hex<32>] }, + ), + ), + ); + + await expect(nodeWithSequencer.reloadKeystore()).rejects.toThrow(BadRequestError); + + // reload rejected before mutation + expect(validatorClient.reloadKeystore).not.toHaveBeenCalled(); + }); + }); + }); }); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index ebc1acae4d93..0706015744ca 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -1,6 +1,7 @@ import { Archiver, createArchiver } from '@aztec/archiver'; import { BBCircuitVerifier, QueuedIVCVerifier, TestCircuitVerifier } from '@aztec/bb-prover'; import { type BlobClientInterface, createBlobClientWithFileStores } from '@aztec/blob-client/client'; +import { Blob } from '@aztec/blob-lib'; import { ARCHIVE_HEIGHT, type L1_TO_L2_MSG_TREE_HEIGHT, type NOTE_HASH_TREE_HEIGHT } from '@aztec/constants'; import { EpochCache, type EpochCacheInterface } from '@aztec/epoch-cache'; import { createEthereumChain } from '@aztec/ethereum/chain'; @@ -8,7 +9,7 @@ import { getPublicClient } from '@aztec/ethereum/client'; import { RegistryContract, RollupContract } from '@aztec/ethereum/contracts'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { compactArray, pick } from '@aztec/foundation/collection'; +import { compactArray, pick, unique } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { BadRequestError } from '@aztec/foundation/json-rpc'; @@ -16,14 +17,13 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; import { count } from '@aztec/foundation/string'; import { DateProvider, Timer } from '@aztec/foundation/timer'; import { MembershipWitness, SiblingPath } from '@aztec/foundation/trees'; -import { KeystoreManager, loadKeystores, mergeKeystores } from '@aztec/node-keystore'; +import { type KeyStore, KeystoreManager, loadKeystores, mergeKeystores } from '@aztec/node-keystore'; import { trySnapshotSync, uploadSnapshot } from '@aztec/node-lib/actions'; -import { - createForwarderL1TxUtilsFromEthSigner, - createL1TxUtilsWithBlobsFromEthSigner, -} from '@aztec/node-lib/factories'; +import { createForwarderL1TxUtilsFromSigners, createL1TxUtilsFromSigners } from '@aztec/node-lib/factories'; import { type P2P, type P2PClientDeps, createP2PClient, getDefaultAllowedSetupFunctions } from '@aztec/p2p'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; +import { type ProverNode, type ProverNodeDeps, createProverNode } from '@aztec/prover-node'; +import { createKeyStoreForProver } from '@aztec/prover-node/config'; import { GlobalVariableBuilder, SequencerClient, type SequencerPublisher } from '@aztec/sequencer-client'; import { PublicProcessorFactory } from '@aztec/simulator/server'; import { @@ -35,7 +35,14 @@ import { } from '@aztec/slasher'; import { CollectionLimitsConfig, PublicSimulatorConfig } from '@aztec/stdlib/avm'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { BlockHash, type BlockParameter, type DataInBlock, L2Block, type L2BlockSource } from '@aztec/stdlib/block'; +import { + type BlockData, + BlockHash, + type BlockParameter, + type DataInBlock, + L2Block, + type L2BlockSource, +} from '@aztec/stdlib/block'; import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, @@ -129,6 +136,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { protected readonly l1ToL2MessageSource: L1ToL2MessageSource, protected readonly worldStateSynchronizer: WorldStateSynchronizer, protected readonly sequencer: SequencerClient | undefined, + protected readonly proverNode: ProverNode | undefined, protected readonly slasherClient: SlasherClientInterface | undefined, protected readonly validatorsSentinel: Sentinel | undefined, protected readonly epochPruneWatcher: EpochPruneWatcher | undefined, @@ -141,6 +149,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { private telemetry: TelemetryClient = getTelemetryClient(), private log = createLogger('node'), private blobClient?: BlobClientInterface, + private validatorClient?: ValidatorClient, + private keyStoreManager?: KeystoreManager, ) { this.metrics = new NodeMetrics(telemetry, 'AztecNodeService'); this.tracer = telemetry.getTracer('AztecNodeService'); @@ -171,10 +181,12 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { publisher?: SequencerPublisher; dateProvider?: DateProvider; p2pClientDeps?: P2PClientDeps; + proverNodeDeps?: Partial; } = {}, options: { prefilledPublicData?: PublicDataTreeLeaf[]; dontStartSequencer?: boolean; + dontStartProverNode?: boolean; } = {}, ): Promise { const config = { ...inputConfig }; // Copy the config so we dont mutate the input object @@ -184,16 +196,29 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { const dateProvider = deps.dateProvider ?? new DateProvider(); const ethereumChain = createEthereumChain(config.l1RpcUrls, config.l1ChainId); - // Build a key store from file if given or from environment otherwise + // Build a key store from file if given or from environment otherwise. + // We keep the raw KeyStore available so we can merge with prover keys if enableProverNode is set. let keyStoreManager: KeystoreManager | undefined; const keyStoreProvided = config.keyStoreDirectory !== undefined && config.keyStoreDirectory.length > 0; if (keyStoreProvided) { const keyStores = loadKeystores(config.keyStoreDirectory!); keyStoreManager = new KeystoreManager(mergeKeystores(keyStores)); } else { - const keyStore = createKeyStoreForValidator(config); - if (keyStore) { - keyStoreManager = new KeystoreManager(keyStore); + const rawKeyStores: KeyStore[] = []; + const validatorKeyStore = createKeyStoreForValidator(config); + if (validatorKeyStore) { + rawKeyStores.push(validatorKeyStore); + } + if (config.enableProverNode) { + const proverKeyStore = createKeyStoreForProver(config); + if (proverKeyStore) { + rawKeyStores.push(proverKeyStore); + } + } + if (rawKeyStores.length > 0) { + keyStoreManager = new KeystoreManager( + rawKeyStores.length === 1 ? rawKeyStores[0] : mergeKeystores(rawKeyStores), + ); } } @@ -204,10 +229,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { if (keyStoreManager === undefined) { throw new Error('Failed to create key store, a requirement for running a validator'); } - if (!keyStoreProvided) { - log.warn( - 'KEY STORE CREATED FROM ENVIRONMENT, IT IS RECOMMENDED TO USE A FILE-BASED KEY STORE IN PRODUCTION ENVIRONMENTS', - ); + if (!keyStoreProvided && process.env.NODE_ENV !== 'test') { + log.warn("Keystore created from env: it's recommended to use a file-based key store for production"); } ValidatorClient.validateKeyStoreConfiguration(keyStoreManager, log); } @@ -249,7 +272,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { ); } - const blobClient = await createBlobClientWithFileStores(config, createLogger('node:blob-client:client')); + const blobClient = await createBlobClientWithFileStores(config, log.createChild('blob-client')); // attempt snapshot sync if possible await trySnapshotSync(config, log); @@ -412,19 +435,19 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { ); await slasherClient.start(); - const l1TxUtils = config.publisherForwarderAddress - ? await createForwarderL1TxUtilsFromEthSigner( + const l1TxUtils = config.sequencerPublisherForwarderAddress + ? await createForwarderL1TxUtilsFromSigners( publicClient, keyStoreManager!.createAllValidatorPublisherSigners(), - config.publisherForwarderAddress, + config.sequencerPublisherForwarderAddress, { ...config, scope: 'sequencer' }, - { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider }, + { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() }, ) - : await createL1TxUtilsWithBlobsFromEthSigner( + : await createL1TxUtilsFromSigners( publicClient, keyStoreManager!.createAllValidatorPublisherSigners(), { ...config, scope: 'sequencer' }, - { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider }, + { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() }, ); // Create and start the sequencer client @@ -461,6 +484,29 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { log.warn(`Sequencer created but not started`); } + // Create prover node subsystem if enabled + let proverNode: ProverNode | undefined; + if (config.enableProverNode) { + proverNode = await createProverNode(config, { + ...deps.proverNodeDeps, + telemetry, + dateProvider, + archiver, + worldStateSynchronizer, + p2pClient, + epochCache, + blobClient, + keyStoreManager, + }); + + if (!options.dontStartProverNode) { + await proverNode.start(); + log.info(`Prover node subsystem started`); + } else { + log.info(`Prover node subsystem created but not started`); + } + } + const globalVariableBuilder = new GlobalVariableBuilder({ ...config, rollupVersion: BigInt(config.rollupVersion), @@ -468,7 +514,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { slotDuration: Number(slotDuration), }); - return new AztecNodeService( + const node = new AztecNodeService( config, p2pClient, archiver, @@ -477,6 +523,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { archiver, worldStateSynchronizer, sequencer, + proverNode, slasherClient, validatorsSentinel, epochPruneWatcher, @@ -489,7 +536,11 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { telemetry, log, blobClient, + validatorClient, + keyStoreManager, ); + + return node; } /** @@ -500,6 +551,11 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { return this.sequencer; } + /** Returns the prover node subsystem, if enabled. */ + public getProverNode(): ProverNode | undefined { + return this.proverNode; + } + public getBlockSource(): L2BlockSource { return this.blockSource; } @@ -803,6 +859,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { await tryStop(this.slasherClient); await tryStop(this.proofVerifier); await tryStop(this.sequencer); + await tryStop(this.proverNode); await tryStop(this.p2pClient); await tryStop(this.worldStateSynchronizer); await tryStop(this.blockSource); @@ -1106,6 +1163,14 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { return await this.blockSource.getBlockHeaderByArchive(archive); } + public getBlockData(number: BlockNumber): Promise { + return this.blockSource.getBlockData(number); + } + + public getBlockDataByArchive(archive: Fr): Promise { + return this.blockSource.getBlockDataByArchive(archive); + } + /** * Simulates the public part of a transaction with the current state. * @param tx - The transaction to simulate. @@ -1129,7 +1194,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { } const txHash = tx.getTxHash(); - const blockNumber = BlockNumber((await this.blockSource.getBlockNumber()) + 1); + const latestBlockNumber = await this.blockSource.getBlockNumber(); + const blockNumber = BlockNumber.add(latestBlockNumber, 1); // If sequencer is not initialized, we just set these values to zero for simulation. const coinbase = EthAddress.ZERO; @@ -1153,10 +1219,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { blockNumber, }); - // Ensure world state is synced to the latest block before forking. - // Without this, the fork may be behind the archiver, causing lookups - // (e.g. L1-to-L2 message existence checks) to fail against stale state. - await this.#syncWorldState(); + // Ensure world-state has caught up with the latest block we loaded from the archiver + await this.worldStateSynchronizer.syncImmediate(latestBlockNumber); const merkleTreeFork = await this.worldStateSynchronizer.fork(); try { const config = PublicSimulatorConfig.from({ @@ -1381,6 +1445,94 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { } } + public async reloadKeystore(): Promise { + if (!this.config.keyStoreDirectory?.length) { + throw new BadRequestError( + 'Cannot reload keystore: node is not using a file-based keystore. ' + + 'Set KEY_STORE_DIRECTORY to use file-based keystores.', + ); + } + if (!this.validatorClient) { + throw new BadRequestError('Cannot reload keystore: validator is not enabled.'); + } + + this.log.info('Reloading keystore from disk'); + + // Re-read and validate keystore files + const keyStores = loadKeystores(this.config.keyStoreDirectory); + const newManager = new KeystoreManager(mergeKeystores(keyStores)); + await newManager.validateSigners(); + ValidatorClient.validateKeyStoreConfiguration(newManager, this.log); + + // Validate that every validator's publisher keys overlap with the L1 signers + // that were initialized at startup. Publishers cannot be hot-reloaded, so a + // validator with a publisher key that doesn't match any existing L1 signer + // would silently fail on every proposer slot. + if (this.keyStoreManager && this.sequencer) { + const oldAdapter = NodeKeystoreAdapter.fromKeyStoreManager(this.keyStoreManager); + const availablePublishers = new Set( + oldAdapter + .getAttesterAddresses() + .flatMap(a => oldAdapter.getPublisherAddresses(a).map(p => p.toString().toLowerCase())), + ); + + const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager); + for (const attester of newAdapter.getAttesterAddresses()) { + const pubs = newAdapter.getPublisherAddresses(attester); + if (pubs.length > 0 && !pubs.some(p => availablePublishers.has(p.toString().toLowerCase()))) { + throw new BadRequestError( + `Cannot reload keystore: validator ${attester} has publisher keys ` + + `[${pubs.map(p => p.toString()).join(', ')}] but none match the L1 signers initialized at startup ` + + `[${[...availablePublishers].join(', ')}]. Publishers cannot be hot-reloaded — ` + + `use an existing publisher key or restart the node.`, + ); + } + } + } + + // Build adapters for old and new keystores to compute diff + const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager); + const newAddresses = newAdapter.getAttesterAddresses(); + const oldAddresses = this.keyStoreManager + ? NodeKeystoreAdapter.fromKeyStoreManager(this.keyStoreManager).getAttesterAddresses() + : []; + + const oldSet = new Set(oldAddresses.map(a => a.toString())); + const newSet = new Set(newAddresses.map(a => a.toString())); + const added = newAddresses.filter(a => !oldSet.has(a.toString())); + const removed = oldAddresses.filter(a => !newSet.has(a.toString())); + + if (added.length > 0) { + this.log.info(`Keystore reload: adding attester keys: ${added.map(a => a.toString()).join(', ')}`); + } + if (removed.length > 0) { + this.log.info(`Keystore reload: removing attester keys: ${removed.map(a => a.toString()).join(', ')}`); + } + if (added.length === 0 && removed.length === 0) { + this.log.info('Keystore reload: attester keys unchanged'); + } + + // Update the validator client (coinbase, feeRecipient, attester keys) + this.validatorClient.reloadKeystore(newManager); + + // Update the publisher factory's keystore so newly-added validators + // can be matched to existing publisher keys when proposing blocks. + if (this.sequencer) { + this.sequencer.updatePublisherNodeKeyStore(newAdapter); + } + + // Update slasher's "don't-slash-self" list with new validator addresses + if (this.slasherClient && !this.config.slashSelfAllowed) { + const slashValidatorsNever = unique( + [...(this.config.slashValidatorsNever ?? []), ...newAddresses].map(a => a.toString()), + ).map(EthAddress.fromString); + this.slasherClient.updateConfig({ slashValidatorsNever }); + } + + this.keyStoreManager = newManager; + this.log.info('Keystore reloaded: coinbase, feeRecipient, and attester keys updated'); + } + #getInitialHeaderHash(): Promise { if (!this.initialHeaderHashPromise) { this.initialHeaderHashPromise = this.worldStateSynchronizer.getCommitted().getInitialHeader().hash(); diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts index 23bd50f032cd..b57b619be97d 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts @@ -589,7 +589,7 @@ describe('sentinel', () => { ts, nowMs: ts * 1000n, }); - archiver.getL2Block.calledWith(blockNumber).mockResolvedValue(mockBlock); + archiver.getBlockHeader.calledWith(blockNumber).mockResolvedValue(mockBlock.header); archiver.getL1Constants.mockResolvedValue(l1Constants); epochCache.getL1Constants.mockReturnValue(l1Constants); @@ -657,6 +657,81 @@ describe('sentinel', () => { }); }); + describe('escape hatch', () => { + it('processSlot skips tracking when escape hatch is open', async () => { + const validator1 = EthAddress.random(); + const validator2 = EthAddress.random(); + const committee = [validator1, validator2]; + + epochCache.getCommittee.mockResolvedValue({ + committee, + seed: 0n, + epoch, + isEscapeHatchOpen: true, + }); + + const updateSpy = jest.spyOn(store, 'updateValidators'); + + await sentinel.doProcessSlot(slot); + + // Should NOT have called updateValidators since escape hatch is open + expect(updateSpy).not.toHaveBeenCalled(); + // But lastProcessedSlot should still advance + expect(sentinel.getLastProcessedSlot()).toEqual(slot); + }); + + it('processSlot tracks normally when escape hatch is closed', async () => { + const signers = times(4, Secp256k1Signer.random); + const validators = signers.map(s => s.address); + const committee = [...validators]; + + epochCache.getCommittee.mockResolvedValue({ + committee, + seed: 0n, + epoch, + isEscapeHatchOpen: false, + }); + epochCache.computeProposerIndex.mockReturnValue(0n); + p2p.getCheckpointAttestationsForSlot.mockResolvedValue([]); + + const updateSpy = jest.spyOn(store, 'updateValidators'); + + await sentinel.doProcessSlot(slot); + + // Should have called updateValidators since escape hatch is closed + expect(updateSpy).toHaveBeenCalled(); + expect(sentinel.getLastProcessedSlot()).toEqual(slot); + }); + + it('handleChainProven skips proven performance when escape hatch is open', async () => { + const blockNumber = BlockNumber(15); + const blockHash = '0xblockhash'; + const mockBlock = await L2Block.random(blockNumber); + const blockSlot = mockBlock.header.getSlot(); + const epochNumber = getEpochAtSlot(blockSlot, l1Constants); + const validator1 = EthAddress.random(); + + archiver.getBlockHeader.calledWith(blockNumber).mockResolvedValue(mockBlock.header); + + epochCache.getCommittee.mockResolvedValue({ + committee: [validator1], + seed: 0n, + epoch: epochNumber, + isEscapeHatchOpen: true, + }); + + const emitSpy = jest.spyOn(sentinel, 'emit'); + const updateProvenSpy = jest.spyOn(store, 'updateProvenPerformance'); + + await sentinel.handleChainProven({ type: 'chain-proven', block: { number: blockNumber, hash: blockHash } }); + + // Should have stored empty performance (no offenses during escape hatch) + expect(updateProvenSpy).toHaveBeenCalledWith(epochNumber, {}); + // Should NOT have emitted any slash events + expect(emitSpy).not.toHaveBeenCalled(); + }); + }); + describe('consecutive epoch inactivity', () => { let validator1: EthAddress; let validator2: EthAddress; @@ -905,4 +980,8 @@ class TestSentinel extends Sentinel { ) { return super.checkPastInactivity(validator, currentEpoch, requiredConsecutiveEpochs); } + + public doProcessSlot(slot: SlotNumber) { + return super.processSlot(slot); + } } diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.ts b/yarn-project/aztec-node/src/sentinel/sentinel.ts index 44e4dc80d022..23f4cb21a613 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.ts @@ -139,15 +139,15 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme return; } const blockNumber = event.block.number; - const block = await this.archiver.getL2Block(blockNumber); - if (!block) { - this.logger.error(`Failed to get block ${blockNumber}`, { block }); + const header = await this.archiver.getBlockHeader(blockNumber); + if (!header) { + this.logger.error(`Failed to get block header ${blockNumber}`); return; } // TODO(palla/slash): We should only be computing proven performance if this is // a full proof epoch and not a partial one, otherwise we'll end up with skewed stats. - const epoch = getEpochAtSlot(block.header.getSlot(), this.epochCache.getL1Constants()); + const epoch = getEpochAtSlot(header.getSlot(), this.epochCache.getL1Constants()); this.logger.debug(`Computing proven performance for epoch ${epoch}`); const performance = await this.computeProvenPerformance(epoch); this.logger.info(`Computed proven performance for epoch ${epoch}`, performance); @@ -158,7 +158,11 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme protected async computeProvenPerformance(epoch: EpochNumber): Promise { const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, this.epochCache.getL1Constants()); - const { committee } = await this.epochCache.getCommittee(fromSlot); + const { committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(fromSlot); + if (isEscapeHatchOpen) { + this.logger.info(`Skipping proven performance for epoch ${epoch} - escape hatch is open`); + return {}; + } if (!committee) { this.logger.trace(`No committee found for slot ${fromSlot}`); return {}; @@ -327,7 +331,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme * and updates overall stats. */ protected async processSlot(slot: SlotNumber) { - const { epoch, seed, committee } = await this.epochCache.getCommittee(slot); + const { epoch, seed, committee, isEscapeHatchOpen } = await this.epochCache.getCommittee(slot); + if (isEscapeHatchOpen) { + this.logger.info(`Skipping slot ${slot} at epoch ${epoch} - escape hatch is open`); + this.lastProcessedSlot = slot; + return; + } if (!committee || committee.length === 0) { this.logger.trace(`No committee found for slot ${slot} at epoch ${epoch}`); this.lastProcessedSlot = slot; diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index 90a91bf65be1..272c00696903 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../blob-client" }, + { + "path": "../blob-lib" + }, { "path": "../constants" }, @@ -54,6 +57,9 @@ { "path": "../prover-client" }, + { + "path": "../prover-node" + }, { "path": "../sequencer-client" }, diff --git a/yarn-project/aztec/src/cli/admin_api_key_store.test.ts b/yarn-project/aztec/src/cli/admin_api_key_store.test.ts new file mode 100644 index 000000000000..c91e05c44865 --- /dev/null +++ b/yarn-project/aztec/src/cli/admin_api_key_store.test.ts @@ -0,0 +1,170 @@ +import { sha256Hash } from '@aztec/foundation/json-rpc/server'; +import { createLogger } from '@aztec/foundation/log'; + +import { promises as fs } from 'fs'; +import { mkdtemp, rm } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { type ResolveAdminApiKeyOptions, resolveAdminApiKey } from './admin_api_key_store.js'; + +describe('resolveAdminApiKey', () => { + const log = createLogger('test:admin-api-key'); + let tempDir: string | undefined; + + beforeEach(() => { + tempDir = undefined; + }); + + afterEach(async () => { + if (tempDir) { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + describe('opt-out (noAdminApiKey = true)', () => { + it('returns undefined when auth is disabled', async () => { + const result = await resolveAdminApiKey({ noAdminApiKey: true }, log); + expect(result).toBeUndefined(); + }); + }); + + describe('ephemeral mode (no dataDirectory)', () => { + it('returns a key resolution with rawKey and apiKeyHash', async () => { + const result = await resolveAdminApiKey({}, log); + expect(result).toBeDefined(); + expect(result!.rawKey).toBeDefined(); + expect(result!.apiKeyHash).toBeDefined(); + }); + + it('returns rawKey that is a 64-char hex string', async () => { + const result = await resolveAdminApiKey({}, log); + expect(result!.rawKey).toMatch(/^[0-9a-f]{64}$/); + }); + + it('returns apiKeyHash that is SHA-256 of rawKey', async () => { + const result = await resolveAdminApiKey({}, log); + expect(result!.apiKeyHash).toEqual(sha256Hash(result!.rawKey!)); + }); + + it('generates a different key each call', async () => { + const result1 = await resolveAdminApiKey({}, log); + const result2 = await resolveAdminApiKey({}, log); + expect(result1!.rawKey).not.toBe(result2!.rawKey); + }); + }); + + describe('persistent mode (with dataDirectory)', () => { + let opts: ResolveAdminApiKeyOptions; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'aztec-api-key-test-')); + opts = { dataDirectory: tempDir }; + }); + + it('generates a new key on first run', async () => { + const result = await resolveAdminApiKey(opts, log); + expect(result).toBeDefined(); + expect(result!.rawKey).toBeDefined(); + expect(result!.rawKey).toMatch(/^[0-9a-f]{64}$/); + expect(result!.apiKeyHash).toEqual(sha256Hash(result!.rawKey!)); + }); + + it('persists the hash to disk on first run', async () => { + const result = await resolveAdminApiKey(opts, log); + const hashFilePath = join(tempDir!, 'admin', 'api_key_hash'); + const storedHash = (await fs.readFile(hashFilePath, 'utf-8')).trim(); + expect(storedHash).toBe(result!.apiKeyHash.toString('hex')); + }); + + it('sets restrictive permissions on the hash file', async () => { + await resolveAdminApiKey(opts, log); + const hashFilePath = join(tempDir!, 'admin', 'api_key_hash'); + const stat = await fs.stat(hashFilePath); + expect(stat.mode & 0o777).toBe(0o600); + }); + + it('loads the stored hash on subsequent runs (no rawKey)', async () => { + // First run, generates and persists + const firstResult = await resolveAdminApiKey(opts, log); + const firstHash = firstResult!.apiKeyHash; + + // Second run, loads from disk + const secondResult = await resolveAdminApiKey(opts, log); + + expect(secondResult).toBeDefined(); + expect(secondResult!.apiKeyHash).toEqual(firstHash); + expect(secondResult!.rawKey).toBeUndefined(); // Not newly generated + }); + + it('regenerates if stored hash is invalid (wrong length)', async () => { + // Write an invalid hash + const adminDir = join(tempDir!, 'admin'); + await fs.mkdir(adminDir, { recursive: true }); + await fs.writeFile(join(adminDir, 'api_key_hash'), 'tooshort', 'utf-8'); + + const result = await resolveAdminApiKey(opts, log); + expect(result).toBeDefined(); + expect(result!.rawKey).toBeDefined(); // Freshly generated + expect(result!.apiKeyHash).toEqual(sha256Hash(result!.rawKey!)); + }); + + it('creates the admin subdirectory if it does not exist', async () => { + await resolveAdminApiKey(opts, log); + const adminDir = join(tempDir!, 'admin'); + const stat = await fs.stat(adminDir); + expect(stat.isDirectory()).toBe(true); + }); + }); + + describe('reset (resetAdminApiKey = true)', () => { + let opts: ResolveAdminApiKeyOptions; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'aztec-api-key-test-')); + opts = { dataDirectory: tempDir, resetAdminApiKey: true }; + }); + + it('generates a new key even when a valid hash already exists', async () => { + // First run, normal generation + const firstResult = await resolveAdminApiKey({ dataDirectory: tempDir }, log); + const firstHash = firstResult!.apiKeyHash; + + // Second run with reset, should generate a new key + const resetResult = await resolveAdminApiKey(opts, log); + + expect(resetResult).toBeDefined(); + expect(resetResult!.rawKey).toBeDefined(); // New raw key returned + expect(resetResult!.apiKeyHash).not.toEqual(firstHash); // Different hash + expect(resetResult!.apiKeyHash).toEqual(sha256Hash(resetResult!.rawKey!)); + }); + + it('overwrites the persisted hash file', async () => { + // First run — normal generation + await resolveAdminApiKey({ dataDirectory: tempDir }, log); + const hashFilePath = join(tempDir!, 'admin', 'api_key_hash'); + const oldHash = (await fs.readFile(hashFilePath, 'utf-8')).trim(); + + // Reset run + const resetResult = await resolveAdminApiKey(opts, log); + const newHash = (await fs.readFile(hashFilePath, 'utf-8')).trim(); + + expect(newHash).not.toBe(oldHash); + expect(newHash).toBe(resetResult!.apiKeyHash.toString('hex')); + }); + + it('works even when no hash file exists yet (first run with reset)', async () => { + const result = await resolveAdminApiKey(opts, log); + + expect(result).toBeDefined(); + expect(result!.rawKey).toBeDefined(); + expect(result!.apiKeyHash).toEqual(sha256Hash(result!.rawKey!)); + }); + + it('has no effect in ephemeral mode (always generates anyway)', async () => { + const result = await resolveAdminApiKey({ resetAdminApiKey: true }, log); + expect(result).toBeDefined(); + expect(result!.rawKey).toBeDefined(); + }); + }); +}); diff --git a/yarn-project/aztec/src/cli/admin_api_key_store.ts b/yarn-project/aztec/src/cli/admin_api_key_store.ts new file mode 100644 index 000000000000..6d0ea07c9c81 --- /dev/null +++ b/yarn-project/aztec/src/cli/admin_api_key_store.ts @@ -0,0 +1,128 @@ +import { randomBytes } from '@aztec/foundation/crypto/random'; +import { sha256Hash } from '@aztec/foundation/json-rpc/server'; +import type { Logger } from '@aztec/foundation/log'; + +import { promises as fs } from 'fs'; +import { join } from 'path'; + +/** Subdirectory under dataDirectory for admin API key storage. */ +const ADMIN_STORE_DIR = 'admin'; +const HASH_FILE_NAME = 'api_key_hash'; + +/** + * Result of resolving the admin API key. + * Contains the SHA-256 hex hash of the API key to be used by the auth middleware, + * and optionally the raw key when newly generated (so the caller can display it). + */ +export interface AdminApiKeyResolution { + /** The SHA-256 hash of the API key. */ + apiKeyHash: Buffer; + /** + * The raw API key, only present when a new key was generated during this call. + * The caller MUST display this to the operator — it will not be stored or returned again. + */ + rawKey?: string; +} + +export interface ResolveAdminApiKeyOptions { + /** SHA-256 hex hash of a pre-generated API key. When set, the node uses this hash directly. */ + adminApiKeyHash?: string; + /** If true, disable admin API key auth entirely. */ + noAdminApiKey?: boolean; + /** If true, force-generate a new key even if one is already persisted. */ + resetAdminApiKey?: boolean; + /** Root data directory for persistent storage. */ + dataDirectory?: string; +} + +/** + * Resolves the admin API key for the admin RPC endpoint. + * + * Strategy: + * 1. If opt-out flag is set (`noAdminApiKey`), return undefined (no auth). + * 2. If a pre-generated hash is provided (`adminApiKeyHash`), use it directly. + * 3. If a data directory exists, look for a persisted hash file + * at `/admin/api_key_hash`: + * - If `resetAdminApiKey` is set, skip loading and force-generate a new key. + * - Found: use the stored hash (operator already saved the key from first run). + * - Not found: auto-generate a random key, display it once, persist the hash. + * 3. If no data directory: generate a random key + * each run and display it (cannot persist). + * + * @param options - The options for resolving the admin API key. + * @param log - Logger for outputting the key and status messages. + * @returns The resolved API key hash, or undefined if auth is disabled. + */ +export async function resolveAdminApiKey( + options: ResolveAdminApiKeyOptions, + log: Logger, +): Promise { + // Operator explicitly opted out of admin auth + if (options.noAdminApiKey) { + log.warn('Admin API key authentication is DISABLED (--no-admin-api-key / AZTEC_NO_ADMIN_API_KEY)'); + return undefined; + } + + // Operator provided a pre-generated hash (e.g. via AZTEC_ADMIN_API_KEY_HASH env var) + if (options.adminApiKeyHash) { + const hex = options.adminApiKeyHash.trim(); + if (hex.length !== 64 || !/^[0-9a-f]{64}$/.test(hex)) { + throw new Error(`Invalid admin API key hash: expected 64-char hex string, got "${hex}"`); + } + log.info('Admin API key authentication enabled (using pre-configured key hash)'); + return { apiKeyHash: Buffer.from(hex, 'hex') }; + } + + // Persistent storage available, load or generate key + if (options.dataDirectory) { + const adminDir = join(options.dataDirectory, ADMIN_STORE_DIR); + const hashFilePath = join(adminDir, HASH_FILE_NAME); + + // Unless a reset is forced, try to load the existing hash from disk + if (!options.resetAdminApiKey) { + try { + const storedHash = (await fs.readFile(hashFilePath, 'utf-8')).trim(); + if (storedHash.length === 64) { + log.info('Admin API key authentication enabled (loaded stored key hash from disk)'); + return { apiKeyHash: Buffer.from(storedHash, 'hex') }; + } + log.warn(`Invalid stored admin API key hash at ${hashFilePath}, regenerating...`); + } catch (err: any) { + if (err.code !== 'ENOENT') { + log.warn(`Failed to read admin API key hash from ${hashFilePath}: ${err.message}`); + } + // File doesn't exist — fall through to generate + } + } else { + log.warn('Admin API key reset requested — generating a new key'); + } + + // Generate a new key, persist the hash, and return the raw key for the caller to display + const { rawKey, hash } = generateApiKey(); + await fs.mkdir(adminDir, { recursive: true }); + await fs.writeFile(hashFilePath, hash.toString('hex'), 'utf-8'); + // Set restrictive permissions (owner read/write only) + await fs.chmod(hashFilePath, 0o600); + + log.info('Admin API key authentication enabled (new key generated and hash persisted to disk)'); + return { apiKeyHash: hash, rawKey }; + } + + // No data directory, generate a temporary key per session + const { rawKey, hash } = generateApiKey(); + + log.warn('No data directory configured — admin API key cannot be persisted.'); + log.warn('A temporary key has been generated for this session only.'); + + return { apiKeyHash: hash, rawKey }; +} + +/** + * Generates a cryptographically random API key and its SHA-256 hash. + * @returns The raw key (hex string) and its SHA-256 hash as a Buffer. + */ +function generateApiKey(): { rawKey: string; hash: Buffer } { + const rawKey = randomBytes(32).toString('hex'); + const hash = sha256Hash(rawKey); + return { rawKey, hash }; +} diff --git a/yarn-project/aztec/src/cli/aztec_start_action.ts b/yarn-project/aztec/src/cli/aztec_start_action.ts index 8217313dd09c..9ffad783f0f1 100644 --- a/yarn-project/aztec/src/cli/aztec_start_action.ts +++ b/yarn-project/aztec/src/cli/aztec_start_action.ts @@ -1,6 +1,7 @@ import { type NamespacedApiHandlers, createNamespacedSafeJsonRpcServer, + getApiKeyAuthMiddleware, startHttpRpcServer, } from '@aztec/foundation/json-rpc/server'; import type { LogFn, Logger } from '@aztec/foundation/log'; @@ -11,6 +12,7 @@ import { getOtelJsonRpcPropagationMiddleware } from '@aztec/telemetry-client'; import { createLocalNetwork } from '../local-network/index.js'; import { github, splash } from '../splash.js'; +import { resolveAdminApiKey } from './admin_api_key_store.js'; import { getCliVersion } from './release_version.js'; import { extractNamespacedOptions, installSignalHandlers } from './util.js'; import { getVersions } from './versioning.js'; @@ -48,15 +50,17 @@ export async function aztecStart(options: any, userLog: LogFn, debugLogger: Logg signalHandlers.push(stop); services.node = [node, AztecNodeApiSchema]; } else { + // Route --prover-node through startNode + if (options.proverNode && !options.node) { + options.node = true; + } + if (options.node) { const { startNode } = await import('./cmds/start_node.js'); ({ config } = await startNode(options, signalHandlers, services, adminServices, userLog)); } else if (options.bot) { const { startBot } = await import('./cmds/start_bot.js'); await startBot(options, signalHandlers, services, userLog); - } else if (options.proverNode) { - const { startProverNode } = await import('./cmds/start_prover_node.js'); - ({ config } = await startProverNode(options, signalHandlers, services, userLog)); } else if (options.archiver) { const { startArchiver } = await import('./cmds/start_archiver.js'); ({ config } = await startArchiver(options, signalHandlers, services)); @@ -99,14 +103,54 @@ export async function aztecStart(options: any, userLog: LogFn, debugLogger: Logg // If there are any admin services, start a separate JSON-RPC server for them if (Object.entries(adminServices).length > 0) { + const adminMiddlewares = [getOtelJsonRpcPropagationMiddleware(), getVersioningMiddleware(versions)]; + + // Resolve the admin API key (auto-generated and persisted, or opt-out) + const apiKeyResolution = await resolveAdminApiKey( + { + adminApiKeyHash: options.adminApiKeyHash, + noAdminApiKey: options.noAdminApiKey, + resetAdminApiKey: options.resetAdminApiKey, + dataDirectory: options.dataDirectory, + }, + debugLogger, + ); + if (apiKeyResolution) { + adminMiddlewares.unshift(getApiKeyAuthMiddleware(apiKeyResolution.apiKeyHash)); + } else { + debugLogger.warn('No admin API key set — admin endpoint is unauthenticated'); + } + const rpcServer = createNamespacedSafeJsonRpcServer(adminServices, { http200OnError: false, log: debugLogger, - middlewares: [getOtelJsonRpcPropagationMiddleware(), getVersioningMiddleware(versions)], + middlewares: adminMiddlewares, maxBatchSize: options.rpcMaxBatchSize, maxBodySizeBytes: options.rpcMaxBodySize, }); const { port } = await startHttpRpcServer(rpcServer, { port: options.adminPort }); debugLogger.info(`Aztec Server admin API listening on port ${port}`, versions); + + // Display the API key after the server has started + // Uses userLog which is never filtered by LOG_LEVEL. + if (apiKeyResolution?.rawKey) { + const separator = '='.repeat(70); + userLog(''); + userLog(separator); + userLog(' ADMIN API KEY (save this — it will NOT be shown again)'); + userLog(''); + userLog(` ${apiKeyResolution.rawKey}`); + userLog(''); + userLog(` Use via header: x-api-key: `); + userLog(` Or via header: Authorization: Bearer `); + if (options.dataDirectory) { + userLog(''); + userLog(' The key hash has been persisted — on next restart, the same key will be used.'); + } + userLog(''); + userLog(' To disable admin auth: --no-admin-api-key or AZTEC_NO_ADMIN_API_KEY=true'); + userLog(separator); + userLog(''); + } } } diff --git a/yarn-project/aztec/src/cli/aztec_start_options.ts b/yarn-project/aztec/src/cli/aztec_start_options.ts index 46ef250c1359..31337c92a92a 100644 --- a/yarn-project/aztec/src/cli/aztec_start_options.ts +++ b/yarn-project/aztec/src/cli/aztec_start_options.ts @@ -142,6 +142,29 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { env: 'AZTEC_ADMIN_PORT', parseVal: val => parseInt(val, 10), }, + { + flag: '--admin-api-key-hash ', + description: + 'SHA-256 hex hash of a pre-generated admin API key. When set, the node uses this hash for authentication instead of auto-generating a key.', + defaultValue: undefined, + env: 'AZTEC_ADMIN_API_KEY_HASH', + }, + { + flag: '--no-admin-api-key', + description: + 'Disable API key authentication on the admin RPC endpoint. By default, a key is auto-generated, displayed once, and its hash is persisted.', + defaultValue: false, + env: 'AZTEC_NO_ADMIN_API_KEY', + parseVal: val => val === 'true' || val === '1', + }, + { + flag: '--reset-admin-api-key', + description: + 'Force-generate a new admin API key, replacing any previously persisted key hash. The new key is displayed once at startup.', + defaultValue: false, + env: 'AZTEC_RESET_ADMIN_API_KEY', + parseVal: val => val === 'true' || val === '1', + }, { flag: '--api-prefix ', description: 'Prefix for API routes on any service that is started', @@ -170,7 +193,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { 'WORLD STATE': [ configToFlag('--world-state-data-directory', worldStateConfigMappings.worldStateDataDirectory), configToFlag('--world-state-db-map-size-kb', worldStateConfigMappings.worldStateDbMapSizeKb), - configToFlag('--world-state-block-history', worldStateConfigMappings.worldStateBlockHistory), + configToFlag('--world-state-checkpoint-history', worldStateConfigMappings.worldStateCheckpointHistory), ], // We can't easily auto-generate node options as they're parts of modules defined below 'AZTEC NODE': [ @@ -222,12 +245,8 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = { 'proverNode', omitConfigMappings(proverNodeConfigMappings, [ // filter out options passed separately - ...getKeys(archiverConfigMappings), ...getKeys(proverBrokerConfigMappings), ...getKeys(proverAgentConfigMappings), - ...getKeys(p2pConfigMappings), - ...getKeys(worldStateConfigMappings), - ...getKeys(sharedNodeConfigMappings), ]), ), ], diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts index cb3e211d2b12..abed355ce4f6 100644 --- a/yarn-project/aztec/src/cli/cmds/start_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -6,13 +6,16 @@ import { getL1Config } from '@aztec/cli/config'; import { getPublicClient } from '@aztec/ethereum/client'; import { SecretValue } from '@aztec/foundation/config'; import type { NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server'; +import { Agent, makeUndiciFetch } from '@aztec/foundation/json-rpc/undici'; import type { LogFn } from '@aztec/foundation/log'; +import { ProvingJobConsumerSchema, createProvingJobBrokerClient } from '@aztec/prover-client/broker'; import { type CliPXEOptions, type PXEConfig, allPxeConfigMappings } from '@aztec/pxe/config'; import { AztecNodeAdminApiSchema, AztecNodeApiSchema } from '@aztec/stdlib/interfaces/client'; -import { P2PApiSchema } from '@aztec/stdlib/interfaces/server'; +import { P2PApiSchema, ProverNodeApiSchema, type ProvingJobBroker } from '@aztec/stdlib/interfaces/server'; import { type TelemetryClientConfig, initTelemetryClient, + makeTracedFetch, telemetryClientConfigMappings, } from '@aztec/telemetry-client'; import { EmbeddedWallet } from '@aztec/wallets/embedded'; @@ -25,6 +28,8 @@ import { preloadCrsDataForVerifying, setupUpdateMonitor, } from '../util.js'; +import { getVersions } from '../versioning.js'; +import { startProverBroker } from './start_prover_broker.js'; export async function startNode( options: any, @@ -45,9 +50,32 @@ export async function startNode( ...relevantOptions, }; + // Prover node configuration and broker setup + // REFACTOR: Move the broker setup out of here and into the prover-node factory + let broker: ProvingJobBroker | undefined = undefined; if (options.proverNode) { - userLog(`Running a Prover Node within a Node is not yet supported`); - process.exit(1); + nodeConfig.enableProverNode = true; + if (nodeConfig.proverAgentCount === 0) { + userLog( + `Running prover node without local prover agent. Connect prover agents or pass --proverAgent.proverAgentCount`, + ); + } + if (nodeConfig.proverBrokerUrl) { + // at 1TPS we'd enqueue ~1k chonk verifier proofs and ~1k AVM proofs immediately + // set a lower connection limit such that we don't overload the server + // Keep retrying up to 30s + const fetch = makeTracedFetch( + [1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3], + false, + makeUndiciFetch(new Agent({ connections: 100 })), + ); + broker = createProvingJobBrokerClient(nodeConfig.proverBrokerUrl, getVersions(nodeConfig), fetch); + } else if (options.proverBroker) { + ({ broker } = await startProverBroker(options, signalHandlers, services, userLog)); + } else { + userLog(`--prover-broker-url or --prover-broker is required to start a Prover Node`); + process.exit(1); + } } await preloadCrsDataForVerifying(nodeConfig, userLog); @@ -101,12 +129,17 @@ export async function startNode( ...extractNamespacedOptions(options, 'sequencer'), }; // If no publisher private keys have been given, use the first validator key - if (sequencerConfig.publisherPrivateKeys === undefined || !sequencerConfig.publisherPrivateKeys.length) { + if ( + sequencerConfig.sequencerPublisherPrivateKeys === undefined || + !sequencerConfig.sequencerPublisherPrivateKeys.length + ) { if (sequencerConfig.validatorPrivateKeys?.getValue().length) { - sequencerConfig.publisherPrivateKeys = [new SecretValue(sequencerConfig.validatorPrivateKeys.getValue()[0])]; + sequencerConfig.sequencerPublisherPrivateKeys = [ + new SecretValue(sequencerConfig.validatorPrivateKeys.getValue()[0]), + ]; } } - nodeConfig.publisherPrivateKeys = sequencerConfig.publisherPrivateKeys; + nodeConfig.sequencerPublisherPrivateKeys = sequencerConfig.sequencerPublisherPrivateKeys; } if (nodeConfig.p2pEnabled) { @@ -120,13 +153,22 @@ export async function startNode( const telemetry = await initTelemetryClient(telemetryConfig); // Create and start Aztec Node - const node = await createAztecNode(nodeConfig, { telemetry }, { prefilledPublicData }); + const node = await createAztecNode(nodeConfig, { telemetry, proverBroker: broker }, { prefilledPublicData }); // Add node and p2p to services list services.node = [node, AztecNodeApiSchema]; services.p2p = [node.getP2P(), P2PApiSchema]; adminServices.nodeAdmin = [node, AztecNodeAdminApiSchema]; + // Register prover-node services if the prover node subsystem is running + const proverNode = node.getProverNode(); + if (proverNode) { + services.prover = [proverNode, ProverNodeApiSchema]; + if (!nodeConfig.proverBrokerUrl) { + services.provingJobSource = [proverNode.getProver().getProvingJobSource(), ProvingJobConsumerSchema]; + } + } + // Add node stop function to signal handlers signalHandlers.push(node.stop.bind(node)); diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts deleted file mode 100644 index 0778616eee3a..000000000000 --- a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { getInitialTestAccountsData } from '@aztec/accounts/testing'; -import { Fr } from '@aztec/aztec.js/fields'; -import { getSponsoredFPCAddress } from '@aztec/cli/cli-utils'; -import { getL1Config } from '@aztec/cli/config'; -import { getPublicClient } from '@aztec/ethereum/client'; -import type { NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server'; -import { Agent, makeUndiciFetch } from '@aztec/foundation/json-rpc/undici'; -import type { LogFn } from '@aztec/foundation/log'; -import { ProvingJobConsumerSchema, createProvingJobBrokerClient } from '@aztec/prover-client/broker'; -import { - type ProverNodeConfig, - createProverNode, - getProverNodeConfigFromEnv, - proverNodeConfigMappings, -} from '@aztec/prover-node'; -import { P2PApiSchema, ProverNodeApiSchema, type ProvingJobBroker } from '@aztec/stdlib/interfaces/server'; -import { initTelemetryClient, makeTracedFetch, telemetryClientConfigMappings } from '@aztec/telemetry-client'; -import { getGenesisValues } from '@aztec/world-state/testing'; - -import { extractRelevantOptions, preloadCrsDataForVerifying, setupUpdateMonitor } from '../util.js'; -import { getVersions } from '../versioning.js'; -import { startProverBroker } from './start_prover_broker.js'; - -export async function startProverNode( - options: any, - signalHandlers: (() => Promise)[], - services: NamespacedApiHandlers, - userLog: LogFn, -): Promise<{ config: ProverNodeConfig }> { - if (options.node || options.sequencer || options.pxe || options.p2pBootstrap || options.txe) { - userLog(`Starting a prover-node with --node, --sequencer, --pxe, --p2p-bootstrap, or --txe is not supported.`); - process.exit(1); - } - - let proverConfig = { - ...getProverNodeConfigFromEnv(), // get default config from env - ...extractRelevantOptions(options, proverNodeConfigMappings, 'proverNode'), // override with command line options - }; - - if (!proverConfig.l1Contracts.registryAddress || proverConfig.l1Contracts.registryAddress.isZero()) { - throw new Error('L1 registry address is required to start a Prover Node'); - } - - const followsCanonicalRollup = typeof proverConfig.rollupVersion !== 'number'; - const { addresses, config } = await getL1Config( - proverConfig.l1Contracts.registryAddress, - proverConfig.l1RpcUrls, - proverConfig.l1ChainId, - proverConfig.rollupVersion, - ); - process.env.ROLLUP_CONTRACT_ADDRESS ??= addresses.rollupAddress.toString(); - proverConfig.l1Contracts = addresses; - proverConfig = { ...proverConfig, ...config }; - - const testAccounts = proverConfig.testAccounts ? (await getInitialTestAccountsData()).map(a => a.address) : []; - const sponsoredFPCAccounts = proverConfig.sponsoredFPC ? [await getSponsoredFPCAddress()] : []; - const initialFundedAccounts = testAccounts.concat(sponsoredFPCAccounts); - - userLog(`Initial funded accounts: ${initialFundedAccounts.map(a => a.toString()).join(', ')}`); - const { genesisArchiveRoot, prefilledPublicData } = await getGenesisValues(initialFundedAccounts); - - userLog(`Genesis archive root: ${genesisArchiveRoot.toString()}`); - - if (!Fr.fromHexString(config.genesisArchiveTreeRoot).equals(genesisArchiveRoot)) { - throw new Error( - `The computed genesis archive tree root ${genesisArchiveRoot} does not match the expected genesis archive tree root ${config.genesisArchiveTreeRoot} for the rollup deployed at ${addresses.rollupAddress}`, - ); - } - - const telemetry = await initTelemetryClient(extractRelevantOptions(options, telemetryClientConfigMappings, 'tel')); - - let broker: ProvingJobBroker; - if (proverConfig.proverBrokerUrl) { - // at 1TPS we'd enqueue ~1k chonk verifier proofs and ~1k AVM proofs immediately - // set a lower connection limit such that we don't overload the server - // Keep retrying up to 30s - const fetch = makeTracedFetch( - [1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3], - false, - makeUndiciFetch(new Agent({ connections: 100 })), - ); - broker = createProvingJobBrokerClient(proverConfig.proverBrokerUrl, getVersions(proverConfig), fetch); - } else if (options.proverBroker) { - ({ broker } = await startProverBroker(options, signalHandlers, services, userLog)); - } else { - userLog(`--prover-broker-url or --prover-broker is required to start a Prover Node`); - process.exit(1); - } - - if (proverConfig.proverAgentCount === 0) { - userLog( - `Running prover node without local prover agent. Connect one or more prover agents to this node or pass --proverAgent.proverAgentCount`, - ); - } - - await preloadCrsDataForVerifying(proverConfig, userLog); - - const proverNode = await createProverNode(proverConfig, { telemetry, broker }, { prefilledPublicData }); - services.proverNode = [proverNode, ProverNodeApiSchema]; - - if (proverNode.getP2P()) { - services.p2p = [proverNode.getP2P(), P2PApiSchema]; - } - - if (!proverConfig.proverBrokerUrl) { - services.provingJobSource = [proverNode.getProver().getProvingJobSource(), ProvingJobConsumerSchema]; - } - - signalHandlers.push(proverNode.stop.bind(proverNode)); - - await proverNode.start(); - - if (proverConfig.autoUpdate !== 'disabled' && proverConfig.autoUpdateUrl) { - await setupUpdateMonitor( - proverConfig.autoUpdate, - new URL(proverConfig.autoUpdateUrl), - followsCanonicalRollup, - getPublicClient(proverConfig), - proverConfig.l1Contracts.registryAddress, - signalHandlers, - ); - } - return { config: proverConfig }; -} diff --git a/yarn-project/aztec/src/local-network/local-network.ts b/yarn-project/aztec/src/local-network/local-network.ts index b25c9bf8dce1..733ddd550a84 100644 --- a/yarn-project/aztec/src/local-network/local-network.ts +++ b/yarn-project/aztec/src/local-network/local-network.ts @@ -18,6 +18,7 @@ import type { LogFn } from '@aztec/foundation/log'; import { DateProvider, TestDateProvider } from '@aztec/foundation/timer'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import { protocolContractsHash } from '@aztec/protocol-contracts'; +import type { ProvingJobBroker } from '@aztec/stdlib/interfaces/server'; import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees'; import { type TelemetryClient, @@ -105,12 +106,14 @@ export async function createLocalNetwork(config: Partial = { }; const hdAccount = mnemonicToAccount(config.l1Mnemonic || DefaultMnemonic); if ( - aztecNodeConfig.publisherPrivateKeys == undefined || - !aztecNodeConfig.publisherPrivateKeys.length || - aztecNodeConfig.publisherPrivateKeys[0].getValue() === NULL_KEY + aztecNodeConfig.sequencerPublisherPrivateKeys == undefined || + !aztecNodeConfig.sequencerPublisherPrivateKeys.length || + aztecNodeConfig.sequencerPublisherPrivateKeys[0].getValue() === NULL_KEY ) { const privKey = hdAccount.getHdKey().privateKey; - aztecNodeConfig.publisherPrivateKeys = [new SecretValue(`0x${Buffer.from(privKey!).toString('hex')}` as const)]; + aztecNodeConfig.sequencerPublisherPrivateKeys = [ + new SecretValue(`0x${Buffer.from(privKey!).toString('hex')}` as const), + ]; } if (!aztecNodeConfig.validatorPrivateKeys?.getValue().length) { const privKey = hdAccount.getHdKey().privateKey; @@ -221,7 +224,12 @@ export async function createLocalNetwork(config: Partial = { */ export async function createAztecNode( config: Partial = {}, - deps: { telemetry?: TelemetryClient; blobClient?: BlobClientInterface; dateProvider?: DateProvider } = {}, + deps: { + telemetry?: TelemetryClient; + blobClient?: BlobClientInterface; + dateProvider?: DateProvider; + proverBroker?: ProvingJobBroker; + } = {}, options: { prefilledPublicData?: PublicDataTreeLeaf[] } = {}, ) { // TODO(#12272): will clean this up. This is criminal. @@ -231,6 +239,10 @@ export async function createAztecNode( ...config, l1Contracts: { ...l1Contracts, ...config.l1Contracts }, }; - const node = await AztecNodeService.createAndSync(aztecNodeConfig, deps, options); + const node = await AztecNodeService.createAndSync( + aztecNodeConfig, + { ...deps, proverNodeDeps: { broker: deps.proverBroker } }, + options, + ); return node; } diff --git a/yarn-project/blob-client/src/blobstore/blob_store_test_suite.ts b/yarn-project/blob-client/src/blobstore/blob_store_test_suite.ts index e7f9df2e627e..3786f1e3bfff 100644 --- a/yarn-project/blob-client/src/blobstore/blob_store_test_suite.ts +++ b/yarn-project/blob-client/src/blobstore/blob_store_test_suite.ts @@ -13,7 +13,7 @@ export function describeBlobStore(getBlobStore: () => Promise) { it('should store and retrieve a blob by hash', async () => { // Create a test blob with random fields const testFields = [Fr.random(), Fr.random(), Fr.random()]; - const blob = Blob.fromFields(testFields); + const blob = await Blob.fromFields(testFields); const blobHash = blob.getEthVersionedBlobHash(); // Store the blob @@ -29,8 +29,8 @@ export function describeBlobStore(getBlobStore: () => Promise) { it('should handle multiple blobs stored and retrieved by their hashes', async () => { // Create two different blobs - const blob1 = Blob.fromFields([Fr.random(), Fr.random()]); - const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]); + const blob1 = await Blob.fromFields([Fr.random(), Fr.random()]); + const blob2 = await Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]); const blobHash1 = blob1.getEthVersionedBlobHash(); const blobHash2 = blob2.getEthVersionedBlobHash(); @@ -57,9 +57,9 @@ export function describeBlobStore(getBlobStore: () => Promise) { it('should handle retrieving subset of stored blobs', async () => { // Store multiple blobs - const blob1 = Blob.fromFields([Fr.random()]); - const blob2 = Blob.fromFields([Fr.random()]); - const blob3 = Blob.fromFields([Fr.random()]); + const blob1 = await Blob.fromFields([Fr.random()]); + const blob2 = await Blob.fromFields([Fr.random()]); + const blob3 = await Blob.fromFields([Fr.random()]); await blobStore.addBlobs([blob1, blob2, blob3]); @@ -75,7 +75,7 @@ export function describeBlobStore(getBlobStore: () => Promise) { }); it('should handle duplicate blob hashes in request', async () => { - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const blobHash = blob.getEthVersionedBlobHash(); await blobStore.addBlobs([blob]); @@ -91,8 +91,8 @@ export function describeBlobStore(getBlobStore: () => Promise) { it('should overwrite blob when storing with same hash', async () => { // Create two blobs that will have the same hash (same content) const fields = [Fr.random(), Fr.random()]; - const blob1 = Blob.fromFields(fields); - const blob2 = Blob.fromFields(fields); + const blob1 = await Blob.fromFields(fields); + const blob2 = await Blob.fromFields(fields); const blobHash = blob1.getEthVersionedBlobHash(); diff --git a/yarn-project/blob-client/src/client/http.test.ts b/yarn-project/blob-client/src/client/http.test.ts index 5d3c6b16422e..c5c7d8eb832f 100644 --- a/yarn-project/blob-client/src/client/http.test.ts +++ b/yarn-project/blob-client/src/client/http.test.ts @@ -15,7 +15,7 @@ import { HttpBlobClient } from './http.js'; describe('HttpBlobClient', () => { it('should handle no sources configured', async () => { const client = new HttpBlobClient({}); - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const blobHash = blob.getEthVersionedBlobHash(); const success = await client.sendBlobsToFilestore([blob]); @@ -40,11 +40,11 @@ describe('HttpBlobClient', () => { let latestSlotNumber: number; let missedSlots: number[]; - beforeEach(() => { + beforeEach(async () => { latestSlotNumber = 1; missedSlots = []; - testBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3)); + testBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3))); testBlobsHashes = testBlobs.map(b => b.getEthVersionedBlobHash()); blobData = testBlobs.map(b => b.toJSON()); @@ -292,7 +292,7 @@ describe('HttpBlobClient', () => { }); // Create a blob that has mismatch data and commitment. - const randomBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3)); + const randomBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3))); const incorrectBlob = new Blob(randomBlobs[0].data, randomBlobs[1].commitment); const incorrectBlobHash = incorrectBlob.getEthVersionedBlobHash(); // Update blobData to include the incorrect blob @@ -312,7 +312,7 @@ describe('HttpBlobClient', () => { it('should accumulate blobs across all three sources (filestore, consensus, archive)', async () => { // Create three blobs for testing - const blobs = Array.from({ length: 3 }, () => makeRandomBlob(3)); + const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(3))); const blobHashes = blobs.map(b => b.getEthVersionedBlobHash()); // Blob 0 only in filestore @@ -368,7 +368,7 @@ describe('HttpBlobClient', () => { it('should preserve blob order when requesting multiple blobs', async () => { // Create three distinct blobs - const blobs = Array.from({ length: 3 }, () => makeRandomBlob(3)); + const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(3))); const blobHashes = blobs.map(b => b.getEthVersionedBlobHash()); // Add all blobs to filestore @@ -477,7 +477,7 @@ describe('HttpBlobClient', () => { it('should return only one blob when multiple blobs with the same blobHash exist on a block', async () => { // Create a blob data array with two blobs that have the same commitment (thus same blobHash) - const blob = makeRandomBlob(3); + const blob = await makeRandomBlob(3); const blobHash = blob.getEthVersionedBlobHash(); const duplicateBlobData = [blob.toJSON(), blob.toJSON()]; @@ -503,7 +503,7 @@ describe('HttpBlobClient', () => { l1ConsensusHostUrls: [`http://localhost:${consensusHostPort}`], }); - const blob = makeRandomBlob(3); + const blob = await makeRandomBlob(3); const blobHash = blob.getEthVersionedBlobHash(); const blobJson = blob.toJSON(); @@ -616,8 +616,8 @@ describe('HttpBlobClient FileStore Integration', () => { let testBlobs: Blob[]; let testBlobsHashes: Buffer[]; - beforeEach(() => { - testBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3)); + beforeEach(async () => { + testBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3))); testBlobsHashes = testBlobs.map(b => b.getEthVersionedBlobHash()); }); diff --git a/yarn-project/blob-client/src/client/http.ts b/yarn-project/blob-client/src/client/http.ts index a27fd2ea1d98..5d626933261a 100644 --- a/yarn-project/blob-client/src/client/http.ts +++ b/yarn-project/blob-client/src/client/http.ts @@ -215,8 +215,8 @@ export class HttpBlobClient implements BlobClientInterface { const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined); // Helper to fill in results from fetched blobs - const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => { - const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log); + const fillResults = async (fetchedBlobs: BlobJson[]): Promise => { + const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log); // Fill in any missing positions with matching blobs for (let i = 0; i < blobHashes.length; i++) { if (resultBlobs[i] === undefined) { @@ -269,7 +269,7 @@ export class HttpBlobClient implements BlobClientInterface { ...ctx, }); const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex); - const result = fillResults(blobs); + const result = await fillResults(blobs); this.log.debug( `Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`, { slotNumber, l1ConsensusHostUrl, ...ctx }, @@ -312,7 +312,7 @@ export class HttpBlobClient implements BlobClientInterface { this.log.debug('No blobs found from archive client', archiveCtx); } else { this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx); - const result = fillResults(allBlobs); + const result = await fillResults(allBlobs); this.log.debug( `Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`, archiveCtx, @@ -345,7 +345,7 @@ export class HttpBlobClient implements BlobClientInterface { */ private async tryFileStores( getMissingBlobHashes: () => Buffer[], - fillResults: (blobs: BlobJson[]) => Blob[], + fillResults: (blobs: BlobJson[]) => Promise, ctx: { blockHash: string; blobHashes: string[] }, ): Promise { // Shuffle clients for load distribution @@ -366,7 +366,7 @@ export class HttpBlobClient implements BlobClientInterface { }); const blobs = await client.getBlobsByHashes(blobHashStrings); if (blobs.length > 0) { - const result = fillResults(blobs); + const result = await fillResults(blobs); this.log.debug( `Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`, { @@ -388,7 +388,7 @@ export class HttpBlobClient implements BlobClientInterface { l1ConsensusHostIndex?: number, ): Promise { const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex); - return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined); + return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined); } public async getBlobsFromHost( @@ -616,7 +616,11 @@ function parseBlobJson(data: any): BlobJson { // Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found // or the data does not match the commitment. -function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Logger): (Blob | undefined)[] { +async function processFetchedBlobs( + blobs: BlobJson[], + blobHashes: Buffer[], + logger: Logger, +): Promise<(Blob | undefined)[]> { const requestedBlobHashes = new Set(blobHashes.map(bufferToHex)); const hashToBlob = new Map(); for (const blobJson of blobs) { @@ -626,7 +630,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo } try { - const blob = Blob.fromJson(blobJson); + const blob = await Blob.fromJson(blobJson); hashToBlob.set(hashHex, blob); } catch (err) { // If the above throws, it's likely that the blob commitment does not match the hash of the blob data. diff --git a/yarn-project/blob-client/src/client/tests.ts b/yarn-project/blob-client/src/client/tests.ts index 826aa286781f..d85430f4fb51 100644 --- a/yarn-project/blob-client/src/client/tests.ts +++ b/yarn-project/blob-client/src/client/tests.ts @@ -28,7 +28,7 @@ export function runBlobClientTests( }); it('should send and retrieve blobs by hash', async () => { - const blob = makeRandomBlob(5); + const blob = await makeRandomBlob(5); const blobHash = blob.getEthVersionedBlobHash(); await client.sendBlobsToFilestore([blob]); @@ -39,7 +39,7 @@ export function runBlobClientTests( }); it('should handle multiple blobs', async () => { - const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7)); + const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(7))); const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash()); await client.sendBlobsToFilestore(blobs); diff --git a/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts b/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts index fe30df599709..872f37cb20b4 100644 --- a/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts +++ b/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts @@ -74,7 +74,7 @@ describe('FileStoreBlobClient', () => { describe('saveBlob', () => { it('should save a blob to the filestore', async () => { - const blob = Blob.fromFields([Fr.random(), Fr.random()]); + const blob = await Blob.fromFields([Fr.random(), Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; await client.saveBlob(blob); @@ -88,7 +88,7 @@ describe('FileStoreBlobClient', () => { }); it('should skip saving if blob already exists and skipIfExists=true', async () => { - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; // Save first time @@ -107,7 +107,7 @@ describe('FileStoreBlobClient', () => { }); it('should overwrite if skipIfExists=false', async () => { - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; // Save first time @@ -130,8 +130,8 @@ describe('FileStoreBlobClient', () => { describe('saveBlobs', () => { it('should save multiple blobs', async () => { - const blob1 = Blob.fromFields([Fr.random()]); - const blob2 = Blob.fromFields([Fr.random()]); + const blob1 = await Blob.fromFields([Fr.random()]); + const blob2 = await Blob.fromFields([Fr.random()]); await client.saveBlobs([blob1, blob2]); @@ -145,7 +145,7 @@ describe('FileStoreBlobClient', () => { describe('getBlobsByHashes', () => { it('should retrieve blobs by their versioned hashes', async () => { - const blob = Blob.fromFields([Fr.random(), Fr.random()]); + const blob = await Blob.fromFields([Fr.random(), Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; await client.saveBlob(blob); @@ -163,8 +163,8 @@ describe('FileStoreBlobClient', () => { }); it('should retrieve multiple blobs', async () => { - const blob1 = Blob.fromFields([Fr.random()]); - const blob2 = Blob.fromFields([Fr.random()]); + const blob1 = await Blob.fromFields([Fr.random()]); + const blob2 = await Blob.fromFields([Fr.random()]); await client.saveBlobs([blob1, blob2]); @@ -177,7 +177,7 @@ describe('FileStoreBlobClient', () => { }); it('should skip blobs that fail to parse', async () => { - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const hash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; // Save invalid JSON @@ -191,7 +191,7 @@ describe('FileStoreBlobClient', () => { describe('exists', () => { it('should return true if blob exists', async () => { - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; await client.saveBlob(blob); @@ -240,14 +240,14 @@ describe('FileStoreBlobClient', () => { const readOnlyStore = new MockReadOnlyFileStore(); const readOnlyClient = new FileStoreBlobClient(readOnlyStore, basePath); - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); await expect(readOnlyClient.saveBlob(blob)).rejects.toThrow('FileStore is read-only'); }); it('should be able to read from read-only store', async () => { const files = new Map(); - const blob = Blob.fromFields([Fr.random()]); + const blob = await Blob.fromFields([Fr.random()]); const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`; const path = `${basePath}/blobs/${versionedHash}.data`; diff --git a/yarn-project/blob-lib/src/blob.test.ts b/yarn-project/blob-lib/src/blob.test.ts index 4939baa09f84..3053eb1d077d 100644 --- a/yarn-project/blob-lib/src/blob.test.ts +++ b/yarn-project/blob-lib/src/blob.test.ts @@ -58,10 +58,10 @@ describe('blob', () => { // This test ensures that the noir blob lib correctly matches the kzg lib const blobFields = Array(400).fill(new Fr(3)); const blobFieldsHash = await poseidon2Hash(blobFields); - const blob = Blob.fromFields(blobFields); + const blob = await Blob.fromFields(blobFields); const challengeZ = await blob.computeChallengeZ(blobFieldsHash); - const { y } = blob.evaluate(challengeZ, true /* verifyProof */); + const { y } = await blob.evaluate(challengeZ, true /* verifyProof */); expect(blob.commitment.toString('hex')).toMatchInlineSnapshot( `"b2803d5fe972914ba3616033e2748bbaa6dbcddefc3721a54895a7a45e77504dd1a971c7e8d8292be943d05bccebcfea"`, @@ -88,10 +88,10 @@ describe('blob', () => { // This test ensures that the noir blob lib correctly matches the kzg lib const blobFields = Array.from({ length: FIELDS_PER_BLOB }).map((_, i) => new Fr(i + 2)); const blobFieldsHash = await poseidon2Hash(blobFields); - const blob = Blob.fromFields(blobFields); + const blob = await Blob.fromFields(blobFields); const challengeZ = await blob.computeChallengeZ(blobFieldsHash); - const { y } = blob.evaluate(challengeZ, true /* verifyProof */); + const { y } = await blob.evaluate(challengeZ, true /* verifyProof */); expect(blob.commitment.toString('hex')).toMatchInlineSnapshot( `"ac771dea41e29fc2b7016c32731602c0812548ba0f491864a4e03fdb94b8d3d195faad1967cdf005acf73088b0e8474a"`, @@ -114,15 +114,15 @@ describe('blob', () => { ); }); - it('should serialize and deserialize a blob', () => { - const blob = makeRandomBlob(5); + it('should serialize and deserialize a blob', async () => { + const blob = await makeRandomBlob(5); const blobBuffer = blob.toBuffer(); expect(Blob.fromBuffer(blobBuffer)).toEqual(blob); }); - it('should create a blob from a JSON object', () => { - const blob = makeRandomBlob(7); + it('should create a blob from a JSON object', async () => { + const blob = await makeRandomBlob(7); const blobJson = blob.toJSON(); - expect(Blob.fromJson(blobJson)).toEqual(blob); + expect(await Blob.fromJson(blobJson)).toEqual(blob); }); }); diff --git a/yarn-project/blob-lib/src/blob.ts b/yarn-project/blob-lib/src/blob.ts index 25f0b087e773..230b2ff957a9 100644 --- a/yarn-project/blob-lib/src/blob.ts +++ b/yarn-project/blob-lib/src/blob.ts @@ -42,8 +42,8 @@ export class Blob { * * @throws If data does not match the expected length (BYTES_PER_BLOB). */ - static fromBlobBuffer(data: Uint8Array): Blob { - const commitment = computeBlobCommitment(data); + static async fromBlobBuffer(data: Uint8Array): Promise { + const commitment = await computeBlobCommitment(data); return new Blob(data, commitment); } @@ -55,13 +55,13 @@ export class Blob { * @param fields - The array of fields to create the Blob from. * @returns A Blob created from the array of fields. */ - static fromFields(fields: Fr[]): Blob { + static async fromFields(fields: Fr[]): Promise { if (fields.length > FIELDS_PER_BLOB) { throw new Error(`Attempted to overfill blob with ${fields.length} fields. The maximum is ${FIELDS_PER_BLOB}.`); } const data = Buffer.concat([serializeToBuffer(fields)], BYTES_PER_BLOB); - const commitment = computeBlobCommitment(data); + const commitment = await computeBlobCommitment(data); return new Blob(data, commitment); } @@ -88,9 +88,9 @@ export class Blob { * @param json - The JSON object to create the Blob from. * @returns A Blob created from the JSON object. */ - static fromJson(json: BlobJson): Blob { + static async fromJson(json: BlobJson): Promise { const blobBuffer = Buffer.from(json.blob.slice(2), 'hex'); - const blob = Blob.fromBlobBuffer(blobBuffer); + const blob = await Blob.fromBlobBuffer(blobBuffer); if (blob.commitment.toString('hex') !== json.kzg_commitment.slice(2)) { throw new Error('KZG commitment does not match'); @@ -134,9 +134,9 @@ export class Blob { * y: BLS12Fr - Evaluation y = p(z), where p() is the blob polynomial. BLS12 field element, rep. as BigNum in nr, bigint in ts. * proof: Buffer - KZG opening proof for y = p(z). The commitment to quotient polynomial Q, used in compressed BLS12 point format (48 bytes). */ - evaluate(challengeZ: Fr, verifyProof = false) { + async evaluate(challengeZ: Fr, verifyProof = false) { const kzg = getKzg(); - const res = kzg.computeKzgProof(this.data, challengeZ.toBuffer()); + const res = await kzg.asyncComputeKzgProof(this.data, challengeZ.toBuffer()); if (verifyProof && !kzg.verifyKzgProof(this.commitment, challengeZ.toBuffer(), res[1], res[0])) { throw new Error(`KZG proof did not verify.`); } diff --git a/yarn-project/blob-lib/src/blob_batching.test.ts b/yarn-project/blob-lib/src/blob_batching.test.ts index 1568be1ef2ea..11e2da774411 100644 --- a/yarn-project/blob-lib/src/blob_batching.test.ts +++ b/yarn-project/blob-lib/src/blob_batching.test.ts @@ -24,9 +24,9 @@ const trustedSetup = JSON.parse( ); describe('Blob Batching', () => { - it.each([10, 100, 400])('our BLS library should correctly commit to a blob of %p items', size => { + it.each([10, 100, 400])('our BLS library should correctly commit to a blob of %p items', async size => { const blobFields = [new Fr(size)].concat(Array.from({ length: size - 1 }).map((_, i) => new Fr(size + i))); - const ourBlob = Blob.fromFields(blobFields); + const ourBlob = await Blob.fromFields(blobFields); const point = BLS12Point.decompress(ourBlob.commitment); @@ -49,7 +49,7 @@ describe('Blob Batching', () => { it('should construct and verify 1 blob', async () => { // Initialize 400 fields. This test shows that a single blob works with batching methods. const blobFields = Array.from({ length: 400 }, (_, i) => new Fr(i + 123)); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); expect(blobs.length).toBe(1); const onlyBlob = blobs[0]; @@ -66,7 +66,7 @@ describe('Blob Batching', () => { const commitment = BLS12Point.decompress(onlyBlob.commitment); // 'Batched' evaluation - const { y, proof } = onlyBlob.evaluate(finalZ); + const { y, proof } = await onlyBlob.evaluate(finalZ); const q = BLS12Point.decompress(proof); const finalBlobCommitmentsHash = sha256ToField([onlyBlob.commitment]); @@ -134,7 +134,7 @@ describe('Blob Batching', () => { blobFields[numBlobFields - 1] = encodeCheckpointEndMarker({ numBlobFields }); } - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); expect(blobs.length).toBe(numBlobs); const finalChallenges = await BatchedBlobAccumulator.precomputeBatchedBlobChallenges([blobFields]); @@ -153,7 +153,7 @@ describe('Blob Batching', () => { // Batched evaluation // NB: we share the same finalZ between blobs - const proofObjects = blobs.map(b => b.evaluate(finalZ)); + const proofObjects = await Promise.all(blobs.map(b => b.evaluate(finalZ))); const evalYs = proofObjects.map(({ y }) => y); const qs = proofObjects.map(({ proof }) => BLS12Point.decompress(proof)); diff --git a/yarn-project/blob-lib/src/blob_batching.ts b/yarn-project/blob-lib/src/blob_batching.ts index 25b58416426a..d4f1b058622b 100644 --- a/yarn-project/blob-lib/src/blob_batching.ts +++ b/yarn-project/blob-lib/src/blob_batching.ts @@ -109,7 +109,7 @@ export class BatchedBlobAccumulator { for (const blobFields of blobFieldsPerCheckpoint) { // Compute the hash of all the fields in the block. const blobFieldsHash = await computeBlobFieldsHash(blobFields); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); for (const blob of blobs) { // Compute the challenge z for each blob and accumulate it. const challengeZ = await blob.computeChallengeZ(blobFieldsHash); @@ -126,7 +126,7 @@ export class BatchedBlobAccumulator { } // Now we have a shared challenge for all blobs, evaluate them... - const proofObjects = allBlobs.map(b => b.evaluate(z)); + const proofObjects = await Promise.all(allBlobs.map(b => b.evaluate(z))); const evaluations = await Promise.all(proofObjects.map(({ y }) => hashNoirBigNumLimbs(y))); // ...and find the challenge for the linear combination of blobs. let gamma = evaluations[0]; @@ -145,7 +145,7 @@ export class BatchedBlobAccumulator { * @returns An updated blob accumulator. */ async accumulateBlob(blob: Blob, blobFieldsHash: Fr) { - const { proof, y: thisY } = blob.evaluate(this.finalBlobChallenges.z); + const { proof, y: thisY } = await blob.evaluate(this.finalBlobChallenges.z); const thisC = BLS12Point.decompress(blob.commitment); const thisQ = BLS12Point.decompress(proof); const blobChallengeZ = await blob.computeChallengeZ(blobFieldsHash); @@ -192,7 +192,7 @@ export class BatchedBlobAccumulator { * @returns An updated blob accumulator. */ async accumulateFields(blobFields: Fr[]) { - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); if (blobs.length > BLOBS_PER_CHECKPOINT) { throw new Error( diff --git a/yarn-project/blob-lib/src/blob_utils.test.ts b/yarn-project/blob-lib/src/blob_utils.test.ts index ba7b9120d5b7..c1e5cbf0820d 100644 --- a/yarn-project/blob-lib/src/blob_utils.test.ts +++ b/yarn-project/blob-lib/src/blob_utils.test.ts @@ -7,33 +7,33 @@ import { makeCheckpointBlobData } from './encoding/fixtures.js'; import { BlobDeserializationError } from './errors.js'; describe('blob fields encoding', () => { - it('can process correct encoding for a single blob', () => { + it('can process correct encoding for a single blob', async () => { const checkpointBlobData = makeCheckpointBlobData(); const blobFields = encodeCheckpointBlobData(checkpointBlobData); expect(blobFields.length).toBeLessThan(FIELDS_PER_BLOB); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); expect(blobs.length).toBe(1); const decoded = decodeCheckpointBlobDataFromBlobs(blobs); expect(decoded).toEqual(checkpointBlobData); }); - it('can process correct encoding for multiple blobs', () => { + it('can process correct encoding for multiple blobs', async () => { const checkpointBlobData = makeCheckpointBlobData({ numBlocks: 2, numTxsPerBlock: 1, isFullTx: true }); const blobFields = encodeCheckpointBlobData(checkpointBlobData); expect(blobFields.length).toBeGreaterThan(FIELDS_PER_BLOB); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); expect(blobs.length).toBeGreaterThan(1); const decoded = decodeCheckpointBlobDataFromBlobs(blobs); expect(decoded).toEqual(checkpointBlobData); }); - it('throws processing random blob data', () => { + it('throws processing random blob data', async () => { const blobFields = Array.from({ length: 10 }, () => Fr.random()); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); expect(blobs.length).toBe(1); expect(() => decodeCheckpointBlobDataFromBlobs(blobs)).toThrow(BlobDeserializationError); diff --git a/yarn-project/blob-lib/src/blob_utils.ts b/yarn-project/blob-lib/src/blob_utils.ts index 14a380b4446b..d0d23804fe43 100644 --- a/yarn-project/blob-lib/src/blob_utils.ts +++ b/yarn-project/blob-lib/src/blob_utils.ts @@ -30,14 +30,16 @@ export function getPrefixedEthBlobCommitments(blobs: Blob[]): `0x${string}` { * * @throws If the number of fields does not match what's indicated by the checkpoint prefix. */ -export function getBlobsPerL1Block(fields: Fr[]): Blob[] { +export async function getBlobsPerL1Block(fields: Fr[]): Promise { if (!fields.length) { throw new Error('Cannot create blobs from empty fields.'); } const numBlobs = Math.ceil(fields.length / FIELDS_PER_BLOB); - return Array.from({ length: numBlobs }, (_, i) => - Blob.fromFields(fields.slice(i * FIELDS_PER_BLOB, (i + 1) * FIELDS_PER_BLOB)), + return await Promise.all( + Array.from({ length: numBlobs }, (_, i) => + Blob.fromFields(fields.slice(i * FIELDS_PER_BLOB, (i + 1) * FIELDS_PER_BLOB)), + ), ); } diff --git a/yarn-project/blob-lib/src/hash.ts b/yarn-project/blob-lib/src/hash.ts index 31f83e6b8524..a1ae9040fc88 100644 --- a/yarn-project/blob-lib/src/hash.ts +++ b/yarn-project/blob-lib/src/hash.ts @@ -44,12 +44,12 @@ export async function computeBlobFieldsHash(fields: Fr[]): Promise { return sponge.squeeze(); } -export function computeBlobCommitment(data: Uint8Array): Buffer { +export async function computeBlobCommitment(data: Uint8Array): Promise { if (data.length !== BYTES_PER_BLOB) { throw new Error(`Expected ${BYTES_PER_BLOB} bytes per blob. Got ${data.length}.`); } - return Buffer.from(getKzg().blobToKzgCommitment(data)); + return Buffer.from(await getKzg().asyncBlobToKzgCommitment(data)); } /** diff --git a/yarn-project/blob-lib/src/testing.ts b/yarn-project/blob-lib/src/testing.ts index 8b7bf19740cd..759f7da0790d 100644 --- a/yarn-project/blob-lib/src/testing.ts +++ b/yarn-project/blob-lib/src/testing.ts @@ -89,6 +89,6 @@ export function makeFinalBlobBatchingChallenges(seed = 1) { * @param length * @returns */ -export function makeRandomBlob(length: number): Blob { +export function makeRandomBlob(length: number): Promise { return Blob.fromFields([...Array.from({ length: length }, () => Fr.random())]); } diff --git a/yarn-project/cli/src/cmds/l1/update_l1_validators.ts b/yarn-project/cli/src/cmds/l1/update_l1_validators.ts index 6bb8ed4af8ac..6329dcdba50d 100644 --- a/yarn-project/cli/src/cmds/l1/update_l1_validators.ts +++ b/yarn-project/cli/src/cmds/l1/update_l1_validators.ts @@ -2,7 +2,7 @@ import { createEthereumChain, isAnvilTestChain } from '@aztec/ethereum/chain'; import { createExtendedL1Client, getPublicClient } from '@aztec/ethereum/client'; import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config'; import { GSEContract, RollupContract } from '@aztec/ethereum/contracts'; -import { createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { EthCheatCodes } from '@aztec/ethereum/test'; import type { EthAddress } from '@aztec/foundation/eth-address'; import type { LogFn, Logger } from '@aztec/foundation/log'; @@ -88,7 +88,7 @@ export async function addL1Validator({ const gse = new GSEContract(l1Client, gseAddress); const registrationTuple = await gse.makeRegistrationTuple(blsSecretKey); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); const proofParamsObj = ZkPassportProofParams.fromBuffer(proofParams); // Step 1: Claim STK tokens from the faucet @@ -194,7 +194,7 @@ export async function addL1ValidatorViaRollup({ const registrationTuple = await gse.makeRegistrationTuple(blsSecretKey); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); const { receipt } = await l1TxUtils.sendAndMonitorTransaction({ to: rollupAddress.toString(), @@ -241,7 +241,7 @@ export async function removeL1Validator({ const account = getAccount(privateKey, mnemonic); const chain = createEthereumChain(rpcUrls, chainId); const l1Client = createExtendedL1Client(rpcUrls, account, chain.chainInfo); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); dualLog(`Removing validator ${validatorAddress.toString()} from rollup ${rollupAddress.toString()}`); const { receipt } = await l1TxUtils.sendAndMonitorTransaction({ @@ -268,7 +268,7 @@ export async function pruneRollup({ const account = getAccount(privateKey, mnemonic); const chain = createEthereumChain(rpcUrls, chainId); const l1Client = createExtendedL1Client(rpcUrls, account, chain.chainInfo); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); dualLog(`Trying prune`); const { receipt } = await l1TxUtils.sendAndMonitorTransaction({ diff --git a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts index 9b6426aa45dd..1ce73ebffd1c 100644 --- a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts +++ b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts @@ -14,14 +14,17 @@ import type { Logger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; import { GovernanceProposerContract } from '@aztec/ethereum/contracts'; import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { Buffer32 } from '@aztec/foundation/buffer'; import { SecretValue } from '@aztec/foundation/config'; import { withLoggerBindings } from '@aztec/foundation/log/server'; import { retryUntil } from '@aztec/foundation/retry'; +import { sleep } from '@aztec/foundation/sleep'; import type { TestDateProvider } from '@aztec/foundation/timer'; import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; import { type AttestationInfo, getAttestationInfoFromPublishedCheckpoint } from '@aztec/stdlib/block'; -import { type DutyRow, DutyStatus } from '@aztec/validator-ha-signer/types'; +import { PostgresSlashingProtectionDatabase } from '@aztec/validator-ha-signer/db'; +import { type DutyRow, DutyStatus, DutyType } from '@aztec/validator-ha-signer/types'; import { jest } from '@jest/globals'; import { Pool } from 'pg'; @@ -132,7 +135,7 @@ describe('HA Full Setup', () => { prefilledPublicData, } = await setup(1, { initialValidators, - publisherPrivateKeys: [new SecretValue(publisherPrivateKeys[0])], + sequencerPublisherPrivateKeys: [new SecretValue(publisherPrivateKeys[0])], aztecTargetCommitteeSize: COMMITTEE_SIZE, minTxsPerBlock: 1, archiverPollingIntervalMS: 200, @@ -148,12 +151,7 @@ describe('HA Full Setup', () => { // Enable slashing for testing governance + slashing vote coordination slasherFlavor: 'tally', slashingRoundSizeInEpochs: 1, // 32 slots (1 epoch) - slashingQuorum: 17, // >50% of 32 slots for tally quorum - // Prover node will use publisherPrivateKeys directly, not Web3Signer - proverNodeConfig: { - web3SignerUrl: undefined, - publisherAddresses: undefined, - }, + slashingQuorum: 17, // >50% of 32 slots for tally quorum, })); logger.info(`Bootstrap node setup complete (validation disabled)`); @@ -200,10 +198,10 @@ describe('HA Full Setup', () => { bootstrapNodes: [bootstrapNodeEnr], web3SignerUrl, validatorAddresses: attesterAddresses.map(addr => EthAddress.fromString(addr)), - publisherAddresses: publisherAddresses.map(addr => EthAddress.fromString(addr)), + sequencerPublisherAddresses: publisherAddresses.map(addr => EthAddress.fromString(addr)), validatorPrivateKeys: new SecretValue(attesterPrivateKeys), // Each node has a unique publisher key - publisherPrivateKeys: [new SecretValue(publisherPrivateKeys[i])], + sequencerPublisherPrivateKeys: [new SecretValue(publisherPrivateKeys[i])], }; const nodeService = await withLoggerBindings({ actor: `HA-${i}` }, async () => { @@ -261,6 +259,9 @@ describe('HA Full Setup', () => { }); afterEach(async () => { + // Restore any mocked functions + jest.restoreAllMocks(); + // Clean up database state between tests try { await mainPool.query('DELETE FROM validator_duties'); @@ -671,4 +672,223 @@ describe('HA Full Setup', () => { } } }); + + describe('Clock Skew and Timezone Safety', () => { + const rollupAddress = EthAddress.random(); + const validatorAddress = EthAddress.random(); + it('should not be affected by process.env.TZ changes', async () => { + const spDb = new PostgresSlashingProtectionDatabase(mainPool); + const originalTZ = process.env.TZ; + + try { + // Node 1 in UTC creates and signs a duty + process.env.TZ = 'UTC'; + const duty1 = await spDb.tryInsertOrGetExisting({ + rollupAddress, + validatorAddress, + slot: SlotNumber(100), + blockNumber: BlockNumber(100), + dutyType: DutyType.ATTESTATION, + messageHash: Buffer32.random().toString(), + nodeId: 'node-utc', + }); + expect(duty1.isNew).toBe(true); + await spDb.updateDutySigned( + rollupAddress, + validatorAddress, + SlotNumber(100), + DutyType.ATTESTATION, + '0xsig', + duty1.record.lockToken, + -1, + ); + + await sleep(100); + + // Node 2 in Tokyo creates and signs a duty at approximately the same time + process.env.TZ = 'Asia/Tokyo'; + const duty2 = await spDb.tryInsertOrGetExisting({ + rollupAddress, + validatorAddress, + slot: SlotNumber(101), + blockNumber: BlockNumber(101), + dutyType: DutyType.ATTESTATION, + messageHash: Buffer32.random().toString(), + nodeId: 'node-tokyo', + }); + expect(duty2.isNew).toBe(true); + await spDb.updateDutySigned( + rollupAddress, + validatorAddress, + SlotNumber(101), + DutyType.ATTESTATION, + '0xsig', + duty2.record.lockToken, + -1, + ); + + // Verify both duties were stored at correct absolute times (seconds apart, not hours) + const result = await mainPool.query<{ slot: string; unix_timestamp: string }>( + `SELECT slot, EXTRACT(EPOCH FROM started_at) as unix_timestamp + FROM validator_duties + WHERE slot IN ('100', '101') + ORDER BY slot DESC`, + ); + + const timestamp1 = parseFloat(result.rows[0].unix_timestamp); + const timestamp2 = parseFloat(result.rows[1].unix_timestamp); + const diffSeconds = Math.abs(timestamp1 - timestamp2); + + // Should be less than 10 seconds apart (not hours due to timezone interpretation) + expect(diffSeconds).toBeLessThan(10); + } finally { + process.env.TZ = originalTZ; + } + }); + + it('should not delete recent duties when node clock is ahead (using cleanupOldDuties)', async () => { + const spDb = new PostgresSlashingProtectionDatabase(mainPool); + + // Ensure clean slate for this test + await mainPool.query('DELETE FROM validator_duties WHERE slot = $1', ['200']); + + // Create and sign a duty using our actual methods + const duty = await spDb.tryInsertOrGetExisting({ + rollupAddress, + validatorAddress, + slot: SlotNumber(200), + blockNumber: BlockNumber(200), + dutyType: DutyType.ATTESTATION, + messageHash: Buffer32.random().toString(), + nodeId: 'test-node', + }); + expect(duty.isNew).toBe(true); + + await spDb.updateDutySigned( + rollupAddress, + validatorAddress, + SlotNumber(200), + DutyType.ATTESTATION, + '0xsig', + duty.record.lockToken, + -1, + ); + + // Verify duty exists before cleanup + const beforeCleanup = await mainPool.query( + `SELECT * FROM validator_duties WHERE slot = $1 AND validator_address = $2`, + ['200', validatorAddress.toString().toLowerCase()], + ); + expect(beforeCleanup.rows.length).toBe(1); + expect(beforeCleanup.rows[0].status).toBe('signed'); + + // Simulate node with clock 2 hours ahead + const realNow = Date.now; + jest.spyOn(Date, 'now').mockImplementation(() => realNow() + 2 * 60 * 60 * 1000); + + // Use our actual cleanupOldDuties method + const numCleaned = await spDb.cleanupOldDuties(60 * 60 * 1000); // 1 hour + + // Should NOT delete the duty we just created (it uses DB's clock, not node's) + expect(numCleaned).toBe(0); + + // Verify duty still exists + const result = await mainPool.query( + `SELECT * FROM validator_duties WHERE slot = $1 AND validator_address = $2`, + ['200', validatorAddress.toString().toLowerCase()], + ); + expect(result.rows.length).toBe(1); + }); + + it('should delete old duties based on DB time, not node time (using cleanupOldDuties)', async () => { + const spDb = new PostgresSlashingProtectionDatabase(mainPool); + + // Ensure clean slate for this test + await mainPool.query('DELETE FROM validator_duties WHERE slot = $1', ['300']); + + // Create and sign a duty using our actual methods + const duty = await spDb.tryInsertOrGetExisting({ + rollupAddress, + validatorAddress, + slot: SlotNumber(300), + blockNumber: BlockNumber(300), + dutyType: DutyType.ATTESTATION, + messageHash: Buffer32.random().toString(), + nodeId: 'test-node', + }); + expect(duty.isNew).toBe(true); + + await spDb.updateDutySigned( + rollupAddress, + validatorAddress, + SlotNumber(300), + DutyType.ATTESTATION, + '0xsig', + duty.record.lockToken, + -1, + ); + + // Manually backdate the duty to 2 hours old (simulating an old duty from DB's perspective) + const updateResult = await mainPool.query( + `UPDATE validator_duties + SET started_at = CURRENT_TIMESTAMP - INTERVAL '2 hours', + completed_at = CURRENT_TIMESTAMP - INTERVAL '2 hours' + WHERE slot = $1 AND validator_address = $2`, + ['300', validatorAddress.toString().toLowerCase()], + ); + expect(updateResult.rowCount).toBe(1); + + // Verify duty is backdated (should be ~2 hours old) + const beforeCleanup = await mainPool.query( + `SELECT *, EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - started_at)) as age_seconds + FROM validator_duties WHERE slot = $1`, + ['300'], + ); + expect(beforeCleanup.rows.length).toBe(1); + expect(beforeCleanup.rows[0].status).toBe('signed'); + expect(parseFloat(beforeCleanup.rows[0].age_seconds)).toBeGreaterThan(7000); // ~2 hours in seconds + + // Simulate node with clock 1 hour behind + const realNow = Date.now; + jest.spyOn(Date, 'now').mockImplementation(() => realNow() - 1 * 60 * 60 * 1000); + + // Use our actual cleanupOldDuties method - should delete based on DB time + const numCleaned = await spDb.cleanupOldDuties(60 * 60 * 1000); // 1 hour + expect(numCleaned).toBeGreaterThanOrEqual(1); + + // Verify duty was deleted + const result = await mainPool.query( + `SELECT * FROM validator_duties WHERE slot = $1 AND validator_address = $2`, + ['300', validatorAddress.toString().toLowerCase()], + ); + expect(result.rows.length).toBe(0); + }); + + it('should not delete recent stuck duties when node clock is ahead (using cleanupOwnStuckDuties)', async () => { + const spDb = new PostgresSlashingProtectionDatabase(mainPool); + + // Create a signing duty (stuck, not completed) using our actual method + const duty = await spDb.tryInsertOrGetExisting({ + rollupAddress, + validatorAddress, + slot: SlotNumber(400), + blockNumber: BlockNumber(400), + dutyType: DutyType.ATTESTATION, + messageHash: Buffer32.random().toString(), + nodeId: 'stuck-node', + }); + expect(duty.isNew).toBe(true); + // Don't call updateDutySigned - leave it in 'signing' state (stuck) + + // Simulate node with clock 3 hours ahead + const realNow = Date.now; + jest.spyOn(Date, 'now').mockImplementation(() => realNow() + 3 * 60 * 60 * 1000); + + // Use our actual cleanupOwnStuckDuties method + const numCleaned = await spDb.cleanupOwnStuckDuties('stuck-node', 60 * 60 * 1000); // 1 hour + + // Should NOT delete the duty (it uses DB's clock, not node's) + expect(numCleaned).toBe(0); + }); + }); }); diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 9452b0dc2826..21aa382720ac 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -79,8 +79,7 @@ describe('e2e_block_building', () => { afterAll(() => teardown()); - // TODO(palla/mbps): We've seen these errors on syncing world state if we abort a tx processing halfway through. - it.skip('processes txs until hitting timetable', async () => { + it('processes txs until hitting timetable', async () => { // We send enough txs so they are spread across multiple blocks, but not // so many so that we don't end up hitting a reorg or timing out the tx wait(). const TX_COUNT = 16; diff --git a/yarn-project/end-to-end/src/e2e_debug_trace.test.ts b/yarn-project/end-to-end/src/e2e_debug_trace.test.ts index 6a61339bd083..71f950086660 100644 --- a/yarn-project/end-to-end/src/e2e_debug_trace.test.ts +++ b/yarn-project/end-to-end/src/e2e_debug_trace.test.ts @@ -1,7 +1,7 @@ import type { AztecNodeConfig } from '@aztec/aztec-node'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { FORWARDER_ABI, deployForwarderProxy } from '@aztec/ethereum/forwarder-proxy'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { PublisherManager } from '@aztec/ethereum/publisher-manager'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -79,7 +79,7 @@ describe('e2e_debug_trace_transaction', () => { // In this test we deploy a simple forwarder contract to L1, this serves as an additional proxy it('can process blocks using debug trace', async () => { // We intercept calls to sendAndMonitorTransaction to forward inner calls via the forwarder - const l1Utils: L1TxUtilsWithBlobs[] = (publisherManager as any).publishers; + const l1Utils: L1TxUtils[] = (publisherManager as any).publishers; // Intercept sendAndMonitorTransaction to access blobInputs directly const originalSendAndMonitor = l1Utils[0].sendAndMonitorTransaction.bind(l1Utils[0]); @@ -146,7 +146,7 @@ describe('e2e_debug_trace_transaction', () => { // 2. Duplicate the inner call to the rollup // 3. Corrupt the first call so it reverts (with allowFailure: true) // 4. Keep the second call intact so it succeeds - const l1Utils: L1TxUtilsWithBlobs[] = (publisherManager as any).publishers; + const l1Utils: L1TxUtils[] = (publisherManager as any).publishers; const originalSendAndMonitor = l1Utils[0].sendAndMonitorTransaction.bind(l1Utils[0]); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index 01ac7e29aec0..fa09091559b8 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -7,8 +7,8 @@ import { RollupContract } from '@aztec/ethereum/contracts'; import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { asyncMap } from '@aztec/foundation/async-map'; -import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; +import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { times, timesAsync } from '@aztec/foundation/collection'; import { SecretValue } from '@aztec/foundation/config'; import { EthAddress } from '@aztec/foundation/eth-address'; import { promiseWithResolvers } from '@aztec/foundation/promise'; @@ -16,14 +16,16 @@ import { retryUntil } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; import { timeoutPromise } from '@aztec/foundation/timer'; import { RollupAbi } from '@aztec/l1-artifacts'; -import type { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; +import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { OffenseType } from '@aztec/slasher'; import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; import { privateKeyToAccount } from 'viem/accounts'; +import { getAnvilPort } from '../fixtures/fixtures.js'; import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { proveInteraction } from '../test-wallet/utils.js'; import { EpochsTestContext } from './epochs_test.js'; jest.setTimeout(1000 * 60 * 10); @@ -31,17 +33,19 @@ jest.setTimeout(1000 * 60 * 10); const NODE_COUNT = 5; const VALIDATOR_COUNT = 5; +const BASE_ANVIL_PORT = getAnvilPort(); + describe('e2e_epochs/epochs_invalidate_block', () => { let context: EndToEndContext; let logger: Logger; let l1Client: ExtendedViemWalletClient; let rollupContract: RollupContract; - let anvilPort = 8545; + let anvilPortOffset = 0; let test: EpochsTestContext; let validators: (Operator & { privateKey: `0x${string}` })[]; let nodes: AztecNodeService[]; - let contract: SpamContract; + let testContract: TestContract; beforeEach(async () => { validators = times(VALIDATOR_COUNT, i => { @@ -51,8 +55,13 @@ describe('e2e_epochs/epochs_invalidate_block', () => { }); // Setup context with the given set of validators, mocked gossip sub network, and no anvil test watcher. + // Uses multiple-blocks-per-slot timing configuration. test = await EpochsTestContext.setup({ ethereumSlotDuration: 8, + aztecSlotDuration: 36, + blockDurationMs: 6000, + l1PublishingTime: 8, + enforceTimeTable: true, numberOfAccounts: 1, initialValidators: validators, mockGossipSubNetwork: true, @@ -62,11 +71,12 @@ describe('e2e_epochs/epochs_invalidate_block', () => { aztecTargetCommitteeSize: VALIDATOR_COUNT, archiverPollingIntervalMS: 200, anvilAccounts: 20, - anvilPort: ++anvilPort, + anvilPort: BASE_ANVIL_PORT + ++anvilPortOffset, slashingRoundSizeInEpochs: 4, slashingOffsetInRounds: 256, slasherFlavor: 'tally', minTxsPerBlock: 1, + maxTxsPerBlock: 1, }); ({ context, logger, l1Client } = test); @@ -88,8 +98,9 @@ describe('e2e_epochs/epochs_invalidate_block', () => { ); logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validatorNodes.map(v => v.attester) }); - // Register spam contract for sending txs. - contract = await test.registerSpamContract(context.wallet); + // Register test contract for lightweight txs + testContract = await test.registerTestContract(context.wallet); + logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); }); @@ -98,23 +109,29 @@ describe('e2e_epochs/epochs_invalidate_block', () => { await test.teardown(); }); - it('proposer invalidates previous block while posting its own', async () => { + it('proposer invalidates previous checkpoint with multiple blocks while posting its own', async () => { const sequencers = nodes.map(node => node.getSequencer()!); - const initialBlockNumber = await nodes[0].getBlockNumber(); + const [initialCheckpointNumber, initialBlockNumber] = await nodes[0] + .getL2Tips() + .then(t => [t.checkpointed.checkpoint.number, t.checkpointed.block.number] as const); // Configure all sequencers to skip collecting attestations before starting + // Also set minBlocksForCheckpoint to ensure multi-block checkpoints logger.warn('Configuring all sequencers to skip attestation collection'); sequencers.forEach(sequencer => { - sequencer.updateConfig({ skipCollectingAttestations: true }); + sequencer.updateConfig({ skipCollectingAttestations: true, minBlocksForCheckpoint: 2 }); }); - // Send a transaction so the sequencer builds a block - logger.warn('Sending transaction to trigger block building'); - const sentTx = await contract.methods.spam(1, 1n, false).send({ from: context.accounts[0], wait: NO_WAIT }); + // Send a few transactions so the sequencer builds multiple blocks in the checkpoint + // We'll later check that the first tx at least was picked up and mined + logger.warn('Sending multiple transactions to trigger block building'); + const [sentTx] = await timesAsync(8, i => + testContract.methods.emit_nullifier(BigInt(i + 1)).send({ from: context.accounts[0], wait: NO_WAIT }), + ); - // Disable skipCollectingAttestations after the first L2 block is mined - test.monitor.once('checkpoint', ({ checkpointNumber }) => { - logger.warn(`Disabling skipCollectingAttestations after L2 block ${checkpointNumber} has been mined`); + // Disable skipCollectingAttestations after the first checkpoint and capture its number + test.monitor.on('checkpoint', ({ checkpointNumber }) => { + logger.warn(`Disabling skipCollectingAttestations after checkpoint ${checkpointNumber} has been mined`); sequencers.forEach(sequencer => { sequencer.updateConfig({ skipCollectingAttestations: false }); }); @@ -133,8 +150,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { toBlock: 'latest', }); - // The next proposer should invalidate the previous block and publish a new one - logger.warn('Waiting for next proposer to invalidate the previous block'); + // The next proposer should invalidate the previous checkpoint and publish a new one + logger.warn('Waiting for next proposer to invalidate the previous checkpoint'); // Wait for the CheckpointInvalidated event const checkpointInvalidatedEvents = await retryUntil( @@ -150,10 +167,10 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Verify the CheckpointInvalidated event was emitted and that the block was removed const [event] = checkpointInvalidatedEvents; logger.warn(`CheckpointInvalidated event emitted`, { event }); - expect(event.args.checkpointNumber).toBeGreaterThan(initialBlockNumber); + expect(event.args.checkpointNumber).toBeGreaterThan(initialCheckpointNumber); expect(test.rollup.address).toEqual(event.address); - // Wait for all nodes to sync the new block + // Wait for all nodes to sync the new block proposed logger.warn('Waiting for all nodes to sync'); await retryUntil( async () => { @@ -167,7 +184,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { ); // Verify the transaction was eventually included - const receipt = await waitForTx(context.aztecNode, sentTx, { timeout: 30 }); + const receipt = await waitForTx(context.aztecNode, sentTx, { timeout: test.L2_SLOT_DURATION_IN_S * 8 }); expect(receipt.isMined()).toBeTrue(); logger.warn(`Transaction included in block ${receipt.blockNumber}`); @@ -177,15 +194,25 @@ describe('e2e_epochs/epochs_invalidate_block', () => { const invalidBlockOffense = offenses.find(o => o.offenseType === OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS); expect(invalidBlockOffense).toBeDefined(); + const currentCheckpoint = await test.rollup.getCheckpointNumber(); + + logger.warn('Sending further transactions to trigger more block building'); + await timesAsync(8, i => + testContract.methods.emit_nullifier(BigInt(i + 100)).send({ from: context.accounts[0], wait: NO_WAIT }), + ); + + logger.warn(`Waiting for checkpoint ${currentCheckpoint + 2} to be mined to ensure chain can progress`); + await test.waitUntilCheckpointNumber(CheckpointNumber(currentCheckpoint + 2), test.L2_SLOT_DURATION_IN_S * 8); + logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); - // Regression for an issue where, if the invalidator proposed another invalid block, the next proposer would + // Regression for an issue where, if the invalidator proposed another invalid checkpoint, the next proposer would // try invalidating the first one, which would fail due to mismatching attestations. For example: - // Slot S: Block N is proposed with invalid attestations - // Slot S+1: Block N is invalidated, and block N' (same number) is proposed instead, but also has invalid attestations - // Slot S+2: Proposer tries to invalidate block N, when they should invalidate block N' instead, and fails - it('chain progresses if a block with insufficient attestations is invalidated with an invalid one', async () => { + // Slot S: Checkpoint N is proposed with invalid attestations + // Slot S+1: Checkpoint N is invalidated, and checkpoint N' (same number) is proposed instead, but also has invalid attestations + // Slot S+2: Proposer tries to invalidate checkpoint N, when they should invalidate checkpoint N' instead, and fails + it('chain progresses if a checkpoint with insufficient attestations is invalidated with an invalid one', async () => { // Configure all sequencers to skip collecting attestations before starting and always build blocks logger.warn('Configuring all sequencers to skip attestation collection'); const sequencers = nodes.map(node => node.getSequencer()!); @@ -212,10 +239,16 @@ describe('e2e_epochs/epochs_invalidate_block', () => { }); await Promise.race([timeoutPromise(1000 * test.L2_SLOT_DURATION_IN_S * 8), invalidatePromise.promise]); - // Disable skipCollectingAttestations + // Disable skipCollectingAttestations and send txs so MBPS can produce multi-block checkpoints sequencers.forEach(sequencer => { sequencer.updateConfig({ skipCollectingAttestations: false }); }); + logger.warn('Sending transactions to enable multi-block checkpoints'); + const from = context.accounts[0]; + for (let i = 0; i < 4; i++) { + const tx = await proveInteraction(context.wallet, testContract.methods.emit_nullifier(new Fr(100 + i)), { from }); + await tx.send({ wait: NO_WAIT }); + } // Ensure chain progresses const targetCheckpointNumber = CheckpointNumber(lastInvalidatedCheckpointNumber! + 2); @@ -236,11 +269,13 @@ describe('e2e_epochs/epochs_invalidate_block', () => { 0.5, ); + await test.assertMultipleBlocksPerSlot(2); + logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); // Regression for Joe's Q42025 London attack. Same as above but with an invalid signature instead of insufficient ones. - it('chain progresses if a block with an invalid attestation is invalidated with an invalid one', async () => { + it('chain progresses if a checkpoint with an invalid attestation is invalidated with an invalid one', async () => { // Configure all sequencers to skip collecting attestations before starting and always build blocks logger.warn('Configuring all sequencers to inject one invalid attestation'); const sequencers = nodes.map(node => node.getSequencer()!); @@ -270,7 +305,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { invalidatePromise.promise, ]); - // Disable injectFakeAttestation + // Disable injectFakeAttestations sequencers.forEach(sequencer => { sequencer.updateConfig({ injectFakeAttestation: false }); }); @@ -297,11 +332,11 @@ describe('e2e_epochs/epochs_invalidate_block', () => { logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); - // Here we disable invalidation checks from two of the proposers. Our goal is to get two invalid blocks + // Here we disable invalidation checks from two of the proposers. Our goal is to get two invalid checkpoints // in a row, so the third proposer invalidates the earliest one, and the chain progresses. Note that the - // second invalid block will also have invalid attestations, we are *not* testing the scenario where the - // committee is malicious (or incompetent) and attests for the descendent of an invalid block. - it('proposer invalidates multiple blocks', async () => { + // second invalid checkpoint will also have invalid attestations, we are *not* testing the scenario where the + // committee is malicious (or incompetent) and attests for the descendent of an invalid checkpoint. + it('proposer invalidates multiple checkpoints', async () => { const initialSlot = (await test.monitor.run()).l2SlotNumber; // Disable validation and attestation gathering for the proposers of two consecutive slots @@ -406,9 +441,9 @@ describe('e2e_epochs/epochs_invalidate_block', () => { logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); - it('proposer invalidates previous block without publishing its own', async () => { + it('proposer invalidates previous checkpoint without publishing its own', async () => { const sequencers = nodes.map(node => node.getSequencer()!); - const initialBlockNumber = await nodes[0].getBlockNumber(); + const initialCheckpointNumber = (await nodes[0].getL2Tips()).checkpointed.checkpoint.number; // Configure all sequencers to skip collecting attestations before starting logger.warn('Configuring all sequencers to skip attestation collection and always publish blocks'); @@ -437,8 +472,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { toBlock: 'latest', }); - // The next proposer should invalidate the previous block - logger.warn('Waiting for next proposer to invalidate the previous block'); + // The next proposer should invalidate the previous checkpoint + logger.warn('Waiting for next proposer to invalidate the previous checkpoint'); // Wait for the CheckpointInvalidated event const checkpointInvalidatedEvents = await retryUntil( @@ -454,8 +489,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Verify the CheckpointInvalidated event was emitted and that the block was removed const [event] = checkpointInvalidatedEvents; logger.warn(`CheckpointInvalidated event emitted`, { event }); - expect(event.args.checkpointNumber).toBeGreaterThan(initialBlockNumber); - const initialCheckpointNumber = await getCheckpointNumberForBlock(nodes[0], initialBlockNumber); + expect(event.args.checkpointNumber).toBeGreaterThan(initialCheckpointNumber); expect(await test.rollup.getCheckpointNumber()).toEqual(initialCheckpointNumber); logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); @@ -465,7 +499,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // REFACTOR: Remove code duplication with above test (and others?) it('proposer invalidates previous block with shuffled attestations', async () => { const sequencers = nodes.map(node => node.getSequencer()!); - const initialBlockNumber = await nodes[0].getBlockNumber(); + const initialCheckpointNumber = (await nodes[0].getL2Tips()).checkpointed.checkpoint.number; // Configure all sequencers to shuffle attestations before starting logger.warn('Configuring all sequencers to shuffle attestations and always publish blocks'); @@ -494,8 +528,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { toBlock: 'latest', }); - // The next proposer should invalidate the previous block - logger.warn('Waiting for next proposer to invalidate the previous block'); + // The next proposer should invalidate the previous checkpoint + logger.warn('Waiting for next proposer to invalidate the previous checkpoint'); // Wait for the CheckpointInvalidated event const checkpointInvalidatedEvents = await retryUntil( @@ -511,8 +545,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Verify the CheckpointInvalidated event was emitted and that the block was removed const [event] = checkpointInvalidatedEvents; logger.warn(`CheckpointInvalidated event emitted`, { event }); - expect(event.args.checkpointNumber).toBeGreaterThan(initialBlockNumber); - const initialCheckpointNumber = await getCheckpointNumberForBlock(nodes[0], initialBlockNumber); + expect(event.args.checkpointNumber).toBeGreaterThan(initialCheckpointNumber); expect(await test.rollup.getCheckpointNumber()).toEqual(initialCheckpointNumber); logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); @@ -520,7 +553,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { it('committee member invalidates a block if proposer does not come through', async () => { const sequencers = nodes.map(node => node.getSequencer()!); - const initialBlockNumber = await nodes[0].getBlockNumber(); + const initialCheckpointNumber = await nodes[0].getL2Tips().then(t => t.checkpointed.checkpoint.number); // Configure all sequencers to skip collecting attestations before starting logger.warn('Configuring all sequencers to skip attestation collection and invalidation as proposer'); @@ -560,8 +593,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { toBlock: 'latest', }); - // Some committee member should invalidate the previous block - logger.warn('Waiting for committee member to invalidate the previous block'); + // Some committee member should invalidate the previous checkpoint + logger.warn('Waiting for committee member to invalidate the previous checkpoint'); // Wait for the CheckpointInvalidated event const checkpointInvalidatedEvents = await retryUntil( @@ -577,7 +610,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Verify the CheckpointInvalidated event was emitted const [event] = checkpointInvalidatedEvents; logger.warn(`CheckpointInvalidated event emitted`, { event }); - expect(event.args.checkpointNumber).toBeGreaterThan(initialBlockNumber); + expect(event.args.checkpointNumber).toBeGreaterThan(initialCheckpointNumber); // And check that the invalidation happened at least after the specified timeout. // We use the checkpoint header timestamp (L2 timestamp) since that's what the sequencer uses @@ -585,20 +618,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { const invalidSlotTimestamp = getTimestampForSlot(invalidCheckpointSlotNumber!, test.constants); const { timestamp: invalidationTimestamp } = await l1Client.getBlock({ blockNumber: event.blockNumber }); expect(invalidationTimestamp).toBeGreaterThanOrEqual(invalidSlotTimestamp + BigInt(invalidationDelay)); + logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); }); - -async function getCheckpointNumberForBlock( - node: AztecNodeService, - blockNumber: BlockNumber, -): Promise { - if (blockNumber === 0) { - return CheckpointNumber(0); - } - const block = await node.getBlock(blockNumber); - if (!block) { - throw new Error(`Block ${blockNumber} not found`); - } - return block.checkpointNumber; -} diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts similarity index 69% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.test.ts rename to yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts index 58bbab1d6806..f2f78d843213 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts @@ -1,12 +1,14 @@ import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import { AztecAddress } from '@aztec/aztec.js/addresses'; +import { NO_WAIT } from '@aztec/aztec.js/contracts'; import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; import { createBlobClient } from '@aztec/blob-client/client'; import { Blob } from '@aztec/blob-lib'; -import type { ChainMonitor, ChainMonitorEventMap, Delayer } from '@aztec/ethereum/test'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; +import type { ChainMonitor, ChainMonitorEventMap } from '@aztec/ethereum/test'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { CheckpointNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; @@ -14,7 +16,7 @@ import { AbortError } from '@aztec/foundation/error'; import { retryUntil } from '@aztec/foundation/retry'; import { hexToBuffer } from '@aztec/foundation/string'; import { executeTimeout } from '@aztec/foundation/timer'; -import type { ProverNode } from '@aztec/prover-node'; +import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { jest } from '@jest/globals'; import 'jest-extended'; @@ -22,16 +24,16 @@ import { keccak256, parseTransaction } from 'viem'; import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; import type { EndToEndContext } from '../fixtures/utils.js'; +import { proveInteraction } from '../test-wallet/utils.js'; import { EpochsTestContext } from './epochs_test.js'; -jest.setTimeout(1000 * 60 * 10); +jest.setTimeout(1000 * 60 * 20); describe('e2e_epochs/epochs_l1_reorgs', () => { let context: EndToEndContext; let logger: Logger; let node: AztecNode; let archiver: Archiver; - let proverNode: ProverNode; let monitor: ChainMonitor; let proverDelayer: Delayer; let sequencerDelayer: Delayer; @@ -40,18 +42,43 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { let L2_SLOT_DURATION_IN_S: number; let test: EpochsTestContext; + let contract: TestContract; + let from: AztecAddress; + + // Number of txs to send at the start of each blocks test to trigger multi-block checkpoints. + const TX_COUNT = 8; + + /** Pre-proves and sends txs to generate L2 activity for multi-block checkpoints. */ + const sendTransactions = async (count: number, offset = 0) => { + logger.warn(`Pre-proving ${count} transactions`); + const txs = await timesAsync(count, i => + proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(offset + i + 1)), { from }), + ); + const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); + logger.warn(`Sent ${txHashes.length} transactions`); + return txHashes; + }; beforeEach(async () => { test = await EpochsTestContext.setup({ + numberOfAccounts: 1, maxSpeedUpAttempts: 0, // Do not speed up l1 txs, we dont want them to land cancelTxOnTimeout: false, - aztecEpochDuration: 8, // Bump epoch duration, epoch 0 is finishing before we had a chance to do anything - ethereumSlotDuration: process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : 4, // Got to speed these tests up for CI + aztecEpochDuration: 4, + ethereumSlotDuration: 4, + aztecSlotDuration: 36, + blockDurationMs: 8000, + l1PublishingTime: 2, + minTxsPerBlock: 0, + maxTxsPerBlock: 1, + enforceTimeTable: true, + aztecProofSubmissionEpochs: 1, }); ({ proverDelayer, sequencerDelayer, context, logger, monitor, L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = test); node = context.aztecNode; archiver = (node as AztecNodeService).getBlockSource() as Archiver; - proverNode = context.proverNode!; + from = context.accounts[0]; + contract = await test.registerTestContract(context.wallet); }); afterEach(async () => { @@ -59,12 +86,12 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { }); describe('blocks', () => { - const getBlobs = (serializedTx: `0x${string}`) => { + const getBlobs = async (serializedTx: `0x${string}`) => { const parsedTx = parseTransaction(serializedTx); if (parsedTx.sidecars === false) { throw new Error('No sidecars found in tx'); } - return parsedTx.sidecars!.map(sidecar => Blob.fromBlobBuffer(hexToBuffer(sidecar.blob))); + return await Promise.all(parsedTx.sidecars!.map(sidecar => Blob.fromBlobBuffer(hexToBuffer(sidecar.blob)))); }; /** Returns the last synced checkpoint number for a node */ @@ -74,6 +101,12 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { const getProvenCheckpointNumber = (node: AztecNode) => node.getL2Tips().then(tips => tips.proven.checkpoint.number); it('prunes L2 blocks if a proof is removed due to an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + // Wait until we have proven something and the nodes have caught up const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; logger.warn(`Waiting for initial proof to land`); @@ -81,7 +114,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { signal => { return new Promise<{ provenCheckpointNumber: number; l1BlockNumber: number }>((res, rej) => { const handleMsg = (...[ev]: ChainMonitorEventMap['checkpoint-proven']) => { - if (ev.provenCheckpointNumber !== 0) { + if (ev.provenCheckpointNumber > initialProvenCheckpoint) { res(ev); monitor.off('checkpoint-proven', handleMsg); } @@ -97,21 +130,24 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { epochDurationSeconds * 4 * 1000, ); - // Stop the prover node so it doesn't re-submit the proof after we've removed it + // Stop the prover node (by stopping its hosting aztec node) so it doesn't re-submit the proof after we've removed it logger.warn(`Proof for block ${provenBlockEvent.provenCheckpointNumber} mined, stopping prover node`); - await proverNode.stop(); + await test.proverNodes[0].stop(); // And remove the proof from L1 await context.cheatCodes.eth.reorgTo(provenBlockEvent.l1BlockNumber - 1); - expect((await monitor.run(true)).provenCheckpointNumber).toEqual(0); + expect((await monitor.run(true)).provenCheckpointNumber).toEqual(initialProvenCheckpoint); - // Wait until the end of the proof submission window for the first epoch - await test.waitUntilLastSlotOfProofSubmissionWindow(0); + // Wait until the end of the proof submission window for the epoch of the proven checkpoint + const provenCheckpointEpoch = await test.rollup.getEpochNumberForCheckpoint( + CheckpointNumber(provenBlockEvent.provenCheckpointNumber), + ); + await test.waitUntilLastSlotOfProofSubmissionWindow(provenCheckpointEpoch); // Ensure that a new node sees the reorg logger.warn(`Syncing new node to test reorg`); const newNode = await executeTimeout(() => test.createNonValidatorNode(), 10_000, `new node sync`); - expect(await newNode.getProvenBlockNumber()).toEqual(0); + expect(await getProvenCheckpointNumber(newNode)).toEqual(initialProvenCheckpoint); // Latest checkpointed block seen by the node may be from the current checkpoint, or one less if it was *just* mined. // This is because the call to createNonValidatorNode will block until the initial sync is completed, @@ -122,65 +158,108 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { // And check that the old node has processed the reorg as well logger.warn(`Testing old node after reorg`); - await retryUntil(() => node.getProvenBlockNumber().then(b => b === 0), 'prune', L2_SLOT_DURATION_IN_S * 4, 0.1); + await retryUntil( + () => getProvenCheckpointNumber(node).then(cp => cp === initialProvenCheckpoint), + 'prune', + L2_SLOT_DURATION_IN_S * 4, + 0.1, + ); expect(await getCheckpointNumber(node)).toBeWithin(monitor.checkpointNumber - 1, monitor.checkpointNumber + 1); + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + logger.warn(`Test succeeded`); await newNode.stop(); }); it('does not prune if a second proof lands within the submission window after the first one is reorged out', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + const targetProvenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); + // Wait until we have proven something and the nodes have caught up + // Use a longer timeout since we need to wait for the epoch to complete (~288s) plus proving time + const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; logger.warn(`Waiting for initial proof to land`); - const provenCheckpoint = await test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)); - const provenBlock = Number(provenCheckpoint); - await retryUntil(() => node.getProvenBlockNumber().then(p => p >= provenBlock), 'node sync', 10, 0.1); + const provenCheckpoint = await test.waitUntilProvenCheckpointNumber( + targetProvenCheckpoint, + epochDurationSeconds * 2, + ); + await retryUntil(() => getProvenCheckpointNumber(node).then(cp => cp >= provenCheckpoint), 'node sync', 10, 0.1); - // Stop the prover node - await proverNode.stop(); + // Stop the prover node (by stopping its hosting aztec node) + await test.proverNodes[0].stop(); // Remove the proof from L1 but do not change the block number await context.cheatCodes.eth.reorgWithReplacement(1); - await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toEqual(0); + await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toEqual(initialProvenCheckpoint); // Create another prover node so it submits a proof and wait until it is submitted - const newProverNode = await test.createProverNode(); + await test.createProverNode(); const provenCheckpointRetry = await test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)); await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toBeGreaterThanOrEqual(1); // Check that the node has followed along logger.warn(`Testing old node`); - const provenBlockRetry = Number(provenCheckpointRetry); - await retryUntil(() => node.getProvenBlockNumber().then(b => b >= provenBlockRetry), 'proof sync', 10, 0.1); + await retryUntil( + () => getProvenCheckpointNumber(node).then(cp => cp >= provenCheckpointRetry), + 'proof sync', + 10, + 0.1, + ); expect(await getCheckpointNumber(node)).toBeWithin(monitor.checkpointNumber - 1, monitor.checkpointNumber + 1); + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + logger.warn(`Test succeeded`); - await newProverNode.stop(); + // New prover's aztec node is stopped in test.teardown() }); it('restores L2 blocks if a proof is added due to an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + const initialCheckpoint = monitor.checkpointNumber; + // Next proof shall not land proverDelayer.cancelNextTx(); // Expect pending chain to advance, so there's something to be pruned - await retryUntil(() => node.getBlockNumber().then(b => b > 1), 'node sync', 60, 0.1); + await retryUntil(() => getCheckpointNumber(node).then(cp => cp > initialCheckpoint), 'node sync', 60, 0.1); - // Wait until the end of the proof submission window for the first epoch - await test.waitUntilLastSlotOfProofSubmissionWindow(0); + // Wait until the end of the proof submission window for the first unproven epoch + const firstUnprovenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); + await test.waitUntilCheckpointNumber(firstUnprovenCheckpoint, 60); + const epochToWaitFor = await test.rollup.getEpochNumberForCheckpoint(firstUnprovenCheckpoint); + await test.waitUntilLastSlotOfProofSubmissionWindow(epochToWaitFor); await monitor.run(true); - logger.warn(`End of epoch 0 submission window (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`); + logger.warn( + `End of epoch ${epochToWaitFor} submission window (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`, + ); - // Grab the prover's tx to submit it later as part of a reorg and stop the prover + // Grab the prover's tx to submit it later as part of a reorg and stop the prover (by stopping its hosting aztec node) const [proofTx] = proverDelayer.getCancelledTxs(); expect(proofTx).toBeDefined(); - await proverNode.stop(); + await test.proverNodes[0].stop(); logger.warn(`Prover node stopped.`); // Wait for the node to prune const syncTimeout = L2_SLOT_DURATION_IN_S * 2; - await retryUntil(() => node.getBlockNumber().then(b => b <= 1), 'node prune', syncTimeout, 0.1); - expect(monitor.provenCheckpointNumber).toEqual(0); - expect(await node.getProvenBlockNumber()).toEqual(0); + await retryUntil( + () => getCheckpointNumber(node).then(cp => cp <= initialProvenCheckpoint + 1), + 'node prune', + syncTimeout, + 0.1, + ); + expect(monitor.provenCheckpointNumber).toEqual(initialProvenCheckpoint); + expect(await getProvenCheckpointNumber(node)).toEqual(initialProvenCheckpoint); // But not all is lost, for a reorg gets the proof back on chain! logger.warn(`Reorging proof back (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`); @@ -190,8 +269,8 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { // Monitor should update to see the proof const { checkpointNumber, provenCheckpointNumber } = await monitor.run(true); - expect(checkpointNumber).toBeGreaterThan(1); - expect(provenCheckpointNumber).toBeGreaterThan(0); + expect(checkpointNumber).toBeGreaterThan(initialCheckpoint); + expect(provenCheckpointNumber).toBeGreaterThan(initialProvenCheckpoint); // And so the node undoes its reorg await retryUntil( @@ -207,18 +286,30 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { 0.1, ); + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + logger.warn(`Test succeeded`); }); it('prunes blocks from pending chain removed from L1 due to an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialCheckpoint = (await monitor.run(true)).checkpointNumber; + // Wait until CHECKPOINT_NUMBER is mined and node synced, and stop the sequencer - const CHECKPOINT_NUMBER = CheckpointNumber(3); - await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * (CHECKPOINT_NUMBER + 4)); + const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); + await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 7); expect(monitor.checkpointNumber).toEqual(CHECKPOINT_NUMBER); const l1BlockNumber = monitor.l1BlockNumber; // Wait for node to sync to the checkpoint. await retryUntil(() => getCheckpointNumber(node).then(b => b === CHECKPOINT_NUMBER), 'node sync', 10, 0.1); + // Verify multi-block checkpoints were built before we do the reorg + await test.assertMultipleBlocksPerSlot(2); + logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}. Stopping block production.`); await context.aztecNodeAdmin.setConfig({ minTxsPerBlock: 100 }); @@ -236,14 +327,23 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { }); it('sees new blocks added in an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialCheckpoint = (await monitor.run(true)).checkpointNumber; + // Wait until the checkpoint *before* CHECKPOINT_NUMBER is mined and node synced - const CHECKPOINT_NUMBER = CheckpointNumber(3); + const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); const prevCheckpointNumber = CheckpointNumber(CHECKPOINT_NUMBER - 1); - await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * (CHECKPOINT_NUMBER + 4)); + await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * 7); expect(monitor.checkpointNumber).toEqual(prevCheckpointNumber); // Wait for node to sync to the checkpoint await retryUntil(() => getCheckpointNumber(node).then(b => b === prevCheckpointNumber), 'node sync', 5, 0.1); + // Verify multi-block checkpoints were built before we do the reorg + await test.assertMultipleBlocksPerSlot(2); + // Cancel the next tx to be mined and pause the sequencer sequencerDelayer.cancelNextTx(); await retryUntil(() => sequencerDelayer.getCancelledTxs().length, 'next block', L2_SLOT_DURATION_IN_S * 2, 0.1); @@ -283,7 +383,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { // We also need to send the blob to the sink, so the node can get it logger.warn(`Sending blobs to blob client`); - const blobs = getBlobs(l2BlockTx); + const blobs = await getBlobs(l2BlockTx); const blobClient = createBlobClient(context.config); await blobClient.sendBlobsToFilestore(blobs); @@ -307,6 +407,9 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { ); it('updates L1 to L2 messages changed due to an L1 reorg', async () => { + // Send L2 txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT, 100); + // Send 3 messages and wait for archiver sync logger.warn(`Sending 3 cross chain messages`); const msgs = await timesAsync(3, async (i: number) => { @@ -334,9 +437,16 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { await retryUntil(() => node.isL1ToL2MessageSynced(newMsg.msgHash), 'new message sync', L1_BLOCK_TIME_IN_S * 6, 1); expect(await node.isL1ToL2MessageSynced(msgs[0].msgHash)).toBe(true); expect(await node.isL1ToL2MessageSynced(msgs.at(-1)!.msgHash)).toBe(false); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); }); it('handles missed message inserted by an L1 reorg', async () => { + // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint + await sendTransactions(TX_COUNT, 200); + await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 4); + // Send a message and wait for node to sync it logger.warn(`Sending first cross chain message`); const firstMsg = await sendMessage(); @@ -368,6 +478,9 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { logger.warn(`Reorged-in second message on L1 block ${secondMsg.txReceipt.blockNumber}. Sending third message.`); const thirdMsg = await sendMessage(); await retryUntil(() => node.isL1ToL2MessageSynced(thirdMsg.msgHash), '3rd msg sync', L1_BLOCK_TIME_IN_S * 3, 1); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts index d640bea37cc6..ddf522d2c102 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts @@ -34,7 +34,7 @@ describe('e2e_epochs/epochs_long_proving_time', () => { await test.teardown(); }); - it('generates proof over multiple epochs', async () => { + it.skip('generates proof over multiple epochs', async () => { const targetProvenEpochs = process.env.TARGET_PROVEN_EPOCHS ? parseInt(process.env.TARGET_PROVEN_EPOCHS) : 1; const targetProvenBlockNumber = targetProvenEpochs * test.epochDuration; logger.info(`Waiting for ${targetProvenEpochs} epochs to be proven at ${targetProvenBlockNumber} L2 blocks`); @@ -42,7 +42,7 @@ describe('e2e_epochs/epochs_long_proving_time', () => { // Wait until we hit the target proven block number, and keep an eye on how many proving jobs are run in parallel. let maxJobCount = 0; while (monitor.provenCheckpointNumber === undefined || monitor.provenCheckpointNumber < targetProvenBlockNumber) { - const jobs = await test.proverNodes[0].getJobs(); + const jobs = await test.proverNodes[0].getProverNode()!.getJobs(); if (jobs.length > maxJobCount) { maxJobCount = jobs.length; logger.info(`Updated max job count to ${maxJobCount}`, jobs); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts index 1cbdd2bf2b15..1917f419e9f4 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts @@ -281,9 +281,18 @@ describe('e2e_epochs/epochs_mbps', () => { // Wait until all txs are mined const timeout = test.L2_SLOT_DURATION_IN_S * 5; - await Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))); + const receipts = await Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))); logger.warn(`All L2→L1 message txs have been mined`); + // wait for the other node to synch + const maxBlockNumber = Math.max(...receipts.map(r => r.blockNumber!)); + await retryUntil( + async () => ((await archiver.getCheckpointedL2BlockNumber()) >= maxBlockNumber ? true : undefined), + `archiver to checkpoint block ${maxBlockNumber}`, + test.L2_SLOT_DURATION_IN_S * 3, + 0.1, + ); + const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(EXPECTED_BLOCKS_PER_CHECKPOINT, logger); // Verify L2→L1 messages are in the blocks diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts index 4c4fb94806b2..b7a8e92fda53 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts @@ -44,8 +44,8 @@ describe('e2e_epochs/epochs_multi_proof', () => { // Add a delay to prover nodes so not all txs land on the same place // We apply patches BEFORE starting the prover nodes to ensure all provers get the delay // This prevents the race condition where multiple provers submit to L1 at the same time - test.proverNodes.forEach((prover, index) => { - const proverManager = prover.getProver(); + test.proverNodes.forEach((proverAztecNode, index) => { + const proverManager = proverAztecNode.getProverNode()!.getProver(); const origCreateEpochProver = proverManager.createEpochProver.bind(proverManager); proverManager.createEpochProver = () => { const epochProver = origCreateEpochProver(); @@ -62,9 +62,9 @@ describe('e2e_epochs/epochs_multi_proof', () => { }); // Now start all prover nodes after patches have been applied - await Promise.all(test.proverNodes.map(prover => prover.start())); + await Promise.all(test.proverNodes.map(node => node.getProverNode()!.start())); - const proverIds = test.proverNodes.map(prover => prover.getProverId()); + const proverIds = test.proverNodes.map(node => node.getProverNode()!.getProverId()); logger.info(`Prover nodes running with ids ${proverIds.map(id => id.toString()).join(', ')}`); // Wait until the start of epoch one and collect info on epoch zero diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts index d0f94c81e433..6703786b0f88 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts @@ -5,7 +5,7 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import { jest } from '@jest/globals'; import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext, WORLD_STATE_BLOCK_HISTORY } from './epochs_test.js'; +import { EpochsTestContext, WORLD_STATE_CHECKPOINT_HISTORY } from './epochs_test.js'; jest.setTimeout(1000 * 60 * 15); @@ -50,9 +50,10 @@ describe('e2e_epochs/epochs_multiple', () => { // Check that finalized blocks are purged from world state // Right now finalization means a checkpoint is two L2 epochs deep. If this rule changes then this test needs to be updated. + // This test is setup as 1 block per checkpoint const provenBlockNumber = epochEndBlockNumber; const finalizedBlockNumber = Math.max(provenBlockNumber - context.config.aztecEpochDuration * 2, 0); - const expectedOldestHistoricBlock = Math.max(finalizedBlockNumber - WORLD_STATE_BLOCK_HISTORY + 1, 1); + const expectedOldestHistoricBlock = Math.max(finalizedBlockNumber - WORLD_STATE_CHECKPOINT_HISTORY + 1, 1); const expectedBlockRemoved = expectedOldestHistoricBlock - 1; await test.waitForNodeToSync(BlockNumber(expectedOldestHistoricBlock), 'historic'); await test.verifyHistoricBlock(BlockNumber(expectedOldestHistoricBlock), true); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts index 86b2395f6406..bf87e55066d1 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts @@ -29,7 +29,7 @@ describe('e2e_epochs/epochs_partial_proof', () => { await test.waitUntilCheckpointNumber(CheckpointNumber(4), test.L2_SLOT_DURATION_IN_S * 6); logger.info(`Kicking off partial proof`); - await test.context.proverNode!.startProof(EpochNumber(0)); + await test.context.proverNode!.getProverNode()!.startProof(EpochNumber(0)); await retryUntil(() => monitor.provenCheckpointNumber > CheckpointNumber(0), 'proof', 120, 1); logger.info(`Test succeeded with proven checkpoint number ${monitor.provenCheckpointNumber}`); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts index 18c6209ef233..37481a05c154 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts @@ -2,12 +2,12 @@ import { getTimestampRangeForEpoch } from '@aztec/aztec.js/block'; import type { Logger } from '@aztec/aztec.js/log'; import { BatchedBlob } from '@aztec/blob-lib/types'; import { RollupContract } from '@aztec/ethereum/contracts'; -import { ChainMonitor, DelayedTxUtils, type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/test'; +import { type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; +import { ChainMonitor } from '@aztec/ethereum/test'; import type { ViemClient } from '@aztec/ethereum/types'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; -import type { ProverNodePublisher } from '@aztec/prover-node'; import type { TestProverNode } from '@aztec/prover-node/test'; import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; import { Proof } from '@aztec/stdlib/proofs'; @@ -56,13 +56,11 @@ describe('e2e_epochs/epochs_proof_fails', () => { // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1 // The proof will be rejected and a re-org will take place - // Ensure that there was at least one block mined in epoch 0, otherwise this test fails, since it + // Ensure that there was at least one checkpoint mined in epoch 0, otherwise this test fails, since it // relies on the proof for epoch zero not landing in time, which will never happen if there is - // nothing to prove on epoch zero. This is flakey because startup times change continuously. - // Also note that there should always be at least a checkpoint before we start since setup - // enforces it (search the comment "waiting for an empty block 1 to be mined" in `setup`). - const firstCheckpointNumber = (await test.monitor.run()).checkpointNumber; - expect(firstCheckpointNumber).toBeGreaterThanOrEqual(CheckpointNumber(1)); + // nothing to prove on epoch zero. We need to wait for the checkpoint L1 tx to be mined, not just + // for the block to appear in the node's world state, since the propose tx may still be in-flight. + await test.waitUntilCheckpointNumber(CheckpointNumber(1)); const firstCheckpoint = await rollup.getCheckpoint(CheckpointNumber(1)); const firstCheckpointEpoch = getEpochAtSlot(firstCheckpoint.slotNumber, test.constants); expect(firstCheckpointEpoch).toEqual(EpochNumber(0)); @@ -72,8 +70,7 @@ describe('e2e_epochs/epochs_proof_fails', () => { context.proverNode = proverNode; // Get the prover delayer from the newly created prover node - proverDelayer = (((proverNode as TestProverNode).publisher as ProverNodePublisher).l1TxUtils as DelayedTxUtils) - .delayer!; + proverDelayer = proverNode.getProverNode()!.getDelayer()!; // Hold off prover tx until end epoch 1 const [epoch2Start] = getTimestampRangeForEpoch(EpochNumber(2), constants); @@ -114,11 +111,11 @@ describe('e2e_epochs/epochs_proof_fails', () => { const proverNode = await test.createProverNode({ cancelTxOnTimeout: false, maxSpeedUpAttempts: 0 }); // Get the prover delayer from the newly created prover node - proverDelayer = (((proverNode as TestProverNode).publisher as ProverNodePublisher).l1TxUtils as DelayedTxUtils) - .delayer!; + const testProverNode = proverNode.getProverNode() as TestProverNode; + proverDelayer = testProverNode.getDelayer()!; // Inject a delay in prover node proving equal to the length of an epoch, to make sure deadline will be hit - const epochProverManager = (proverNode as TestProverNode).prover; + const epochProverManager = testProverNode.prover; const originalCreate = epochProverManager.createEpochProver.bind(epochProverManager); const finalizeEpochPromise = promiseWithResolvers(); let hasFinalizeEpochWaited = false; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts index 3c71f20c852b..846cd5f82b96 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts @@ -32,7 +32,7 @@ describe('e2e_epochs/epochs_proof_public_cross_chain', () => { numberOfAccounts: 1, minTxsPerBlock: 1, disableAnvilTestWatcher: true, - publisherAllowInvalidStates: true, + sequencerPublisherAllowInvalidStates: true, }); ({ context, logger } = test); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts index 4d0c64b7980e..a272fc521d94 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts @@ -1,3 +1,4 @@ +import type { Archiver } from '@aztec/archiver'; import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; import { getTimestampRangeForEpoch } from '@aztec/aztec.js/block'; import { getContractInstanceFromInstantiationParams } from '@aztec/aztec.js/contracts'; @@ -9,7 +10,8 @@ import { EpochCache } from '@aztec/epoch-cache'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { DefaultL1ContractsConfig } from '@aztec/ethereum/config'; import { RollupContract } from '@aztec/ethereum/contracts'; -import { ChainMonitor, DelayedTxUtils, type Delayer, waitUntilL1Timestamp, withDelayer } from '@aztec/ethereum/test'; +import { Delayer, createDelayer, waitUntilL1Timestamp, wrapClientWithDelayer } from '@aztec/ethereum/l1-tx-utils'; +import { ChainMonitor } from '@aztec/ethereum/test'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types'; import { SecretValue } from '@aztec/foundation/config'; @@ -20,16 +22,9 @@ import { sleep } from '@aztec/foundation/sleep'; import { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers'; -import { ProverNode, type ProverNodeConfig, ProverNodePublisher } from '@aztec/prover-node'; -import type { TestProverNode } from '@aztec/prover-node/test'; +import type { ProverNodeConfig } from '@aztec/prover-node'; import type { PXEConfig } from '@aztec/pxe/config'; -import { - type SequencerClient, - type SequencerEvents, - type SequencerPublisher, - SequencerState, -} from '@aztec/sequencer-client'; -import type { TestSequencerClient } from '@aztec/sequencer-client/test'; +import { type SequencerClient, type SequencerEvents, SequencerState } from '@aztec/sequencer-client'; import { type BlockParameter, EthAddress } from '@aztec/stdlib/block'; import { type L1RollupConstants, getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers'; import { tryStop } from '@aztec/stdlib/interfaces/server'; @@ -46,7 +41,7 @@ import { setup, } from '../fixtures/utils.js'; -export const WORLD_STATE_BLOCK_HISTORY = 2; +export const WORLD_STATE_CHECKPOINT_HISTORY = 2; export const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50; export const ARCHIVER_POLL_INTERVAL = 50; export const DEFAULT_L1_BLOCK_TIME = process.env.CI ? 12 : 8; @@ -81,7 +76,7 @@ export class EpochsTestContext { public proverDelayer!: Delayer; public sequencerDelayer!: Delayer; - public proverNodes: ProverNode[] = []; + public proverNodes: AztecNodeService[] = []; public nodes: AztecNodeService[] = []; public epochDuration!: number; @@ -147,7 +142,7 @@ export class EpochsTestContext { // using the prover's eth address if the proverId is used for something in the rollup contract // Use numeric EthAddress for deterministic prover id proverId: EthAddress.fromNumber(1), - worldStateBlockHistory: WORLD_STATE_BLOCK_HISTORY, + worldStateCheckpointHistory: WORLD_STATE_CHECKPOINT_HISTORY, exitDelaySeconds: DefaultL1ContractsConfig.exitDelaySeconds, slasherFlavor: 'none', l1PublishingTime, @@ -169,17 +164,8 @@ export class EpochsTestContext { // Loop that tracks L1 and L2 block numbers and logs whenever there's a new one. this.monitor = new ChainMonitor(this.rollup, context.dateProvider, this.logger).start(); - // This is hideous. - // We ought to have a definite reference to the l1TxUtils that we're using in both places, provided by the test context. - this.proverDelayer = context.proverNode - ? (((context.proverNode as TestProverNode).publisher as ProverNodePublisher).l1TxUtils as DelayedTxUtils).delayer! - : undefined!; - this.sequencerDelayer = context.sequencer - ? ( - ((context.sequencer as TestSequencerClient).sequencer.publisher as SequencerPublisher) - .l1TxUtils as DelayedTxUtils - ).delayer! - : undefined!; + this.proverDelayer = context.proverDelayer!; + this.sequencerDelayer = context.sequencerDelayer!; if ((context.proverNode && !this.proverDelayer) || (context.sequencer && !this.sequencerDelayer)) { throw new Error(`Could not find prover or sequencer delayer`); @@ -214,26 +200,29 @@ export class EpochsTestContext { const proverNodePrivateKey = this.getNextPrivateKey(); const proverIndex = this.proverNodes.length + 1; const { mockGossipSubNetwork } = this.context; - const proverNode = await withLoggerBindings({ actor: `prover-${proverIndex}` }, () => + const { proverNode } = await withLoggerBindings({ actor: `prover-${proverIndex}` }, () => createAndSyncProverNode( proverNodePrivateKey, { ...this.context.config, p2pEnabled: this.context.config.p2pEnabled || mockGossipSubNetwork !== undefined, - }, - { - dataDirectory: join(this.context.config.dataDirectory!, randomBytes(8).toString('hex')), proverId: EthAddress.fromNumber(proverIndex), dontStart: opts.dontStart, ...opts, }, - this.context.aztecNode, - this.context.prefilledPublicData ?? [], + { + dataDirectory: join(this.context.config.dataDirectory!, randomBytes(8).toString('hex')), + }, { dateProvider: this.context.dateProvider, - p2pClientDeps: mockGossipSubNetwork - ? { p2pServiceFactory: getMockPubSubP2PServiceFactory(mockGossipSubNetwork) } - : undefined, + p2pClientDeps: { + p2pServiceFactory: mockGossipSubNetwork ? getMockPubSubP2PServiceFactory(mockGossipSubNetwork) : undefined, + rpcTxProviders: [this.context.aztecNode], + }, + }, + { + prefilledPublicData: this.context.prefilledPublicData ?? [], + dontStart: opts.dontStart, }, ), ); @@ -248,15 +237,13 @@ export class EpochsTestContext { public createValidatorNode( privateKeys: `0x${string}`[], - opts: Partial & { txDelayerMaxInclusionTimeIntoSlot?: number; dontStartSequencer?: boolean } = {}, + opts: Partial & { dontStartSequencer?: boolean } = {}, ) { this.logger.warn('Creating and syncing a validator node...'); return this.createNode({ ...opts, disableValidator: false, validatorPrivateKeys: new SecretValue(privateKeys) }); } - private async createNode( - opts: Partial & { txDelayerMaxInclusionTimeIntoSlot?: number; dontStartSequencer?: boolean } = {}, - ) { + private async createNode(opts: Partial & { dontStartSequencer?: boolean } = {}) { const nodeIndex = this.nodes.length + 1; const actorPrefix = opts.disableValidator ? 'node' : 'validator'; const { mockGossipSubNetwork } = this.context; @@ -285,26 +272,6 @@ export class EpochsTestContext { ), ); - // REFACTOR: We're getting too much into the internals of the sequencer here. - // We should have a single method for constructing an aztec node that returns a TestAztecNodeService - // which directly exposes the delayer and sets any test config. - if (opts.txDelayerMaxInclusionTimeIntoSlot !== undefined) { - this.logger.info( - `Setting tx delayer max inclusion time into slot to ${opts.txDelayerMaxInclusionTimeIntoSlot} seconds`, - ); - // Here we reach into the sequencer and hook in a tx delayer. The problem is that the sequencer's l1 utils only uses a public client, not a wallet. - // The delayer needs a wallet (a client that can sign), so we have to create one here. - const l1Client = createExtendedL1Client( - resolvedConfig.l1RpcUrls!, - resolvedConfig.publisherPrivateKeys![0]!.getValue(), - ); - const sequencer = node.getSequencer() as TestSequencerClient; - const publisher = sequencer.sequencer.publisher; - const delayed = DelayedTxUtils.fromL1TxUtils(publisher.l1TxUtils, this.L1_BLOCK_TIME_IN_S, l1Client); - delayed.delayer!.setMaxInclusionTimeIntoSlot(opts.txDelayerMaxInclusionTimeIntoSlot); - publisher.l1TxUtils = delayed; - } - this.nodes.push(node); return node; } @@ -356,7 +323,10 @@ export class EpochsTestContext { this.logger.info(`Waiting until last slot of submission window for epoch ${epochNumber} at ${date}`, { oneSlotBefore, }); - await waitUntilL1Timestamp(this.l1Client, oneSlotBefore); + // Use a timeout that accounts for the full proof submission window + const proofSubmissionWindowDuration = + this.constants.proofSubmissionEpochs * this.epochDuration * this.L2_SLOT_DURATION_IN_S; + await waitUntilL1Timestamp(this.l1Client, oneSlotBefore, undefined, proofSubmissionWindowDuration * 2); } /** Waits for the aztec node to sync to the target block number. */ @@ -408,15 +378,13 @@ export class EpochsTestContext { /** Creates an L1 client using a fresh account with funds from anvil, with a tx delayer already set up. */ public async createL1Client() { - const { client, delayer } = withDelayer( - createExtendedL1Client( - [...this.l1Client.chain.rpcUrls.default.http], - privateKeyToAccount(this.getNextPrivateKey()), - this.l1Client.chain, - ), - this.context.dateProvider, - { ethereumSlotDuration: this.L1_BLOCK_TIME_IN_S }, + const rawClient = createExtendedL1Client( + [...this.l1Client.chain.rpcUrls.default.http], + privateKeyToAccount(this.getNextPrivateKey()), + this.l1Client.chain, ); + const delayer = createDelayer(this.context.dateProvider, { ethereumSlotDuration: this.L1_BLOCK_TIME_IN_S }, {}); + const client = wrapClientWithDelayer(rawClient, delayer); expect(await client.getBalance({ address: client.account.address })).toBeGreaterThan(0n); return { client, delayer }; } @@ -433,6 +401,38 @@ export class EpochsTestContext { expect(result).toBe(expectedSuccess); } + /** Verifies at least one checkpoint has the target number of blocks (for MBPS validation). */ + public async assertMultipleBlocksPerSlot(targetBlockCount: number) { + const archiver = (this.context.aztecNode as AztecNodeService).getBlockSource() as Archiver; + const checkpoints = await archiver.getCheckpoints(CheckpointNumber(1), 50); + + this.logger.warn(`Retrieved ${checkpoints.length} checkpoints from archiver`, { + checkpoints: checkpoints.map(pc => pc.checkpoint.getStats()), + }); + + let expectedBlockNumber = checkpoints[0].checkpoint.blocks[0].number; + let targetFound = false; + + for (const checkpoint of checkpoints) { + const blockCount = checkpoint.checkpoint.blocks.length; + targetFound = targetFound || blockCount >= targetBlockCount; + + this.logger.verbose(`Checkpoint ${checkpoint.checkpoint.number} has ${blockCount} blocks`, { + checkpoint: checkpoint.checkpoint.getStats(), + }); + + for (let i = 0; i < blockCount; i++) { + const block = checkpoint.checkpoint.blocks[i]; + expect(block.indexWithinCheckpoint).toBe(i); + expect(block.checkpointNumber).toBe(checkpoint.checkpoint.number); + expect(block.number).toBe(expectedBlockNumber); + expectedBlockNumber++; + } + } + + expect(targetFound).toBe(true); + } + public watchSequencerEvents( sequencers: SequencerClient[], getMetadata: (i: number) => Record = () => ({}), diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts index 096868366a5c..28c24b6a3133 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts @@ -52,7 +52,7 @@ describe('e2e_epochs/epochs_upload_failed_proof', () => { it('uploads failed proving job state and re-runs it on a fresh instance', async () => { // Make initial prover node fail to prove - const proverNode = test.proverNodes[0] as TestProverNode; + const proverNode = test.proverNodes[0].getProverNode() as TestProverNode; const proverManager = proverNode.getProver(); const origCreateEpochProver = proverManager.createEpochProver.bind(proverManager); proverManager.createEpochProver = () => { diff --git a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts index 593a096647f2..7eb5d7ab15a0 100644 --- a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts @@ -8,6 +8,7 @@ import { createBlobClient } from '@aztec/blob-client/client'; import { BatchedBlob, BatchedBlobAccumulator, + Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments, } from '@aztec/blob-lib'; @@ -24,8 +25,7 @@ import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config'; import { GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; import { type DeployAztecL1ContractsArgs, deployAztecL1Contracts } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; -import { TxUtilsState } from '@aztec/ethereum/l1-tx-utils'; -import { createL1TxUtilsWithBlobsFromViemWallet } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import { TxUtilsState, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { EthCheatCodesWithState, RollupCheatCodes, startAnvil } from '@aztec/ethereum/test'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { range } from '@aztec/foundation/array'; @@ -249,13 +249,17 @@ describe('L1Publisher integration', () => { const worldStateConfig: WorldStateConfig = { worldStateBlockCheckIntervalMS: 10000, worldStateDbMapSizeKb: 10 * 1024 * 1024, - worldStateBlockHistory: 0, + worldStateCheckpointHistory: 0, }; worldStateSynchronizer = new ServerWorldStateSynchronizer(builderDb, blockSource, worldStateConfig); await worldStateSynchronizer.start(); const sequencerL1Client = createExtendedL1Client(config.l1RpcUrls, sequencerPK, foundry); - const l1TxUtils = createL1TxUtilsWithBlobsFromViemWallet(sequencerL1Client, { logger, dateProvider }, config); + const l1TxUtils = createL1TxUtils( + sequencerL1Client, + { logger, dateProvider, kzg: Blob.getViemKzgInstance() }, + config, + ); const rollupContract = new RollupContract(sequencerL1Client, l1ContractAddresses.rollupAddress.toString()); const slashingProposerContract = await rollupContract.getSlashingProposer(); governanceProposerContract = new GovernanceProposerContract( @@ -268,12 +272,7 @@ describe('L1Publisher integration', () => { publisher = new SequencerPublisher( { - l1RpcUrls: config.l1RpcUrls, - l1DebugRpcUrls: [], - l1Contracts: l1ContractAddresses, - publisherPrivateKeys: [new SecretValue(sequencerPK)], l1ChainId: chainId, - viemPollingIntervalMS: 100, ethereumSlotDuration: config.ethereumSlotDuration, }, { @@ -350,6 +349,7 @@ describe('L1Publisher integration', () => { chainId: globalVariables.chainId, version: globalVariables.version, slotNumber: globalVariables.slotNumber, + timestamp: globalVariables.timestamp, coinbase: globalVariables.coinbase, feeRecipient: globalVariables.feeRecipient, gasFees: globalVariables.gasFees, @@ -457,7 +457,7 @@ describe('L1Publisher integration', () => { blockSource.getL1ToL2Messages.mockResolvedValueOnce(currentL1ToL2Messages); const checkpointBlobFields = checkpoint.toBlobFields(); - const blockBlobs = getBlobsPerL1Block(checkpointBlobFields); + const blockBlobs = await getBlobsPerL1Block(checkpointBlobFields); let prevBlobAccumulatorHash = (await rollup.getCurrentBlobCommitmentsHash()).toBuffer(); diff --git a/yarn-project/end-to-end/src/e2e_mempool_limit.test.ts b/yarn-project/end-to-end/src/e2e_mempool_limit.test.ts index a6649b39bfad..a158c05540f7 100644 --- a/yarn-project/end-to-end/src/e2e_mempool_limit.test.ts +++ b/yarn-project/end-to-end/src/e2e_mempool_limit.test.ts @@ -1,7 +1,6 @@ import { AztecAddress } from '@aztec/aztec.js/addresses'; import { NO_WAIT } from '@aztec/aztec.js/contracts'; import { TxStatus } from '@aztec/aztec.js/tx'; -import { retryUntil } from '@aztec/foundation/retry'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; import type { AztecNode, AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; @@ -70,24 +69,9 @@ describe('e2e_mempool_limit', () => { expect.objectContaining({ status: TxStatus.PENDING }), ); - const txHash3 = await tx3.send({ wait: NO_WAIT }); - - const txDropped = await retryUntil( - async () => { - // one of the txs will be dropped. Which one is picked is somewhat random because all three will have the same fee - const receipts = await Promise.all([ - aztecNode.getTxReceipt(txHash1), - aztecNode.getTxReceipt(txHash2), - aztecNode.getTxReceipt(txHash3), - ]); - const numPending = receipts.reduce((count, r) => (r.status === TxStatus.PENDING ? count + 1 : count), 0); - return numPending < 3; - }, - 'Waiting for one of the txs to be evicted from the mempool', - 60, - 1, - ); - - expect(txDropped).toBe(true); + // tx3 should be rejected because pool is at capacity and its priority is not higher than existing txs + await expect(tx3.send({ wait: NO_WAIT })).rejects.toMatchObject({ + data: { code: 'LOW_PRIORITY_FEE' }, + }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_multi_eoa.test.ts b/yarn-project/end-to-end/src/e2e_multi_eoa.test.ts index 7cce3650c9f9..02bad7926b51 100644 --- a/yarn-project/end-to-end/src/e2e_multi_eoa.test.ts +++ b/yarn-project/end-to-end/src/e2e_multi_eoa.test.ts @@ -4,7 +4,7 @@ import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; import { EthCheatCodes } from '@aztec/aztec/testing'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { PublisherManager } from '@aztec/ethereum/publisher-manager'; import type { ViemClient } from '@aztec/ethereum/types'; import { times } from '@aztec/foundation/collection'; @@ -80,7 +80,7 @@ describe('e2e_multi_eoa', () => { sequencerPollingIntervalMS: 200, worldStateBlockCheckIntervalMS: 200, blockCheckIntervalMS: 200, - publisherPrivateKeys: sequencerKeysAndAddresses.map(k => k.key), + sequencerPublisherPrivateKeys: sequencerKeysAndAddresses.map(k => k.key), l1PublisherKey: allKeysAndAddresses[0].key, maxSpeedUpAttempts: 0, // Disable speed ups, so that cancellation txs never make it through })); @@ -116,7 +116,7 @@ describe('e2e_multi_eoa', () => { from: defaultAccountAddress, }); - const l1Utils: L1TxUtilsWithBlobs[] = (publisherManager as any).publishers; + const l1Utils: L1TxUtils[] = (publisherManager as any).publishers; const blockedSender = l1Utils[expectedFirstSender].getSenderAddress(); const blockedTxs: Hex[] = []; diff --git a/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts b/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts index ab66bef2f868..35dccd29e152 100644 --- a/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts +++ b/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts @@ -79,7 +79,7 @@ describe('e2e_multi_validator_node', () => { } = await setup(1, { initialValidators, aztecTargetCommitteeSize: COMMITTEE_SIZE, - publisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)), + sequencerPublisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)), minTxsPerBlock: 1, archiverPollingIntervalMS: 200, sequencerPollingIntervalMS: 200, diff --git a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts index 5f5ad6e0f1d7..a1964ef8f74b 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts @@ -9,7 +9,7 @@ import { FeeAssetHandlerContract, RegistryContract, RollupContract } from '@azte import { deployRollupForUpgrade } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; -import { L1TxUtils, createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { retryUntil } from '@aztec/foundation/retry'; @@ -25,7 +25,6 @@ import { import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { protocolContractsHash } from '@aztec/protocol-contracts'; -import type { ProverNode } from '@aztec/prover-node'; import { getPXEConfig } from '@aztec/pxe/server'; import { computeL2ToL1MessageHash } from '@aztec/stdlib/hash'; import { tryStop } from '@aztec/stdlib/interfaces/server'; @@ -65,7 +64,7 @@ jest.setTimeout(1000 * 60 * 10); describe('e2e_p2p_add_rollup', () => { let t: P2PNetworkTest; let nodes: AztecNodeService[]; - let proverNode: ProverNode; + let proverAztecNode: AztecNodeService; let l1TxUtils: L1TxUtils; beforeAll(async () => { @@ -88,13 +87,13 @@ describe('e2e_p2p_add_rollup', () => { await t.applyBaseSetup(); await t.removeInitialNode(); - l1TxUtils = createL1TxUtilsFromViemWallet(t.ctx.deployL1ContractsValues.l1Client); + l1TxUtils = createL1TxUtils(t.ctx.deployL1ContractsValues.l1Client); t.ctx.watcher.setIsMarkingAsProven(false); }); afterAll(async () => { - await tryStop(proverNode); + await tryStop(proverAztecNode); await t.stopNodes(nodes); await t.teardown(); for (let i = 0; i < NUM_VALIDATORS; i++) { @@ -246,7 +245,7 @@ describe('e2e_p2p_add_rollup', () => { // create a prover node that uses p2p only (not rpc) to gather txs to test prover tx collection t.logger.warn(`Creating prover node`); - proverNode = await createProverNode( + ({ proverNode: proverAztecNode } = await createProverNode( t.ctx.aztecNodeConfig, BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, t.bootstrapNodeEnr, @@ -255,8 +254,7 @@ describe('e2e_p2p_add_rollup', () => { t.prefilledPublicData, `${DATA_DIR}-prover`, shouldCollectMetrics(), - ); - await proverNode.start(); + )); await sleep(4000); @@ -501,8 +499,8 @@ describe('e2e_p2p_add_rollup', () => { `Attesters new before: ${attestersBeforeNew.length}. Attesters new after: ${attestersAfterNew.length}`, ); - // Stop the prover node. - await proverNode.stop(); + // Stop the prover aztec node (which stops the prover subsystem). + await proverAztecNode.stop(); // stop all nodes for (let i = 0; i < NUM_VALIDATORS; i++) { @@ -561,7 +559,7 @@ describe('e2e_p2p_add_rollup', () => { ); t.logger.warn(`Creating new prover node`); - proverNode = await createProverNode( + ({ proverNode: proverAztecNode } = await createProverNode( newConfig, BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, t.bootstrapNodeEnr, @@ -570,8 +568,7 @@ describe('e2e_p2p_add_rollup', () => { prefilledPublicData, `${DATA_DIR_NEW}-prover`, shouldCollectMetrics(), - ); - await proverNode.start(); + )); // wait a bit for peers to discover each other await sleep(4000); diff --git a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts index 8b385c374b6e..6f43df3cefec 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts @@ -6,7 +6,6 @@ import { CheckpointNumber } from '@aztec/foundation/branded-types'; import { Signature } from '@aztec/foundation/eth-signature'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; -import type { ProverNode } from '@aztec/prover-node'; import type { SequencerClient } from '@aztec/sequencer-client'; import { tryStop } from '@aztec/stdlib/interfaces/server'; import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; @@ -47,7 +46,7 @@ const qosAlerts: AlertConfig[] = [ describe('e2e_p2p_network', () => { let t: P2PNetworkTest; let nodes: AztecNodeService[]; - let proverNode: ProverNode; + let proverNode: AztecNodeService; beforeEach(async () => { t = await P2PNetworkTest.create({ @@ -125,7 +124,7 @@ describe('e2e_p2p_network', () => { ); t.logger.warn(`Creating prover node`); - proverNode = await createProverNode( + ({ proverNode } = await createProverNode( { ...t.ctx.aztecNodeConfig, minTxsPerBlock: 0 }, BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, t.bootstrapNodeEnr, @@ -134,8 +133,7 @@ describe('e2e_p2p_network', () => { t.prefilledPublicData, `${DATA_DIR}-prover`, shouldCollectMetrics(), - ); - await proverNode.start(); + )); // wait a bit for peers to discover each other await sleep(8000); diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index 1978c217722b..ff561ecfff69 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -4,7 +4,6 @@ import { waitForTx } from '@aztec/aztec.js/node'; import { TxHash } from '@aztec/aztec.js/tx'; import { Signature } from '@aztec/foundation/eth-signature'; import { retryUntil } from '@aztec/foundation/retry'; -import type { ProverNode } from '@aztec/prover-node'; import type { SequencerClient } from '@aztec/sequencer-client'; import { tryStop } from '@aztec/stdlib/interfaces/server'; import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; @@ -49,7 +48,7 @@ const qosAlerts: AlertConfig[] = [ describe('e2e_p2p_network', () => { let t: P2PNetworkTest; let nodes: AztecNodeService[]; - let proverNode: ProverNode; + let proverAztecNode: AztecNodeService; let monitoringNode: AztecNodeService; beforeEach(async () => { @@ -75,7 +74,7 @@ describe('e2e_p2p_network', () => { }); afterEach(async () => { - await tryStop(proverNode); + await tryStop(proverAztecNode); await tryStop(monitoringNode); await t.stopNodes(nodes); await t.teardown(); @@ -119,7 +118,7 @@ describe('e2e_p2p_network', () => { // create a prover node that uses p2p only (not rpc) to gather txs to test prover tx collection t.logger.warn(`Creating prover node`); - proverNode = await createProverNode( + ({ proverNode: proverAztecNode } = await createProverNode( t.ctx.aztecNodeConfig, BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, t.bootstrapNodeEnr, @@ -128,8 +127,7 @@ describe('e2e_p2p_network', () => { t.prefilledPublicData, `${DATA_DIR}-prover`, shouldCollectMetrics(), - ); - await proverNode.start(); + )); t.logger.warn(`Creating non validator node`); const monitoringNodeConfig: AztecNodeConfig = { ...t.ctx.aztecNodeConfig, alwaysReexecuteBlockProposals: true }; diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts index 7dc521775561..8747f2b7251d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts @@ -10,17 +10,18 @@ import 'jest-extended'; import os from 'os'; import path from 'path'; +import { getBootNodeUdpPort } from '../fixtures/fixtures.js'; import { createNodes, createNonValidatorNode } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest } from './p2p_network.js'; const NUM_NODES = 2; const VALIDATORS_PER_NODE = 3; const NUM_VALIDATORS = NUM_NODES * VALIDATORS_PER_NODE; -const BOOT_NODE_UDP_PORT = 4500; +const BOOT_NODE_UDP_PORT = getBootNodeUdpPort(); const SLOT_COUNT = 3; const EPOCH_DURATION = 2; -const ETHEREUM_SLOT_DURATION = 4; -const AZTEC_SLOT_DURATION = 8; +const ETHEREUM_SLOT_DURATION = 8; +const AZTEC_SLOT_DURATION = 36; const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'validators-sentinel-')); @@ -46,6 +47,9 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { aztecTargetCommitteeSize: NUM_VALIDATORS, aztecSlotDuration: AZTEC_SLOT_DURATION, ethereumSlotDuration: ETHEREUM_SLOT_DURATION, + blockDurationMs: 6000, + l1PublishingTime: 8, + enforceTimeTable: true, aztecProofSubmissionEpochs: 1024, // effectively do not reorg listenAddress: '127.0.0.1', minTxsPerBlock: 0, diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 43d6213534fd..ed470053f9b0 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -12,7 +12,7 @@ import { import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; import { MultiAdderArtifact } from '@aztec/ethereum/l1-artifacts'; -import { createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { ChainMonitor } from '@aztec/ethereum/test'; import type { ExtendedViemWalletClient, ViemClient } from '@aztec/ethereum/types'; import { EpochNumber } from '@aztec/foundation/branded-types'; @@ -343,7 +343,7 @@ export class P2PNetworkTest { } private async _sendDummyTx(l1Client: ExtendedViemWalletClient) { - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client); + const l1TxUtils = createL1TxUtils(l1Client); return await l1TxUtils.sendAndMonitorTransaction({ to: l1Client.account!.address, value: 1n, diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp/utils.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp/utils.ts index b5860060ba5f..05fe3e3429be 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reqresp/utils.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reqresp/utils.ts @@ -3,24 +3,24 @@ import { createLogger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; import { Tx } from '@aztec/aztec.js/tx'; import { RollupContract } from '@aztec/ethereum/contracts'; -import { SlotNumber } from '@aztec/foundation/branded-types'; +import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; import { retryUntil } from '@aztec/foundation/retry'; -import { jest } from '@jest/globals'; +import { expect, jest } from '@jest/globals'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import { shouldCollectMetrics } from '../../fixtures/fixtures.js'; +import { getBootNodeUdpPort, shouldCollectMetrics } from '../../fixtures/fixtures.js'; import { createNodes } from '../../fixtures/setup_p2p_test.js'; -import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from '../p2p_network.js'; +import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from '../p2p_network.js'; import { prepareTransactions } from '../shared.js'; // Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds export const NUM_VALIDATORS = 6; -export const NUM_TXS_PER_NODE = 2; -export const BOOT_NODE_UDP_PORT = 4500; +export const NUM_TXS_PER_NODE = 4; +export const BOOT_NODE_UDP_PORT = getBootNodeUdpPort(); export const createReqrespDataDir = () => fs.mkdtempSync(path.join(os.tmpdir(), 'reqresp-')); @@ -38,8 +38,14 @@ export async function createReqrespTest(options: ReqrespOptions = {}): Promise

= 2; + + for (let i = 0; i < blockCount; i++) { + const block = published.checkpoint.blocks[i]; + expect(block.indexWithinCheckpoint).toBe(i); + expect(block.checkpointNumber).toBe(published.checkpoint.number); + expect(block.number).toBe(expectedBlockNumber); + expectedBlockNumber++; + } + } + + expect(mbpsFound).toBe(true); return nodes; } diff --git a/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts b/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts index f7cd63e5bfc0..532b94e0dd98 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts @@ -13,7 +13,7 @@ import { SlasherArtifact, TallySlashingProposerArtifact, } from '@aztec/ethereum/l1-artifacts'; -import { L1TxUtils, createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { tryJsonStringify } from '@aztec/foundation/json-rpc'; import { promiseWithResolvers } from '@aztec/foundation/promise'; @@ -115,7 +115,7 @@ describe('veto slash', () => { t.ctx.aztecNodeConfig.l1RpcUrls, bufferToHex(getPrivateKeyFromIndex(VETOER_PRIVATE_KEY_INDEX)!), ); - vetoerL1TxUtils = createL1TxUtilsFromViemWallet(vetoerL1Client, { + vetoerL1TxUtils = createL1TxUtils(vetoerL1Client, { logger: t.logger, dateProvider: t.ctx.dateProvider, }); @@ -199,7 +199,7 @@ describe('veto slash', () => { } debugLogger.info(`\n\ninitializing slasher with proposer: ${proposer}\n\n`); - const txUtils = createL1TxUtilsFromViemWallet(deployerClient, { + const txUtils = createL1TxUtils(deployerClient, { logger: t.logger, dateProvider: t.ctx.dateProvider, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts index 0eed2f499a95..d523ef582e5b 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts @@ -1,7 +1,7 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import { RollupContract } from '@aztec/ethereum/contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; -import { L1TxUtils, createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { SlotNumber } from '@aztec/foundation/branded-types'; import { sleep } from '@aztec/foundation/sleep'; import { @@ -60,7 +60,7 @@ describe('e2e_p2p_governance_proposer', () => { await t.setup(); await t.applyBaseSetup(); - l1TxUtils = createL1TxUtilsFromViemWallet(t.ctx.deployL1ContractsValues.l1Client); + l1TxUtils = createL1TxUtils(t.ctx.deployL1ContractsValues.l1Client); }); afterEach(async () => { diff --git a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts index 6349efaecb31..8683138cb02a 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts @@ -54,7 +54,7 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => { initialConfig: { enforceTimeTable: true, cancelTxOnTimeout: false, - publisherAllowInvalidStates: true, + sequencerPublisherAllowInvalidStates: true, listenAddress: '127.0.0.1', aztecEpochDuration, ethereumSlotDuration, diff --git a/yarn-project/end-to-end/src/e2e_pruned_blocks.test.ts b/yarn-project/end-to-end/src/e2e_pruned_blocks.test.ts index 65f1bd64dfad..50a8db649dd6 100644 --- a/yarn-project/end-to-end/src/e2e_pruned_blocks.test.ts +++ b/yarn-project/end-to-end/src/e2e_pruned_blocks.test.ts @@ -31,7 +31,7 @@ describe('e2e_pruned_blocks', () => { const MINT_AMOUNT = 1000n; // Don't make this value too high since we need to mine this number of empty blocks, which is relatively slow. - const WORLD_STATE_BLOCK_HISTORY = 2; + const WORLD_STATE_CHECKPOINT_HISTORY = 2; const EPOCH_LENGTH = 2; const WORLD_STATE_CHECK_INTERVAL_MS = 300; const ARCHIVER_POLLING_INTERVAL_MS = 300; @@ -47,7 +47,7 @@ describe('e2e_pruned_blocks', () => { accounts: [admin, sender, recipient], } = await setup(3, { aztecEpochDuration: EPOCH_LENGTH, - worldStateBlockHistory: WORLD_STATE_BLOCK_HISTORY, + worldStateCheckpointHistory: WORLD_STATE_CHECKPOINT_HISTORY, worldStateBlockCheckIntervalMS: WORLD_STATE_CHECK_INTERVAL_MS, archiverPollingIntervalMS: ARCHIVER_POLLING_INTERVAL_MS, aztecProofSubmissionEpochs: 1024, // effectively do not reorg @@ -93,8 +93,9 @@ describe('e2e_pruned_blocks', () => { // We now mine dummy blocks, mark them as proven and wait for the node to process them, which should result in older // blocks (notably the one with the minted note) being pruned. Given world state prunes based on the finalized tip, // and we are defining the finalized tip as two epochs behind the proven one, we need to mine two extra epochs. + // This test assumes 1 block per checkpoint await aztecNodeAdmin!.setConfig({ minTxsPerBlock: 0 }); - await waitBlocks(WORLD_STATE_BLOCK_HISTORY + EPOCH_LENGTH * 2 + 1); + await waitBlocks(WORLD_STATE_CHECKPOINT_HISTORY + EPOCH_LENGTH * 2 + 1); await cheatCodes.rollup.markAsProven(); // The same historical query we performed before should now fail since this block is not available anymore. We poll diff --git a/yarn-project/end-to-end/src/e2e_sequencer/reload_keystore.test.ts b/yarn-project/end-to-end/src/e2e_sequencer/reload_keystore.test.ts new file mode 100644 index 000000000000..09deed9db69d --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_sequencer/reload_keystore.test.ts @@ -0,0 +1,216 @@ +import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; +import { NO_WAIT } from '@aztec/aztec.js/contracts'; +import { ContractDeployer } from '@aztec/aztec.js/deployment'; +import { Fr } from '@aztec/aztec.js/fields'; +import { type AztecNode, waitForTx } from '@aztec/aztec.js/node'; +import type { Wallet } from '@aztec/aztec.js/wallet'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { SecretValue } from '@aztec/foundation/config'; +import type { EthPrivateKey } from '@aztec/node-keystore'; +import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; +import type { SequencerClient } from '@aztec/sequencer-client'; +import type { TestSequencer, TestSequencerClient } from '@aztec/sequencer-client/test'; +import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; +import type { ValidatorClient } from '@aztec/validator-client'; + +import { jest } from '@jest/globals'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { privateKeyToAccount } from 'viem/accounts'; + +import { getPrivateKeyFromIndex, setup } from '../fixtures/utils.js'; + +const VALIDATOR_KEY_INDICES = [0, 2, 4, 5]; +const PUBLISHER_KEY_INDEX = 3; + +// 4 validators staked on L1, committee size 4 → quorum = floor(4*2/3)+1 = 3. +// Only 3 validators are in the initial keystore (enough for quorum). +// After reload, the 4th validator is added. +const VALIDATOR_COUNT = 4; +const COMMITTEE_SIZE = VALIDATOR_COUNT; +const INITIAL_KEYSTORE_COUNT = 3; + +describe('e2e_reload_keystore', () => { + jest.setTimeout(300_000); + + let teardown: () => Promise; + let aztecNode: AztecNode; + let aztecNodeAdmin: AztecNodeAdmin | undefined; + let wallet: Wallet; + let ownerAddress: AztecAddress; + let keyStoreDirectory: string; + let sequencerClient: SequencerClient | undefined; + + const validatorKeys: EthPrivateKey[] = []; + const validatorAddresses: string[] = []; + let publisherKey: EthPrivateKey; + + const initialCoinbase = EthAddress.fromNumber(42); + const initialFeeRecipient = AztecAddress.fromNumber(42); + + const artifact = StatefulTestContractArtifact; + + beforeAll(async () => { + // Derive keys from the test mnemonic (these accounts are funded in Anvil) + for (const idx of VALIDATOR_KEY_INDICES) { + const key = `0x${getPrivateKeyFromIndex(idx)!.toString('hex')}` as EthPrivateKey; + validatorKeys.push(key); + validatorAddresses.push(privateKeyToAccount(key).address); + } + publisherKey = `0x${getPrivateKeyFromIndex(PUBLISHER_KEY_INDEX)!.toString('hex')}` as EthPrivateKey; + + // Create temp directory for keystore files + keyStoreDirectory = await mkdtemp(join(tmpdir(), 'reload-keystore-')); + + // Write initial keystore: first 3 validators only (validator 4 is deliberately excluded). + // All share the same coinbase X so we can detect a change after reload. + const initialKeystore = { + schemaVersion: 1, + validators: validatorKeys.slice(0, INITIAL_KEYSTORE_COUNT).map(key => ({ + attester: key, + coinbase: initialCoinbase.toChecksumString(), + publisher: [publisherKey], + feeRecipient: initialFeeRecipient.toString(), + })), + }; + await writeFile(join(keyStoreDirectory, 'keystore.json'), JSON.stringify(initialKeystore, null, 2)); + + // Stake ALL 4 validators on L1 so they are part of the committee + const initialValidators = validatorKeys.map((key, i) => ({ + attester: EthAddress.fromString(validatorAddresses[i]), + withdrawer: EthAddress.fromString(validatorAddresses[i]), + privateKey: key, + bn254SecretKey: new SecretValue(Fr.random().toBigInt()), + })); + + ({ + teardown, + aztecNode, + aztecNodeAdmin, + wallet, + accounts: [ownerAddress], + sequencer: sequencerClient, + } = await setup(1, { + initialValidators, + aztecTargetCommitteeSize: COMMITTEE_SIZE, + keyStoreDirectory, + minTxsPerBlock: 1, + maxTxsPerBlock: 1, + })); + + if (!aztecNodeAdmin) { + throw new Error('Aztec node admin API must be available for this test'); + } + }); + + afterAll(async () => { + await teardown(); + await rm(keyStoreDirectory, { recursive: true, force: true }); + }); + + it('should reload keystore, add a new validator, and use updated coinbase in blocks', async () => { + // Access the sequencer's validator client to inspect keystore state + const sequencer = (sequencerClient! as TestSequencerClient).getSequencer(); + const validatorClient: ValidatorClient = (sequencer as TestSequencer).validatorClient; + + // Verify initial keystore state and block production + // Only the first 3 validators should be loaded + const initialAddrs = validatorClient.getValidatorAddresses(); + expect(initialAddrs).toHaveLength(INITIAL_KEYSTORE_COUNT); + for (let i = 0; i < INITIAL_KEYSTORE_COUNT; i++) { + const attestor = EthAddress.fromString(validatorAddresses[i]); + expect(validatorClient.getCoinbaseForAttestor(attestor)).toEqual(initialCoinbase); + expect(validatorClient.getFeeRecipientForAttestor(attestor)).toEqual(initialFeeRecipient); + } + + // Validator 4 should NOT be in the keystore yet + const addr4Lower = validatorAddresses[3].toLowerCase(); + expect(initialAddrs.map(a => a.toString().toLowerCase())).not.toContain(addr4Lower); + + // Send a tx and verify the block uses the initial coinbase + const deployer = new ContractDeployer(artifact, wallet); + const sentTx1 = await deployer.deploy(ownerAddress, ownerAddress, 1).send({ + from: ownerAddress, + contractAddressSalt: new Fr(1), + skipClassPublication: true, + skipInstancePublication: true, + wait: NO_WAIT, + }); + const receipt1 = await waitForTx(aztecNode, sentTx1); + + const block1 = await aztecNode.getBlock(BlockNumber(receipt1.blockNumber!)); + expect(block1).toBeDefined(); + expect(block1!.header.globalVariables.coinbase.toString().toLowerCase()).toEqual( + initialCoinbase.toString().toLowerCase(), + ); + + // Write updated keystore and reload + // Each validator gets its own new coinbase so we can verify per-validator updates. + const newCoinbases = VALIDATOR_KEY_INDICES.map((_, i) => EthAddress.fromNumber(100 + i)); + const newFeeRecipients = VALIDATOR_KEY_INDICES.map((_, i) => AztecAddress.fromNumber(100 + i)); + + // Build updated keystore: all 4 validators (including the previously-excluded validator 4) + const updatedKeystore = { + schemaVersion: 1, + validators: validatorKeys.map((key, i) => ({ + attester: key, + coinbase: newCoinbases[i].toChecksumString(), + publisher: [publisherKey], + feeRecipient: newFeeRecipients[i].toString(), + })), + }; + await writeFile(join(keyStoreDirectory, 'keystore.json'), JSON.stringify(updatedKeystore, null, 2)); + + // Reload keystore via the admin API + await aztecNodeAdmin!.reloadKeystore(); + + // Verify the reload took effect + // All 4 validators should now be loaded + const updatedAddrs = validatorClient.getValidatorAddresses(); + expect(updatedAddrs).toHaveLength(VALIDATOR_COUNT); + + for (let i = 0; i < VALIDATOR_COUNT; i++) { + const attestor = EthAddress.fromString(validatorAddresses[i]); + expect(validatorClient.getCoinbaseForAttestor(attestor)).toEqual(newCoinbases[i]); + expect(validatorClient.getFeeRecipientForAttestor(attestor)).toEqual(newFeeRecipients[i]); + } + + // Specifically confirm validator 4 is now present + expect(updatedAddrs.map(a => a.toString().toLowerCase())).toContain(addr4Lower); + + // Deterministically prove validator 4 CAN publish blocks + // Directly ask the publisher factory to create a publisher for validator 4. + // This exercises the full chain: keystore lookup → publisher filter → L1 signer match. + // If the publisher key weren't in the L1TxUtils pool, this would throw. + const publisherFactory = (sequencer as TestSequencer).publisherFactory; + const validator4Attestor = EthAddress.fromString(validatorAddresses[3]); + const { attestorAddress: returnedAttestor, publisher: validator4Publisher } = + await publisherFactory.create(validator4Attestor); + + expect(returnedAttestor.equals(validator4Attestor)).toBe(true); + expect(validator4Publisher).toBeDefined(); + expect(validator4Publisher.getSenderAddress()).toBeDefined(); + + // Verify block production uses new coinbases (not old) + // Send a tx and confirm the block uses one of the new per-validator coinbases. + // Whichever validator is the proposer, its coinbase must be from the reloaded keystore. + const allNewCoinbasesLower = newCoinbases.map(c => c.toString().toLowerCase()); + + const sentTx2 = await deployer.deploy(ownerAddress, ownerAddress, 2).send({ + from: ownerAddress, + contractAddressSalt: new Fr(2), + skipClassPublication: true, + skipInstancePublication: true, + wait: NO_WAIT, + }); + const receipt2 = await waitForTx(aztecNode, sentTx2); + + const block2 = await aztecNode.getBlock(BlockNumber(receipt2.blockNumber!)); + expect(block2).toBeDefined(); + + const actualCoinbase = block2!.header.globalVariables.coinbase.toString().toLowerCase(); + expect(allNewCoinbasesLower).toContain(actualCoinbase); + expect(actualCoinbase).not.toEqual(initialCoinbase.toString().toLowerCase()); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts index cc77061328af..728a9d0e398b 100644 --- a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts +++ b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts @@ -10,14 +10,12 @@ import { tryRmDir } from '@aztec/foundation/fs'; import { logger } from '@aztec/foundation/log'; import { withLoggerBindings } from '@aztec/foundation/log/server'; import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { ProverNode, type ProverNodeConfig } from '@aztec/prover-node'; import { cp, mkdtemp, readFile, readdir, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; -import { type EndToEndContext, createAndSyncProverNode, getPrivateKeyFromIndex, setup } from './fixtures/utils.js'; +import { type EndToEndContext, setup } from './fixtures/utils.js'; const L1_BLOCK_TIME_IN_S = process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : 8; const L2_TARGET_BLOCK_NUM = 3; @@ -68,19 +66,7 @@ describe('e2e_snapshot_sync', () => { ); }; - const createTestProverNode = async (config: Partial = {}) => { - log.warn('Creating and syncing a prover node...'); - const dataDirectory = join(context.config.dataDirectory!, randomBytes(8).toString('hex')); - return await createAndSyncProverNode( - bufferToHex(getPrivateKeyFromIndex(5)!), - context.config, - { ...config, realProofs: false, dataDirectory }, - context.aztecNode, - context.prefilledPublicData ?? [], - ); - }; - - const expectNodeSyncedToL2Block = async (node: AztecNode | ProverNode, blockNumber: number) => { + const expectNodeSyncedToL2Block = async (node: AztecNode, blockNumber: number) => { const tips = await node.getL2Tips(); expect(tips.proposed.number).toBeGreaterThanOrEqual(blockNumber); const worldState = await node.getWorldStateSyncStatus(); @@ -123,17 +109,6 @@ describe('e2e_snapshot_sync', () => { await node.stop(); }); - it('downloads snapshot when syncing new prover node', async () => { - log.warn(`Syncing brand new prover node with snapshot sync`); - const node = await createTestProverNode({ snapshotsUrls: [snapshotLocation], syncMode: 'snapshot' }); - - log.warn(`New node prover synced`); - await expectNodeSyncedToL2Block(node, L2_TARGET_BLOCK_NUM); - - log.warn(`Stopping new prover node`); - await node.stop(); - }); - it('downloads snapshot from multiple sources', async () => { log.warn(`Setting up multiple snapshot locations with different L1 block heights`); diff --git a/yarn-project/end-to-end/src/e2e_synching.test.ts b/yarn-project/end-to-end/src/e2e_synching.test.ts index 75515681bf78..76a54cd05832 100644 --- a/yarn-project/end-to-end/src/e2e_synching.test.ts +++ b/yarn-project/end-to-end/src/e2e_synching.test.ts @@ -40,12 +40,12 @@ import { type Logger, createLogger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; import { AnvilTestWatcher } from '@aztec/aztec/testing'; import { createBlobClientWithFileStores } from '@aztec/blob-client/client'; +import { Blob } from '@aztec/blob-lib'; import { EpochCache } from '@aztec/epoch-cache'; import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config'; import { EmpireSlashingProposerContract, GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; -import { createL1TxUtilsWithBlobsFromViemWallet } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import { createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { CheckpointNumber } from '@aztec/foundation/branded-types'; -import { SecretValue } from '@aztec/foundation/config'; import { Signature } from '@aztec/foundation/eth-signature'; import { sleep } from '@aztec/foundation/sleep'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; @@ -66,7 +66,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { getContract } from 'viem'; import { mintTokensToPrivate } from './fixtures/token_utils.js'; -import { type EndToEndContext, getPrivateKeyFromIndex, setup, setupPXEAndGetWallet } from './fixtures/utils.js'; +import { type EndToEndContext, setup, setupPXEAndGetWallet } from './fixtures/utils.js'; import { TestWallet } from './test-wallet/test_wallet.js'; const AZTEC_GENERATE_TEST_DATA = !!process.env.AZTEC_GENERATE_TEST_DATA; @@ -407,11 +407,9 @@ describe('e2e_synching', () => { const blobClient = await createBlobClientWithFileStores(config, createLogger('test:blob-client:client')); - const sequencerPK: `0x${string}` = `0x${getPrivateKeyFromIndex(0)!.toString('hex')}`; - - const l1TxUtils = createL1TxUtilsWithBlobsFromViemWallet( + const l1TxUtils = createL1TxUtils( deployL1ContractsValues.l1Client, - { logger, dateProvider }, + { logger, dateProvider, kzg: Blob.getViemKzgInstance() }, config, ); const rollupAddress = deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(); @@ -434,12 +432,7 @@ describe('e2e_synching', () => { const sequencerPublisherMetrics: MockProxy = mock(); const publisher = new SequencerPublisher( { - l1RpcUrls: config.l1RpcUrls, - l1DebugRpcUrls: [], - l1Contracts: deployL1ContractsValues.l1ContractAddresses, - publisherPrivateKeys: [new SecretValue(sequencerPK)], l1ChainId: 31337, - viemPollingIntervalMS: 100, ethereumSlotDuration: ETHEREUM_SLOT_DURATION, }, { diff --git a/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts b/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts index d870f8c73231..6dad123c594a 100644 --- a/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts @@ -1,5 +1,5 @@ import type { InitialAccountData } from '@aztec/accounts/testing'; -import { type Archiver, createArchiver } from '@aztec/archiver'; +import { AztecNodeService } from '@aztec/aztec-node'; import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; import { type Logger, createLogger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; @@ -11,13 +11,11 @@ import { TestCircuitVerifier, } from '@aztec/bb-prover'; import { BackendType, Barretenberg } from '@aztec/bb.js'; -import { createBlobClientWithFileStores } from '@aztec/blob-client/client'; import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { Buffer32 } from '@aztec/foundation/buffer'; import { SecretValue } from '@aztec/foundation/config'; import { FeeAssetHandlerAbi } from '@aztec/l1-artifacts'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; -import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; import type { ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; import { getGenesisValues } from '@aztec/world-state/testing'; @@ -73,8 +71,8 @@ export class FullProverTest { circuitProofVerifier?: ClientProtocolCircuitVerifier; provenAsset!: TokenContract; context!: EndToEndContext; - private proverNode!: ProverNode; - private simulatedProverNode!: ProverNode; + private proverAztecNode!: AztecNodeService; + private simulatedProverAztecNode!: AztecNodeService; public l1Contracts!: DeployAztecL1ContractsReturnType; public proverAddress!: EthAddress; private minNumberOfTxsPerBlock: number; @@ -146,7 +144,7 @@ export class FullProverTest { // We don't wish to mark as proven automatically, so we set the flag to false this.context.watcher.setIsMarkingAsProven(false); - this.simulatedProverNode = this.context.proverNode!; + this.simulatedProverAztecNode = this.context.proverNode!; ({ aztecNode: this.aztecNode, deployL1ContractsValues: this.l1Contracts, @@ -155,7 +153,6 @@ export class FullProverTest { this.aztecNodeAdmin = this.context.aztecNodeService; const config = this.context.aztecNodeConfig; - const blobClient = await createBlobClientWithFileStores(config, this.logger); // Configure a full prover PXE let acvmConfig: Awaited> | undefined; @@ -217,20 +214,13 @@ export class FullProverTest { this.provenWallet = provenWallet; this.logger.info(`Full prover PXE started`); - // Shutdown the current, simulated prover node + // Shutdown the current, simulated prover node (by stopping its hosting aztec node) this.logger.verbose('Shutting down simulated prover node'); - await this.simulatedProverNode.stop(); - - // Creating temp store and archiver for fully proven prover node - this.logger.verbose('Starting archiver for new prover node'); - const archiver = await createArchiver( - { ...this.context.aztecNodeConfig, dataDirectory: undefined }, - { blobClient, dateProvider: this.context.dateProvider }, - { blockUntilSync: true }, - ); + await this.simulatedProverAztecNode.stop(); // The simulated prover node (now shutdown) used private key index 2 const proverNodePrivateKey = getPrivateKeyFromIndex(2); + const proverNodePrivateKeyHex = `0x${proverNodePrivateKey!.toString('hex')}` as const; const proverNodeSenderAddress = privateKeyToAddress(new Buffer32(proverNodePrivateKey!).toString()); this.proverAddress = EthAddress.fromString(proverNodeSenderAddress); @@ -238,14 +228,21 @@ export class FullProverTest { await this.mintFeeJuice(proverNodeSenderAddress); this.logger.verbose('Starting prover node'); - const proverConfig: ProverNodeConfig = { - ...this.context.aztecNodeConfig, - txCollectionNodeRpcUrls: [], + const sponsoredFPCAddress = await getSponsoredFPCAddress(); + const { prefilledPublicData } = await getGenesisValues( + this.context.initialFundedAccounts.map(a => a.address).concat(sponsoredFPCAddress), + ); + + const proverNodeConfig: Parameters[0] = { + ...config, + enableProverNode: true, + disableValidator: true, dataDirectory: undefined, + txCollectionNodeRpcUrls: [], proverId: this.proverAddress, realProofs: this.realProofs, proverAgentCount: 2, - publisherPrivateKeys: [new SecretValue(`0x${proverNodePrivateKey!.toString('hex')}` as const)], + proverPublisherPrivateKeys: [new SecretValue(proverNodePrivateKeyHex)], proverNodeMaxPendingJobs: 100, proverNodeMaxParallelBlocksPerEpoch: 32, proverNodePollingIntervalMs: 100, @@ -255,21 +252,14 @@ export class FullProverTest { txGatheringTimeoutMs: 24_000, proverNodeFailedEpochStore: undefined, proverNodeEpochProvingDelayMs: undefined, + validatorPrivateKeys: new SecretValue([]), }; - const sponsoredFPCAddress = await getSponsoredFPCAddress(); - const { prefilledPublicData } = await getGenesisValues( - this.context.initialFundedAccounts.map(a => a.address).concat(sponsoredFPCAddress), - ); - this.proverNode = await createProverNode( - proverConfig, - { - aztecNodeTxProvider: this.aztecNode, - archiver: archiver as Archiver, - }, + + this.proverAztecNode = await AztecNodeService.createAndSync( + proverNodeConfig, + { dateProvider: this.context.dateProvider, p2pClientDeps: { rpcTxProviders: [this.aztecNode] } }, { prefilledPublicData }, ); - await this.proverNode.start(); - this.logger.warn(`Proofs are now enabled`, { realProofs: this.realProofs }); return this; } @@ -289,8 +279,8 @@ export class FullProverTest { await this.provenComponents[i].teardown(); } - // clean up the full prover node - await this.proverNode.stop(); + // clean up the full prover node (by stopping its hosting aztec node) + await this.proverAztecNode.stop(); await Barretenberg.destroySingleton(); await this.bbConfigCleanup?.(); diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index edf72bd67b53..6b2002c6ffcd 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -7,6 +7,16 @@ export const shouldCollectMetrics = () => { return undefined; }; +/** Returns the boot node UDP port from environment variable or default value. */ +export function getBootNodeUdpPort(): number { + return process.env.BOOT_NODE_UDP_PORT ? parseInt(process.env.BOOT_NODE_UDP_PORT, 10) : 4500; +} + +/** Returns the anvil port from environment variable or default value. */ +export function getAnvilPort(): number { + return process.env.ANVIL_PORT ? parseInt(process.env.ANVIL_PORT, 10) : 8545; +} + export const TEST_PEER_CHECK_INTERVAL_MS = 1000; export const TEST_MAX_PENDING_TX_POOL_COUNT = 10_000; // Number of max pending TXs ~ 1.56GB diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index 3413bb82fc64..6def0cbc67f6 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -1,6 +1,5 @@ import { SchnorrAccountContractArtifact } from '@aztec/accounts/schnorr'; import { type InitialAccountData, generateSchnorrAccounts } from '@aztec/accounts/testing'; -import { type Archiver, createArchiver } from '@aztec/archiver'; import { type AztecNodeConfig, AztecNodeService, getConfigEnvVars } from '@aztec/aztec-node'; import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; import { @@ -16,7 +15,6 @@ import { type Logger, createLogger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; import type { Wallet } from '@aztec/aztec.js/wallet'; import { AnvilTestWatcher, CheatCodes } from '@aztec/aztec/testing'; -import { createBlobClientWithFileStores } from '@aztec/blob-client/client'; import { SPONSORED_FPC_SALT } from '@aztec/constants'; import { isAnvilTestChain } from '@aztec/ethereum/chain'; import { createExtendedL1Client } from '@aztec/ethereum/client'; @@ -30,13 +28,8 @@ import { type ZKPassportArgs, deployAztecL1Contracts, } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { - DelayedTxUtils, - EthCheatCodes, - EthCheatCodesWithState, - createDelayedL1TxUtilsFromViemWallet, - startAnvil, -} from '@aztec/ethereum/test'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; +import { EthCheatCodes, EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test'; import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; import { SecretValue } from '@aztec/foundation/config'; import { randomBytes } from '@aztec/foundation/crypto/random'; @@ -45,16 +38,14 @@ import { withLoggerBindings } from '@aztec/foundation/log/server'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { DateProvider, TestDateProvider } from '@aztec/foundation/timer'; -import type { DataStoreConfig } from '@aztec/kv-store/config'; import { SponsoredFPCContract } from '@aztec/noir-contracts.js/SponsoredFPC'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import type { P2PClientDeps } from '@aztec/p2p'; import { MockGossipSubNetwork, getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers'; import { protocolContractsHash } from '@aztec/protocol-contracts'; -import { type ProverNode, type ProverNodeConfig, type ProverNodeDeps, createProverNode } from '@aztec/prover-node'; +import type { ProverNodeConfig } from '@aztec/prover-node'; import { type PXEConfig, getPXEConfig } from '@aztec/pxe/server'; import type { SequencerClient } from '@aztec/sequencer-client'; -import type { TestSequencerClient } from '@aztec/sequencer-client/test'; import { type ContractInstanceWithAddress, getContractInstanceFromInstantiationParams } from '@aztec/stdlib/contract'; import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; import { tryStop } from '@aztec/stdlib/interfaces/server'; @@ -219,8 +210,8 @@ export type EndToEndContext = { aztecNodeService: AztecNodeService; /** Client to the Aztec Node admin interface. */ aztecNodeAdmin: AztecNodeAdmin; - /** The prover node service (only set if startProverNode is true) */ - proverNode: ProverNode | undefined; + /** The aztec node running the prover node subsystem (only set if startProverNode is true). */ + proverNode: AztecNodeService | undefined; /** A client to the sequencer service. */ sequencer: SequencerClient | undefined; /** Return values from deployAztecL1Contracts function. */ @@ -249,6 +240,10 @@ export type EndToEndContext = { telemetryClient: TelemetryClient; /** Mock gossip sub network used for gossipping messages (only if mockGossipSubNetwork was set to true in opts) */ mockGossipSubNetwork: MockGossipSubNetwork | undefined; + /** Delayer for sequencer L1 txs (only when enableDelayer is true). */ + sequencerDelayer: Delayer | undefined; + /** Delayer for prover node L1 txs (only when enableDelayer and startProverNode are true). */ + proverDelayer: Delayer | undefined; /** Prefilled public data used for setting up nodes. */ prefilledPublicData: PublicDataTreeLeaf[] | undefined; /** ACVM config (only set if running locally). */ @@ -288,6 +283,8 @@ export async function setup( config.realProofs = !!opts.realProofs; // Only enforce the time table if requested config.enforceTimeTable = !!opts.enforceTimeTable; + // Enable the tx delayer for tests (default config has it disabled, so we force-enable it here) + config.enableDelayer = true; config.listenAddress = '127.0.0.1'; config.minTxPoolAgeMs = opts.minTxPoolAgeMs ?? 0; @@ -339,11 +336,11 @@ export async function setup( publisherPrivKeyHex = opts.l1PublisherKey.getValue(); publisherHdAccount = privateKeyToAccount(publisherPrivKeyHex); } else if ( - config.publisherPrivateKeys && - config.publisherPrivateKeys.length > 0 && - config.publisherPrivateKeys[0].getValue() != NULL_KEY + config.sequencerPublisherPrivateKeys && + config.sequencerPublisherPrivateKeys.length > 0 && + config.sequencerPublisherPrivateKeys[0].getValue() != NULL_KEY ) { - publisherPrivKeyHex = config.publisherPrivateKeys[0].getValue(); + publisherPrivKeyHex = config.sequencerPublisherPrivateKeys[0].getValue(); publisherHdAccount = privateKeyToAccount(publisherPrivKeyHex); } else if (!MNEMONIC) { throw new Error(`Mnemonic not provided and no publisher private key`); @@ -352,7 +349,7 @@ export async function setup( const publisherPrivKeyRaw = publisherHdAccount.getHdKey().privateKey; const publisherPrivKey = publisherPrivKeyRaw === null ? null : Buffer.from(publisherPrivKeyRaw); publisherPrivKeyHex = `0x${publisherPrivKey!.toString('hex')}` as const; - config.publisherPrivateKeys = [new SecretValue(publisherPrivKeyHex)]; + config.sequencerPublisherPrivateKeys = [new SecretValue(publisherPrivKeyHex)]; } if (config.coinbase === undefined) { @@ -499,36 +496,33 @@ export async function setup( ); const sequencerClient = aztecNodeService.getSequencer(); - if (sequencerClient) { - const publisher = (sequencerClient as TestSequencerClient).sequencer.publisher; - publisher.l1TxUtils = DelayedTxUtils.fromL1TxUtils(publisher.l1TxUtils, config.ethereumSlotDuration, l1Client); - } - - let proverNode: ProverNode | undefined = undefined; + let proverNode: AztecNodeService | undefined = undefined; if (opts.startProverNode) { logger.verbose('Creating and syncing a simulated prover node...'); const proverNodePrivateKey = getPrivateKeyFromIndex(2); const proverNodePrivateKeyHex: Hex = `0x${proverNodePrivateKey!.toString('hex')}`; const proverNodeDataDirectory = path.join(directoryToCleanup, randomBytes(8).toString('hex')); - const proverNodeConfig = { - ...config.proverNodeConfig, - dataDirectory: proverNodeDataDirectory, - p2pEnabled: !!mockGossipSubNetwork, + + const p2pClientDeps: Partial> = { + p2pServiceFactory: mockGossipSubNetwork && getMockPubSubP2PServiceFactory(mockGossipSubNetwork!), + rpcTxProviders: [aztecNodeService], }; - proverNode = await createAndSyncProverNode( + + ({ proverNode } = await createAndSyncProverNode( proverNodePrivateKeyHex, config, - proverNodeConfig, - aztecNodeService, - prefilledPublicData, { - p2pClientDeps: mockGossipSubNetwork - ? { p2pServiceFactory: getMockPubSubP2PServiceFactory(mockGossipSubNetwork) } - : undefined, + ...config.proverNodeConfig, + dataDirectory: proverNodeDataDirectory, }, - ); + { dateProvider, p2pClientDeps, telemetry: telemetryClient }, + { prefilledPublicData }, + )); } + const sequencerDelayer = sequencerClient?.getDelayer(); + const proverDelayer = proverNode?.getProverNode()?.getDelayer(); + logger.verbose('Creating a pxe...'); const pxeConfig = { ...getPXEConfig(), ...pxeOpts }; pxeConfig.dataDirectory = path.join(directoryToCleanup, randomBytes(8).toString('hex')); @@ -629,6 +623,8 @@ export async function setup( mockGossipSubNetwork, prefilledPublicData, proverNode, + sequencerDelayer, + proverDelayer, sequencer: sequencerClient, teardown, telemetryClient, @@ -712,81 +708,42 @@ export async function waitForProvenChain(node: AztecNode, targetBlock?: BlockNum ); } +/** + * Creates an AztecNodeService with the prover node enabled as a subsystem. + * Returns both the aztec node service (for lifecycle management) and the prover node (for test internals access). + */ export function createAndSyncProverNode( proverNodePrivateKey: `0x${string}`, - aztecNodeConfig: AztecNodeConfig, - proverNodeConfig: Partial & Pick & { dontStart?: boolean }, - aztecNode: AztecNode | undefined, - prefilledPublicData: PublicDataTreeLeaf[] = [], - proverNodeDeps: ProverNodeDeps = {}, -) { + baseConfig: AztecNodeConfig, + configOverrides: Pick, + deps: { + telemetry?: TelemetryClient; + dateProvider: DateProvider; + p2pClientDeps?: P2PClientDeps; + }, + options: { prefilledPublicData: PublicDataTreeLeaf[]; dontStart?: boolean }, +): Promise<{ proverNode: AztecNodeService }> { return withLoggerBindings({ actor: 'prover-0' }, async () => { - const aztecNodeTxProvider = aztecNode && { - getTxByHash: aztecNode.getTxByHash.bind(aztecNode), - getTxsByHash: aztecNode.getTxsByHash.bind(aztecNode), - stop: () => Promise.resolve(), - }; - - const blobClient = await createBlobClientWithFileStores(aztecNodeConfig, createLogger('blob-client:prover-node')); - - const archiverConfig = { ...aztecNodeConfig, dataDirectory: proverNodeConfig.dataDirectory }; - const archiver = await createArchiver( - archiverConfig, - { blobClient, dateProvider: proverNodeDeps.dateProvider }, - { blockUntilSync: true }, - ); - - const proverConfig: ProverNodeConfig = { - ...aztecNodeConfig, - txCollectionNodeRpcUrls: [], - realProofs: false, - proverAgentCount: 2, - publisherPrivateKeys: [new SecretValue(proverNodePrivateKey)], - proverNodeMaxPendingJobs: 10, - proverNodeMaxParallelBlocksPerEpoch: 32, - proverNodePollingIntervalMs: 200, - txGatheringIntervalMs: 1000, - txGatheringBatchSize: 10, - txGatheringMaxParallelRequestsPerNode: 10, - txGatheringTimeoutMs: 24_000, - proverNodeFailedEpochStore: undefined, - proverId: EthAddress.fromNumber(1), - proverNodeEpochProvingDelayMs: undefined, - ...proverNodeConfig, - }; - - const l1TxUtils = createDelayedL1TxUtils( - aztecNodeConfig, - proverNodePrivateKey, - 'prover-node', - proverNodeDeps.dateProvider, + const proverNode = await AztecNodeService.createAndSync( + { + ...baseConfig, + ...configOverrides, + p2pPort: 0, + enableProverNode: true, + disableValidator: true, + proverPublisherPrivateKeys: [new SecretValue(proverNodePrivateKey)], + }, + deps, + { ...options, dontStartProverNode: options.dontStart }, ); - const proverNode = await createProverNode( - proverConfig, - { ...proverNodeDeps, aztecNodeTxProvider, archiver: archiver as Archiver, l1TxUtils }, - { prefilledPublicData }, - ); - getLogger().info(`Created and synced prover node`, { publisherAddress: l1TxUtils.client.account!.address }); - if (!proverNodeConfig.dontStart) { - await proverNode.start(); + if (!proverNode.getProverNode()) { + throw new Error('Prover node subsystem was not created despite enableProverNode being set'); } - return proverNode; - }); -} -function createDelayedL1TxUtils( - aztecNodeConfig: AztecNodeConfig, - privateKey: `0x${string}`, - logName: string, - dateProvider?: DateProvider, -) { - const l1Client = createExtendedL1Client(aztecNodeConfig.l1RpcUrls, privateKey, foundry); - - const log = createLogger(logName); - const l1TxUtils = createDelayedL1TxUtilsFromViemWallet(l1Client, log, dateProvider, aztecNodeConfig); - l1TxUtils.enableDelayer(aztecNodeConfig.ethereumSlotDuration); - return l1TxUtils; + getLogger().info(`Created and synced prover node`); + return { proverNode }; + }); } export type BalancesFn = ReturnType; diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 925bb76dd86d..5c1ed24a68fd 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -7,7 +7,6 @@ import { SecretValue } from '@aztec/foundation/config'; import { withLoggerBindings } from '@aztec/foundation/log/server'; import { bufferToHex } from '@aztec/foundation/string'; import type { DateProvider } from '@aztec/foundation/timer'; -import type { ProverNodeConfig, ProverNodeDeps } from '@aztec/prover-node'; import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees'; import getPort from 'get-port'; @@ -131,7 +130,7 @@ export async function createNonValidatorNode( ...p2pConfig, disableValidator: true, validatorPrivateKeys: undefined, - publisherPrivateKeys: [], + sequencerPublisherPrivateKeys: [], }; const telemetry = await getEndToEndTestTelemetryClient(metricsPort); return await AztecNodeService.createAndSync(config, { telemetry, dateProvider }, { prefilledPublicData }); @@ -143,31 +142,24 @@ export async function createProverNode( tcpPort: number, bootstrapNode: string | undefined, addressIndex: number, - proverNodeDeps: ProverNodeDeps & Required>, + deps: { dateProvider: DateProvider }, prefilledPublicData?: PublicDataTreeLeaf[], dataDirectory?: string, metricsPort?: number, -) { +): Promise<{ proverNode: AztecNodeService }> { const actorIndex = proverCounter++; return await withLoggerBindings({ actor: `prover-${actorIndex}` }, async () => { const proverNodePrivateKey = getPrivateKeyFromIndex(ATTESTER_PRIVATE_KEYS_START_INDEX + addressIndex)!; const telemetry = await getEndToEndTestTelemetryClient(metricsPort); - const proverConfig: Partial = await createP2PConfig( - config, - bootstrapNode, - tcpPort, - dataDirectory, - ); + const p2pConfig = await createP2PConfig(config, bootstrapNode, tcpPort, dataDirectory); - const aztecNodeRpcTxProvider = undefined; return await createAndSyncProverNode( bufferToHex(proverNodePrivateKey), - config, - { ...proverConfig, dataDirectory }, - aztecNodeRpcTxProvider, - prefilledPublicData, - { ...proverNodeDeps, telemetry }, + { ...config, ...p2pConfig }, + { dataDirectory }, + { ...deps, telemetry }, + { prefilledPublicData: prefilledPublicData ?? [] }, ); }); } @@ -215,7 +207,7 @@ export async function createValidatorConfig( ...config, ...p2pConfig, validatorPrivateKeys: new SecretValue(attesterPrivateKeys), - publisherPrivateKeys: [new SecretValue(attesterPrivateKeys[0])], + sequencerPublisherPrivateKeys: [new SecretValue(attesterPrivateKeys[0])], }; return nodeConfig; diff --git a/yarn-project/end-to-end/src/spartan/n_tps.test.ts b/yarn-project/end-to-end/src/spartan/n_tps.test.ts index 8e8e022171de..95405e1b9e84 100644 --- a/yarn-project/end-to-end/src/spartan/n_tps.test.ts +++ b/yarn-project/end-to-end/src/spartan/n_tps.test.ts @@ -345,7 +345,9 @@ describe('sustained N TPS test', () => { let lowValueTxs = 0; const lowValueSendTx = async (wallet: TestWallet) => { lowValueTxs++; - const feeAmount = Number(randomBigInt(10n)) + 1; + //const feeAmount = Number(randomBigInt(100n)) + 1; + //const feeAmount = 1; + const feeAmount = Math.floor(lowValueTxs / 1000) + 1; const fee = new GasFees(0, feeAmount); logger.info('Sending low value tx ' + lowValueTxs + ' with fee ' + feeAmount); @@ -358,7 +360,7 @@ describe('sustained N TPS test', () => { let highValueTxs = 0; const highValueSendTx = async (wallet: TestWallet) => { highValueTxs++; - const feeAmount = Number(randomBigInt(10n)) + 11; + const feeAmount = Number(randomBigInt(10n)) + 1000; const fee = new GasFees(0, feeAmount); logger.info('Sending high value tx ' + highValueTxs + ' with fee ' + feeAmount); diff --git a/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts b/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts index fe2ffc7dd233..5c1750c41b24 100644 --- a/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts +++ b/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts @@ -1,6 +1,9 @@ -import { NO_WAIT } from '@aztec/aztec.js/contracts'; +import { SchnorrAccountContract } from '@aztec/accounts/schnorr'; +import { AztecAddress } from '@aztec/aztec.js/addresses'; +import { toSendOptions } from '@aztec/aztec.js/contracts'; import { SponsoredFeePaymentMethod } from '@aztec/aztec.js/fee'; import { type AztecNode, createAztecNodeClient } from '@aztec/aztec.js/node'; +import { AccountManager } from '@aztec/aztec.js/wallet'; import { RollupCheatCodes } from '@aztec/aztec/testing'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; import { EthCheatCodesWithState } from '@aztec/ethereum/test'; @@ -13,6 +16,7 @@ import { sleep } from '@aztec/foundation/sleep'; import { DateProvider } from '@aztec/foundation/timer'; import { BenchmarkingContract } from '@aztec/noir-test-contracts.js/Benchmarking'; import { GasFees } from '@aztec/stdlib/gas'; +import { deriveSigningKey } from '@aztec/stdlib/keys'; import { Tx, TxHash } from '@aztec/stdlib/tx'; import { jest } from '@jest/globals'; @@ -20,18 +24,14 @@ import type { ChildProcess } from 'child_process'; import { mkdir, writeFile } from 'fs/promises'; import { dirname } from 'path'; -import { getSponsoredFPCAddress } from '../fixtures/utils.js'; +import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js'; import { PrometheusClient } from '../quality_of_service/prometheus_client.js'; -import type { TestWallet } from '../test-wallet/test_wallet.js'; -import { ProvenTx, proveInteraction } from '../test-wallet/utils.js'; -import { - type WalletWrapper, - createWalletAndAztecNodeClient, - deploySponsoredTestAccounts, -} from './setup_test_wallets.js'; +import type { WorkerWallet } from '../test-wallet/worker_wallet.js'; +import { type WorkerWalletWrapper, createWorkerWalletClient } from './setup_test_wallets.js'; import { ProvingMetrics } from './tx_metrics.js'; import { getExternalIP, + scaleProverAgents, setupEnvironment, startPortForwardForEthereum, startPortForwardForPrometeheus, @@ -45,6 +45,8 @@ if (!Number.isFinite(TARGET_TPS)) { throw new Error('Invalid TPS: ' + process.env.TPS); } +const TARGET_PROVER_AGENTS = parseInt(process.env.TARGET_PROVER_AGENTS ?? '200'); + const epochDurationSlots = config.AZTEC_EPOCH_DURATION; const slotDurationSeconds = config.AZTEC_SLOT_DURATION; const epochDurationSeconds = epochDurationSlots * slotDurationSeconds; @@ -99,9 +101,10 @@ type MetricsSnapshot = { /** A wallet that produces transactions in the background. */ type WalletTxProducer = { - wallet: TestWallet; - prototypeTx: ProvenTx | undefined; // Each wallet's own prototype (for fake proving) - readyTx: ProvenTx | null; + wallet: WorkerWallet; + accountAddress: AztecAddress; + prototypeTx: Tx | undefined; // Each wallet's own prototype (for fake proving) + readyTx: Tx | null; }; describe(`prove ${TARGET_TPS}TPS test`, () => { @@ -110,8 +113,9 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { const logger = createLogger(`e2e:spartan-test:prove-${TARGET_TPS}tps`); - let testWallets: WalletWrapper[]; - let wallets: TestWallet[]; + let testWallets: WorkerWalletWrapper[]; + let wallets: WorkerWallet[]; + let accountAddresses: AztecAddress[]; let producers: WalletTxProducer[]; let producerAbortController: AbortController; @@ -267,18 +271,41 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { logger.info(`Creating ${NUM_WALLETS} wallet(s)...`); testWallets = await timesAsync(NUM_WALLETS, i => { logger.info(`Creating wallet ${i + 1}/${NUM_WALLETS}`); - return createWalletAndAztecNodeClient(rpcUrl, config.REAL_VERIFIER, logger); + return createWorkerWalletClient(rpcUrl, config.REAL_VERIFIER, logger); }); - - const localTestAccounts = await Promise.all( - testWallets.map(tw => deploySponsoredTestAccounts(tw.wallet, aztecNode, logger, 0)), - ); - wallets = localTestAccounts.map(acc => acc.wallet); + wallets = testWallets.map(tw => tw.wallet); + + // Register FPC and create/deploy accounts + const fpcAddress = await getSponsoredFPCAddress(); + const sponsor = new SponsoredFeePaymentMethod(fpcAddress); + accountAddresses = []; + for (const wallet of wallets) { + const secret = Fr.random(); + const salt = Fr.random(); + // Register account inside worker (populates TestWallet.accounts map) + const address = await wallet.registerAccount(secret, salt); + // Register FPC in worker's PXE + await registerSponsoredFPC(wallet); + // Deploy via standard AccountManager flow (from: ZERO -> SignerlessAccount, no account lookup) + const manager = await AccountManager.create( + wallet, + secret, + new SchnorrAccountContract(deriveSigningKey(secret)), + salt, + ); + const deployMethod = await manager.getDeployMethod(); + await deployMethod.send({ + from: AztecAddress.ZERO, + fee: { paymentMethod: sponsor }, + wait: { timeout: 2400 }, + }); + logger.info(`Account deployed at ${address}`); + accountAddresses.push(address); + } logger.info('Deploying benchmark contract...'); - const sponsor = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress()); - benchmarkContract = await BenchmarkingContract.deploy(localTestAccounts[0].wallet).send({ - from: localTestAccounts[0].recipientAddress, + benchmarkContract = await BenchmarkingContract.deploy(wallets[0]).send({ + from: accountAddresses[0], fee: { paymentMethod: sponsor }, }); @@ -288,9 +315,12 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { beforeEach(async () => { logger.info(`Creating ${wallets.length} tx producers`); producers = await Promise.all( - wallets.map(async wallet => { - const proto = config.REAL_VERIFIER ? undefined : await createTx(wallet, benchmarkContract, logger); - return { wallet, prototypeTx: proto, readyTx: null }; + wallets.map(async (wallet, i) => { + const accountAddress = accountAddresses[i]; + const proto = config.REAL_VERIFIER + ? undefined + : await createTx(wallet, accountAddress, benchmarkContract, logger); + return { wallet, accountAddress, prototypeTx: proto, readyTx: null }; }), ); @@ -330,6 +360,9 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { ); await sleep(secondsToWait * 1000); } + + // scale to 10 agents in order to be able to prove the current epoch which contains up to 10 account contracts and the benchmark contract + await scaleProverAgents(config.NAMESPACE, 10, logger); }); it(`sends ${TARGET_TPS} TPS for a full epoch and waits for proof`, async () => { @@ -344,10 +377,18 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { const msPerTx = 1000 / TARGET_TPS; logger.info(`Will send ${txsToSend} transactions at ${TARGET_TPS} TPS over ${epochDurationSeconds} seconds`); + const scaleUpAtTx = Math.max(0, txsToSend - Math.ceil(TARGET_TPS * 8 * slotDurationSeconds)); const sentTxs: TxHash[] = []; const sendStartTime = performance.now(); for (let i = 0; i < txsToSend; i++) { + if (i === scaleUpAtTx) { + logger.info(`Scaling prover agents to ${TARGET_PROVER_AGENTS} (8 slots before end of tx sending)`); + void scaleProverAgents(config.NAMESPACE, TARGET_PROVER_AGENTS, logger).catch(err => + logger.error(`Failed to scale prover agents: ${err}`), + ); + } + const loopStart = performance.now(); // look for a wallet with an available tx @@ -362,7 +403,8 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { // consume tx const tx = producer.readyTx; producer.readyTx = null; - sentTxs.push(await tx.send({ wait: NO_WAIT })); + await aztecNode.sendTx(tx); + sentTxs.push(tx.getTxHash()); logger.info(`Sent tx ${i + 1}/${txsToSend}`); @@ -486,48 +528,51 @@ describe(`prove ${TARGET_TPS}TPS test`, () => { }); async function createTx( - wallet: TestWallet, + wallet: WorkerWallet, + accountAddress: AztecAddress, benchmarkContract: BenchmarkingContract, logger: Logger, -): Promise { +): Promise { logger.info('Creating prototype transaction...'); const sponsor = new SponsoredFeePaymentMethod(await getSponsoredFPCAddress()); - const tx = await proveInteraction(wallet, benchmarkContract.methods.sha256_hash_1024(Array(1024).fill(42)), { - from: (await wallet.getAccounts())[0].item, + const options = { + from: accountAddress, fee: { paymentMethod: sponsor, gasSettings: { maxPriorityFeesPerGas: GasFees.empty() } }, - }); + }; + const interaction = benchmarkContract.methods.sha256_hash_1024(Array(1024).fill(42)); + const execPayload = await interaction.request(options); + const tx = await wallet.proveTx(execPayload, toSendOptions(options)); logger.info('Prototype transaction created'); return tx; } -async function cloneTx(tx: ProvenTx, aztecNode: AztecNode): Promise { - const clonedTxData = Tx.clone(tx, false); +async function cloneTx(tx: Tx, aztecNode: AztecNode): Promise { + const clonedTx = Tx.clone(tx, false); // Fetch current minimum fees and apply 50% buffer for safety const currentFees = await aztecNode.getCurrentMinFees(); const paddedFees = currentFees.mul(1.5); // Update gas settings with current fees - (clonedTxData.data.constants.txContext.gasSettings as any).maxFeesPerGas = paddedFees; + (clonedTx.data.constants.txContext.gasSettings as any).maxFeesPerGas = paddedFees; // Randomize nullifiers to avoid conflicts - if (clonedTxData.data.forRollup) { - for (let i = 0; i < clonedTxData.data.forRollup.end.nullifiers.length; i++) { - if (clonedTxData.data.forRollup.end.nullifiers[i].isZero()) { + if (clonedTx.data.forRollup) { + for (let i = 0; i < clonedTx.data.forRollup.end.nullifiers.length; i++) { + if (clonedTx.data.forRollup.end.nullifiers[i].isZero()) { continue; } - clonedTxData.data.forRollup.end.nullifiers[i] = Fr.random(); + clonedTx.data.forRollup.end.nullifiers[i] = Fr.random(); } - } else if (clonedTxData.data.forPublic) { - for (let i = 0; i < clonedTxData.data.forPublic.nonRevertibleAccumulatedData.nullifiers.length; i++) { - if (clonedTxData.data.forPublic.nonRevertibleAccumulatedData.nullifiers[i].isZero()) { + } else if (clonedTx.data.forPublic) { + for (let i = 0; i < clonedTx.data.forPublic.nonRevertibleAccumulatedData.nullifiers.length; i++) { + if (clonedTx.data.forPublic.nonRevertibleAccumulatedData.nullifiers[i].isZero()) { continue; } - clonedTxData.data.forPublic.nonRevertibleAccumulatedData.nullifiers[i] = Fr.random(); + clonedTx.data.forPublic.nonRevertibleAccumulatedData.nullifiers[i] = Fr.random(); } } - const clonedTx = new ProvenTx((tx as any).node, clonedTxData, tx.offchainEffects, tx.stats); await clonedTx.recomputeHash(); return clonedTx; } @@ -548,7 +593,7 @@ async function startProducing( try { const tx = config.REAL_VERIFIER - ? await createTx(producer.wallet, benchmarkContract, logger) + ? await createTx(producer.wallet, producer.accountAddress, benchmarkContract, logger) : await cloneTx(producer.prototypeTx!, aztecNode); producer.readyTx = tx; diff --git a/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts b/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts index a0af523c70b7..c7567e794750 100644 --- a/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts +++ b/yarn-project/end-to-end/src/spartan/setup_test_wallets.ts @@ -20,6 +20,7 @@ import { getBBConfig } from '../fixtures/get_bb_config.js'; import { getSponsoredFPCAddress, registerSponsoredFPC } from '../fixtures/utils.js'; import { TestWallet } from '../test-wallet/test_wallet.js'; import { proveInteraction } from '../test-wallet/utils.js'; +import { WorkerWallet } from '../test-wallet/worker_wallet.js'; export interface TestAccounts { aztecNode: AztecNode; @@ -397,3 +398,42 @@ export async function createWalletAndAztecNodeClient( }, }; } + +export type WorkerWalletWrapper = { + wallet: WorkerWallet; + aztecNode: AztecNode; + cleanup: () => Promise; +}; + +export async function createWorkerWalletClient( + nodeUrl: string, + proverEnabled: boolean, + logger: Logger, +): Promise { + const aztecNode = createAztecNodeClient(nodeUrl); + const [bbConfig, acvmConfig] = await Promise.all([getBBConfig(logger), getACVMConfig(logger)]); + + // Strip cleanup functions — they can't be structured-cloned for worker transfer + const { cleanup: bbCleanup, ...bbPaths } = bbConfig ?? {}; + const { cleanup: acvmCleanup, ...acvmPaths } = acvmConfig ?? {}; + + const pxeConfig = { + dataDirectory: undefined, + dataStoreMapSizeKb: 1024 * 1024, + ...bbPaths, + ...acvmPaths, + proverEnabled, + }; + + const wallet = await WorkerWallet.create(nodeUrl, pxeConfig); + + return { + wallet, + aztecNode, + async cleanup() { + await wallet.stop(); + await bbCleanup?.(); + await acvmCleanup?.(); + }, + }; +} diff --git a/yarn-project/end-to-end/src/spartan/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/spartan/upgrade_governance_proposer.test.ts index 476584522266..242db2945d6d 100644 --- a/yarn-project/end-to-end/src/spartan/upgrade_governance_proposer.test.ts +++ b/yarn-project/end-to-end/src/spartan/upgrade_governance_proposer.test.ts @@ -4,7 +4,7 @@ import { createEthereumChain } from '@aztec/ethereum/chain'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; -import { createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { NewGovernanceProposerPayloadAbi } from '@aztec/l1-artifacts/NewGovernanceProposerPayloadAbi'; @@ -161,7 +161,7 @@ describe('spartan_upgrade_governance_proposer', () => { debugLogger.info(`Executing proposal ${info.round}`); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); const { receipt } = await governanceProposer.submitRoundWinner(executableRound, l1TxUtils); expect(receipt).toBeDefined(); expect(receipt.status).toEqual('success'); diff --git a/yarn-project/end-to-end/src/spartan/upgrade_rollup_version.test.ts b/yarn-project/end-to-end/src/spartan/upgrade_rollup_version.test.ts index bec7c674e878..02eaa828162a 100644 --- a/yarn-project/end-to-end/src/spartan/upgrade_rollup_version.test.ts +++ b/yarn-project/end-to-end/src/spartan/upgrade_rollup_version.test.ts @@ -3,7 +3,7 @@ import { createEthereumChain } from '@aztec/ethereum/chain'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { GovernanceProposerContract, RegistryContract, RollupContract } from '@aztec/ethereum/contracts'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; -import { createL1TxUtilsFromViemWallet } from '@aztec/ethereum/l1-tx-utils'; +import { createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { retryUntil } from '@aztec/foundation/retry'; @@ -264,7 +264,7 @@ describe('spartan_upgrade_rollup_version', () => { ({ round } = await govInfo()); } - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client, { logger: debugLogger }); + const l1TxUtils = createL1TxUtils(l1Client, { logger: debugLogger }); const { receipt: proposerReceipt, proposalId } = await governanceProposer.submitRoundWinner( executableRound, l1TxUtils, diff --git a/yarn-project/end-to-end/src/spartan/utils/bot.ts b/yarn-project/end-to-end/src/spartan/utils/bot.ts index 6314d2850292..abd275cd80a6 100644 --- a/yarn-project/end-to-end/src/spartan/utils/bot.ts +++ b/yarn-project/end-to-end/src/spartan/utils/bot.ts @@ -35,6 +35,7 @@ export async function installTransferBot({ replicas = 1, txIntervalSeconds = 10, followChain = 'CHECKPOINTED', + pxeSyncChainTip = 'proposed', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', @@ -49,6 +50,7 @@ export async function installTransferBot({ replicas?: number; txIntervalSeconds?: number; followChain?: string; + pxeSyncChainTip?: string; mnemonic?: string; mnemonicStartIndex?: number | string; botPrivateKey?: string; @@ -67,6 +69,7 @@ export async function installTransferBot({ 'bot.replicaCount': replicas, 'bot.txIntervalSeconds': txIntervalSeconds, 'bot.followChain': followChain, + 'bot.pxeSyncChainTip': pxeSyncChainTip, 'bot.botPrivateKey': botPrivateKey, 'bot.nodeUrl': resolvedNodeUrl, 'bot.mnemonic': mnemonic, diff --git a/yarn-project/end-to-end/src/spartan/utils/index.ts b/yarn-project/end-to-end/src/spartan/utils/index.ts index 5d945f48e758..b4ecc612825f 100644 --- a/yarn-project/end-to-end/src/spartan/utils/index.ts +++ b/yarn-project/end-to-end/src/spartan/utils/index.ts @@ -25,6 +25,7 @@ export { getRPCEndpoint, getEthereumEndpoint, createResilientPrometheusConnection, + scaleProverAgents, } from './k8s.js'; // Chaos Mesh diff --git a/yarn-project/end-to-end/src/spartan/utils/k8s.ts b/yarn-project/end-to-end/src/spartan/utils/k8s.ts index 70088963fbea..e9329839b7bb 100644 --- a/yarn-project/end-to-end/src/spartan/utils/k8s.ts +++ b/yarn-project/end-to-end/src/spartan/utils/k8s.ts @@ -522,6 +522,14 @@ export function createResilientPrometheusConnection( return { connect, runAlertCheck }; } +/** Scales the prover-agent Deployment to the given number of replicas. */ +export async function scaleProverAgents(namespace: string, replicas: number, log: Logger): Promise { + const label = 'app.kubernetes.io/component=prover-agent'; + const command = `kubectl scale deployment -l ${label} -n ${namespace} --replicas=${replicas} --timeout=2m`; + log.info(`Scaling prover agents to ${replicas}: ${command}`); + await execAsync(command); +} + export function getChartDir(spartanDir: string, chartName: string) { return path.join(spartanDir.trim(), chartName); } diff --git a/yarn-project/end-to-end/src/test-wallet/wallet_worker_script.ts b/yarn-project/end-to-end/src/test-wallet/wallet_worker_script.ts new file mode 100644 index 000000000000..820c1c402e95 --- /dev/null +++ b/yarn-project/end-to-end/src/test-wallet/wallet_worker_script.ts @@ -0,0 +1,43 @@ +import { createAztecNodeClient } from '@aztec/aztec.js/node'; +import { jsonStringify } from '@aztec/foundation/json-rpc'; +import type { ApiSchema } from '@aztec/foundation/schemas'; +import { parseWithOptionals, schemaHasMethod } from '@aztec/foundation/schemas'; +import { NodeListener, TransportServer } from '@aztec/foundation/transport'; + +import { workerData } from 'worker_threads'; + +import { TestWallet } from './test_wallet.js'; +import { WorkerWalletSchema } from './worker_wallet_schema.js'; + +const { nodeUrl, pxeConfig } = workerData as { nodeUrl: string; pxeConfig?: Record }; + +const node = createAztecNodeClient(nodeUrl); +const wallet = await TestWallet.create(node, pxeConfig); + +/** Handlers for methods that need custom implementation (not direct wallet passthrough). */ +const handlers: Record Promise> = { + proveTx: async (exec, opts) => { + const provenTx = await wallet.proveTx(exec, opts); + // ProvenTx has non-serializable fields (node proxy, etc.) — extract only Tx-compatible fields + const { data, chonkProof, contractClassLogFields, publicFunctionCalldata } = provenTx; + return { data, chonkProof, contractClassLogFields, publicFunctionCalldata }; + }, + registerAccount: async (secret, salt) => { + const manager = await wallet.createSchnorrAccount(secret, salt); + return manager.address; + }, +}; + +const schema = WorkerWalletSchema as ApiSchema; +const listener = new NodeListener(); +const server = new TransportServer<{ fn: string; args: string }>(listener, async msg => { + if (!schemaHasMethod(schema, msg.fn)) { + throw new Error(`Unknown method: ${msg.fn}`); + } + const jsonParams = JSON.parse(msg.args) as unknown[]; + const args = await parseWithOptionals(jsonParams, schema[msg.fn].parameters()); + const handler = handlers[msg.fn]; + const result = handler ? await handler(...args) : await (wallet as any)[msg.fn](...args); + return jsonStringify(result); +}); +server.start(); diff --git a/yarn-project/end-to-end/src/test-wallet/worker_wallet.ts b/yarn-project/end-to-end/src/test-wallet/worker_wallet.ts new file mode 100644 index 000000000000..d5f8b34c591b --- /dev/null +++ b/yarn-project/end-to-end/src/test-wallet/worker_wallet.ts @@ -0,0 +1,165 @@ +import type { CallIntent, IntentInnerHash } from '@aztec/aztec.js/authorization'; +import type { InteractionWaitOptions, SendReturn } from '@aztec/aztec.js/contracts'; +import type { + Aliased, + AppCapabilities, + BatchResults, + BatchedMethod, + ContractClassMetadata, + ContractMetadata, + PrivateEvent, + PrivateEventFilter, + ProfileOptions, + SendOptions, + SimulateOptions, + SimulateUtilityOptions, + Wallet, + WalletCapabilities, +} from '@aztec/aztec.js/wallet'; +import type { ChainInfo } from '@aztec/entrypoints/interfaces'; +import type { Fr } from '@aztec/foundation/curves/bn254'; +import { jsonStringify } from '@aztec/foundation/json-rpc'; +import type { ApiSchema } from '@aztec/foundation/schemas'; +import { NodeConnector, TransportClient } from '@aztec/foundation/transport'; +import type { PXEConfig } from '@aztec/pxe/config'; +import type { ContractArtifact, EventMetadataDefinition, FunctionCall } from '@aztec/stdlib/abi'; +import type { AuthWitness } from '@aztec/stdlib/auth-witness'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; +import type { ContractInstanceWithAddress } from '@aztec/stdlib/contract'; +import type { ExecutionPayload, TxProfileResult, TxSimulationResult, UtilitySimulationResult } from '@aztec/stdlib/tx'; +import { Tx } from '@aztec/stdlib/tx'; + +import { Worker } from 'worker_threads'; + +import { WorkerWalletSchema } from './worker_wallet_schema.js'; + +type WorkerMsg = { fn: string; args: string }; + +/** + * Wallet implementation that offloads all work to a worker thread. + * Implements the Wallet interface by proxying calls over a transport layer + * using JSON serialization with Zod schema parsing on both ends. + */ +export class WorkerWallet implements Wallet { + private constructor( + private worker: Worker, + private client: TransportClient, + ) {} + + /** + * Creates a WorkerWallet by spawning a worker thread that creates a TestWallet internally. + * @param nodeUrl - URL of the Aztec node to connect to. + * @param pxeConfig - Optional PXE configuration overrides. + * @returns A WorkerWallet ready to use. + */ + static async create(nodeUrl: string, pxeConfig?: Partial): Promise { + const worker = new Worker(new URL('./wallet_worker_script.js', import.meta.url), { + workerData: { nodeUrl, pxeConfig }, + }); + + const connector = new NodeConnector(worker); + const client = new TransportClient(connector); + await client.open(); + + const wallet = new WorkerWallet(worker, client); + // Warmup / readiness check — blocks until the worker has finished creating the TestWallet. + await wallet.getChainInfo(); + return wallet; + } + + private async callRaw(fn: string, ...args: any[]): Promise { + const argsJson = jsonStringify(args); + return (await this.client.request({ fn, args: argsJson })) as string; + } + + private async call(fn: string, ...args: any[]): Promise { + const resultJson = await this.callRaw(fn, ...args); + const methodSchema = (WorkerWalletSchema as ApiSchema)[fn]; + return methodSchema.returnType().parseAsync(JSON.parse(resultJson)); + } + + getChainInfo(): Promise { + return this.call('getChainInfo'); + } + + getContractMetadata(address: AztecAddress): Promise { + return this.call('getContractMetadata', address); + } + + getContractClassMetadata(id: Fr): Promise { + return this.call('getContractClassMetadata', id); + } + + getPrivateEvents( + eventMetadata: EventMetadataDefinition, + eventFilter: PrivateEventFilter, + ): Promise[]> { + return this.call('getPrivateEvents', eventMetadata, eventFilter); + } + + registerSender(address: AztecAddress, alias?: string): Promise { + return this.call('registerSender', address, alias); + } + + getAddressBook(): Promise[]> { + return this.call('getAddressBook'); + } + + getAccounts(): Promise[]> { + return this.call('getAccounts'); + } + + registerContract( + instance: ContractInstanceWithAddress, + artifact?: ContractArtifact, + secretKey?: Fr, + ): Promise { + return this.call('registerContract', instance, artifact, secretKey); + } + + simulateTx(exec: ExecutionPayload, opts: SimulateOptions): Promise { + return this.call('simulateTx', exec, opts); + } + + simulateUtility(call: FunctionCall, opts: SimulateUtilityOptions): Promise { + return this.call('simulateUtility', call, opts); + } + + profileTx(exec: ExecutionPayload, opts: ProfileOptions): Promise { + return this.call('profileTx', exec, opts); + } + + sendTx( + exec: ExecutionPayload, + opts: SendOptions, + ): Promise> { + return this.call('sendTx', exec, opts); + } + + proveTx(exec: ExecutionPayload, opts: Omit): Promise { + return this.call('proveTx', exec, opts); + } + + /** Registers an account inside the worker's TestWallet, populating its accounts map. */ + registerAccount(secret: Fr, salt: Fr): Promise { + return this.call('registerAccount', secret, salt); + } + + createAuthWit(from: AztecAddress, messageHashOrIntent: IntentInnerHash | CallIntent): Promise { + return this.call('createAuthWit', from, messageHashOrIntent); + } + + requestCapabilities(manifest: AppCapabilities): Promise { + return this.call('requestCapabilities', manifest); + } + + batch(methods: T): Promise> { + return this.call('batch', methods); + } + + /** Shuts down the worker thread and closes the transport. */ + async stop(): Promise { + this.client.close(); + await this.worker.terminate(); + } +} diff --git a/yarn-project/end-to-end/src/test-wallet/worker_wallet_schema.ts b/yarn-project/end-to-end/src/test-wallet/worker_wallet_schema.ts new file mode 100644 index 000000000000..7e2a47c4d8bf --- /dev/null +++ b/yarn-project/end-to-end/src/test-wallet/worker_wallet_schema.ts @@ -0,0 +1,13 @@ +import { ExecutionPayloadSchema, SendOptionsSchema, WalletSchema } from '@aztec/aztec.js/wallet'; +import { schemas } from '@aztec/foundation/schemas'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { Tx } from '@aztec/stdlib/tx'; + +import { z } from 'zod'; + +/** Schema for the WorkerWallet API — extends WalletSchema with proveTx and registerAccount. */ +export const WorkerWalletSchema = { + ...WalletSchema, + proveTx: z.function().args(ExecutionPayloadSchema, SendOptionsSchema).returns(Tx.schema), + registerAccount: z.function().args(schemas.Fr, schemas.Fr).returns(AztecAddress.schema), +}; diff --git a/yarn-project/ethereum/src/config.ts b/yarn-project/ethereum/src/config.ts index c9ca44d1ca38..5c271cd915cb 100644 --- a/yarn-project/ethereum/src/config.ts +++ b/yarn-project/ethereum/src/config.ts @@ -6,6 +6,7 @@ import { getConfigFromMappings, getDefaultConfig, numberConfigHelper, + omitConfigMappings, optionalNumberConfigHelper, } from '@aztec/foundation/config'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -238,7 +239,7 @@ export const l1ContractsConfigMappings: ConfigMappingsType = description: 'The delay before a validator can exit the set', ...numberConfigHelper(l1ContractsDefaultEnv.AZTEC_EXIT_DELAY_SECONDS), }, - ...l1TxUtilsConfigMappings, + ...omitConfigMappings(l1TxUtilsConfigMappings, ['ethereumSlotDuration']), }; /** diff --git a/yarn-project/ethereum/src/contracts/empire_base.ts b/yarn-project/ethereum/src/contracts/empire_base.ts index 470137b78604..377c279b3a2d 100644 --- a/yarn-project/ethereum/src/contracts/empire_base.ts +++ b/yarn-project/ethereum/src/contracts/empire_base.ts @@ -22,6 +22,8 @@ export interface IEmpireBase { signerAddress: Hex, signer: (msg: TypedDataDefinition) => Promise, ): Promise; + /** Checks if a payload was ever submitted to governance via submitRoundWinner. */ + hasPayloadBeenProposed(payload: Hex, fromBlock: bigint): Promise; } export function encodeSignal(payload: Hex): Hex { diff --git a/yarn-project/ethereum/src/contracts/empire_slashing_proposer.ts b/yarn-project/ethereum/src/contracts/empire_slashing_proposer.ts index 565d62260c16..8fc38f21b46b 100644 --- a/yarn-project/ethereum/src/contracts/empire_slashing_proposer.ts +++ b/yarn-project/ethereum/src/contracts/empire_slashing_proposer.ts @@ -126,6 +126,12 @@ export class EmpireSlashingProposerContract extends EventEmitter implements IEmp }; } + /** Checks if a payload was ever submitted to governance via submitRoundWinner. */ + public async hasPayloadBeenProposed(payload: Hex, fromBlock: bigint): Promise { + const events = await this.proposer.getEvents.PayloadSubmitted({ payload }, { fromBlock, strict: true }); + return events.length > 0; + } + public listenToSubmittablePayloads(callback: (args: { payload: `0x${string}`; round: bigint }) => unknown) { return this.proposer.watchEvent.PayloadSubmittable( {}, diff --git a/yarn-project/ethereum/src/contracts/fee_asset_handler.test.ts b/yarn-project/ethereum/src/contracts/fee_asset_handler.test.ts index d74c6b10e1c7..f49407b1c08d 100644 --- a/yarn-project/ethereum/src/contracts/fee_asset_handler.test.ts +++ b/yarn-project/ethereum/src/contracts/fee_asset_handler.test.ts @@ -12,7 +12,7 @@ import { foundry } from 'viem/chains'; import { createExtendedL1Client } from '../client.js'; import { DefaultL1ContractsConfig } from '../config.js'; import { deployAztecL1Contracts } from '../deploy_aztec_l1_contracts.js'; -import { L1TxUtils, createL1TxUtilsFromViemWallet } from '../l1_tx_utils/index.js'; +import { L1TxUtils, createL1TxUtils } from '../l1_tx_utils/index.js'; import { startAnvil } from '../test/start_anvil.js'; import type { ExtendedViemWalletClient } from '../types.js'; import { FeeAssetHandlerContract } from './fee_asset_handler.js'; @@ -48,7 +48,7 @@ describe('FeeAssetHandler', () => { }); // Since the registry cannot "see" the slash factory, we omit it from the addresses for this test const deployedAddresses = omit(deployed.l1ContractAddresses, 'slashFactoryAddress'); - txUtils = createL1TxUtilsFromViemWallet(l1Client, { logger }); + txUtils = createL1TxUtils(l1Client, { logger }); feeAssetHandler = new FeeAssetHandlerContract(l1Client, deployedAddresses.feeAssetHandlerAddress!); feeAsset = getContract({ address: deployedAddresses.feeJuiceAddress!.toString(), diff --git a/yarn-project/ethereum/src/contracts/governance.ts b/yarn-project/ethereum/src/contracts/governance.ts index ad4c38b32ef4..d4aa7b396e1e 100644 --- a/yarn-project/ethereum/src/contracts/governance.ts +++ b/yarn-project/ethereum/src/contracts/governance.ts @@ -14,7 +14,7 @@ import { } from 'viem'; import type { L1ContractAddresses } from '../l1_contract_addresses.js'; -import { createL1TxUtilsFromViemWallet } from '../l1_tx_utils/index.js'; +import { createL1TxUtils } from '../l1_tx_utils/index.js'; import { type ExtendedViemWalletClient, type ViemClient, isExtendedClient } from '../types.js'; export type L1GovernanceContractAddresses = Pick< @@ -194,7 +194,7 @@ export class GovernanceContract extends ReadOnlyGovernanceContract { retries: number; logger: Logger; }) { - const l1TxUtils = createL1TxUtilsFromViemWallet(this.client, { logger }); + const l1TxUtils = createL1TxUtils(this.client, { logger }); const retryDelaySeconds = 12; voteAmount = voteAmount ?? (await this.getPowerForProposal(proposalId)); @@ -252,7 +252,7 @@ export class GovernanceContract extends ReadOnlyGovernanceContract { retries: number; logger: Logger; }) { - const l1TxUtils = createL1TxUtilsFromViemWallet(this.client, { logger }); + const l1TxUtils = createL1TxUtils(this.client, { logger }); const retryDelaySeconds = 12; let success = false; for (let i = 0; i < retries; i++) { diff --git a/yarn-project/ethereum/src/contracts/governance_proposer.ts b/yarn-project/ethereum/src/contracts/governance_proposer.ts index fe4611105bdf..a7c203a67bdf 100644 --- a/yarn-project/ethereum/src/contracts/governance_proposer.ts +++ b/yarn-project/ethereum/src/contracts/governance_proposer.ts @@ -110,6 +110,12 @@ export class GovernanceProposerContract implements IEmpireBase { }; } + /** Checks if a payload was ever submitted to governance via submitRoundWinner. */ + public async hasPayloadBeenProposed(payload: Hex, fromBlock: bigint): Promise { + const events = await this.proposer.getEvents.PayloadSubmitted({ payload }, { fromBlock, strict: true }); + return events.length > 0; + } + public async submitRoundWinner( round: bigint, l1TxUtils: L1TxUtils, diff --git a/yarn-project/ethereum/src/contracts/multicall.test.ts b/yarn-project/ethereum/src/contracts/multicall.test.ts index 391e1107ff3c..650598743b5e 100644 --- a/yarn-project/ethereum/src/contracts/multicall.test.ts +++ b/yarn-project/ethereum/src/contracts/multicall.test.ts @@ -14,7 +14,7 @@ import { createExtendedL1Client } from '../client.js'; import { DefaultL1ContractsConfig } from '../config.js'; import { type DeployAztecL1ContractsReturnType, deployAztecL1Contracts } from '../deploy_aztec_l1_contracts.js'; import { deployL1Contract } from '../deploy_l1_contract.js'; -import { L1TxUtils, createL1TxUtilsFromViemWallet } from '../l1_tx_utils/index.js'; +import { L1TxUtils, createL1TxUtils } from '../l1_tx_utils/index.js'; import { startAnvil } from '../test/start_anvil.js'; import type { ExtendedViemWalletClient } from '../types.js'; import { FormattedViemError } from '../utils.js'; @@ -67,7 +67,7 @@ describe('Multicall3', () => { client: walletClient, }); - l1TxUtils = createL1TxUtilsFromViemWallet(walletClient, { logger }); + l1TxUtils = createL1TxUtils(walletClient, { logger }); const addMinterHash = await tokenContract.write.addMinter([MULTI_CALL_3_ADDRESS], { account: privateKey }); await walletClient.waitForTransactionReceipt({ hash: addMinterHash }); diff --git a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts index 6497f5c0400e..50ec9d32e54f 100644 --- a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts +++ b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts @@ -23,7 +23,6 @@ import type { L1ContractsConfig } from './config.js'; import { deployMulticall3 } from './contracts/multicall.js'; import { RollupContract } from './contracts/rollup.js'; import type { L1ContractAddresses } from './l1_contract_addresses.js'; -import type { L1TxUtilsConfig } from './l1_tx_utils/config.js'; import type { ExtendedViemWalletClient } from './types.js'; const logger = createLogger('ethereum:deploy_aztec_l1_contracts'); @@ -491,7 +490,25 @@ export type VerificationRecord = { libraries: VerificationLibraryEntry[]; }; -export interface DeployAztecL1ContractsArgs extends Omit { +export interface DeployAztecL1ContractsArgs + extends Omit< + L1ContractsConfig, + | 'gasLimitBufferPercentage' + | 'maxGwei' + | 'maxBlobGwei' + | 'priorityFeeBumpPercentage' + | 'priorityFeeRetryBumpPercentage' + | 'minimumPriorityFeePerGas' + | 'maxSpeedUpAttempts' + | 'checkIntervalMs' + | 'stallTimeMs' + | 'txTimeoutMs' + | 'cancelTxOnTimeout' + | 'txCancellationFinalTimeoutMs' + | 'txUnseenConsideredDroppedMs' + | 'enableDelayer' + | 'txDelayerMaxInclusionTimeIntoSlot' + > { /** The vk tree root. */ vkTreeRoot: Fr; /** The hash of the protocol contracts. */ diff --git a/yarn-project/ethereum/src/deploy_l1_contract.ts b/yarn-project/ethereum/src/deploy_l1_contract.ts index fce35b26ff71..28228a3c3bf7 100644 --- a/yarn-project/ethereum/src/deploy_l1_contract.ts +++ b/yarn-project/ethereum/src/deploy_l1_contract.ts @@ -24,7 +24,7 @@ import { } from './deploy_aztec_l1_contracts.js'; import { RegisterNewRollupVersionPayloadArtifact } from './l1_artifacts.js'; import { type L1TxUtilsConfig, getL1TxUtilsConfigEnvVars } from './l1_tx_utils/config.js'; -import { createL1TxUtilsFromViemWallet } from './l1_tx_utils/factory.js'; +import { createL1TxUtils } from './l1_tx_utils/factory.js'; import type { L1TxUtils } from './l1_tx_utils/l1_tx_utils.js'; import type { GasPrice, L1TxConfig, L1TxRequest } from './l1_tx_utils/types.js'; import type { ExtendedViemWalletClient } from './types.js'; @@ -46,7 +46,7 @@ export class L1Deployer { private createVerificationJson: boolean = false, ) { this.salt = maybeSalt ? padHex(numberToHex(maybeSalt), { size: 32 }) : undefined; - this.l1TxUtils = createL1TxUtilsFromViemWallet( + this.l1TxUtils = createL1TxUtils( this.client, { logger: this.logger, dateProvider }, { ...this.txUtilsConfig, debugMaxGasLimit: acceleratedTestDeployments }, @@ -179,7 +179,7 @@ export async function deployL1Contract( if (!l1TxUtils) { const config = getL1TxUtilsConfigEnvVars(); - l1TxUtils = createL1TxUtilsFromViemWallet( + l1TxUtils = createL1TxUtils( extendedClient, { logger }, { ...config, debugMaxGasLimit: acceleratedTestDeployments }, diff --git a/yarn-project/ethereum/src/l1_tx_utils/config.ts b/yarn-project/ethereum/src/l1_tx_utils/config.ts index 958fef38d16e..531dc6579f4f 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/config.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/config.ts @@ -5,6 +5,7 @@ import { getConfigFromMappings, getDefaultConfig, numberConfigHelper, + optionalNumberConfigHelper, } from '@aztec/foundation/config'; export interface L1TxUtilsConfig { @@ -60,6 +61,12 @@ export interface L1TxUtilsConfig { * How long a tx nonce can be unseen in the mempool before considering it dropped */ txUnseenConsideredDroppedMs?: number; + /** Enable tx delayer. When true, wraps the viem client to intercept and delay txs. Test-only. */ + enableDelayer?: boolean; + /** Max seconds into an L1 slot for tx inclusion. Txs sent later are deferred to next slot. Only used when enableDelayer is true. */ + txDelayerMaxInclusionTimeIntoSlot?: number; + /** How many seconds an L1 slot lasts. */ + ethereumSlotDuration?: number; } export const l1TxUtilsConfigMappings: ConfigMappingsType = { @@ -142,6 +149,19 @@ export const l1TxUtilsConfigMappings: ConfigMappingsType = { env: 'L1_TX_MONITOR_TX_UNSEEN_CONSIDERED_DROPPED_MS', ...numberConfigHelper(6 * 12 * 1000), // 6 L1 blocks }, + enableDelayer: { + description: 'Enable tx delayer for testing.', + ...booleanConfigHelper(false), + }, + txDelayerMaxInclusionTimeIntoSlot: { + description: 'Max seconds into L1 slot for tx inclusion when delayer is enabled.', + ...optionalNumberConfigHelper(), + }, + ethereumSlotDuration: { + env: 'ETHEREUM_SLOT_DURATION', + description: 'How many seconds an L1 slot lasts.', + ...numberConfigHelper(12), + }, }; // We abuse the fact that all mappings above have a non null default value and force-type this to Required diff --git a/yarn-project/ethereum/src/l1_tx_utils/factory.ts b/yarn-project/ethereum/src/l1_tx_utils/factory.ts index 10d75b9f8eda..9ac1560b7aed 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/factory.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/factory.ts @@ -1,64 +1,64 @@ +import type { BlobKzgInstance } from '@aztec/blob-lib/types'; import { EthAddress } from '@aztec/foundation/eth-address'; import type { Logger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; -import type { TransactionSerializable } from 'viem'; - import type { EthSigner } from '../eth-signer/eth-signer.js'; import type { ExtendedViemWalletClient, ViemClient } from '../types.js'; import type { L1TxUtilsConfig } from './config.js'; import type { IL1TxMetrics, IL1TxStore } from './interfaces.js'; import { L1TxUtils } from './l1_tx_utils.js'; import { createViemSigner } from './signer.js'; +import { Delayer } from './tx_delayer.js'; import type { SigningCallback } from './types.js'; -export function createL1TxUtilsFromViemWallet( - client: ExtendedViemWalletClient, - deps?: { - logger?: Logger; - dateProvider?: DateProvider; - store?: IL1TxStore; - metrics?: IL1TxMetrics; - }, - config?: Partial & { debugMaxGasLimit?: boolean }, -): L1TxUtils { - return new L1TxUtils( +/** Source of signing capability: either a wallet client or a separate client + signer. */ +export type L1SignerSource = ExtendedViemWalletClient | { client: ViemClient; signer: EthSigner }; + +export function resolveSignerSource(source: L1SignerSource): { + client: ViemClient; + address: EthAddress; + signingCallback: SigningCallback; +} { + if ('account' in source && source.account) { + return { + client: source as ExtendedViemWalletClient, + address: EthAddress.fromString((source as ExtendedViemWalletClient).account.address), + signingCallback: createViemSigner(source as ExtendedViemWalletClient), + }; + } + const { client, signer } = source as { client: ViemClient; signer: EthSigner }; + return { client, - EthAddress.fromString(client.account.address), - createViemSigner(client), - deps?.logger, - deps?.dateProvider, - config, - config?.debugMaxGasLimit ?? false, - deps?.store, - deps?.metrics, - ); + address: signer.address, + signingCallback: async (tx, _addr) => (await signer.signTransaction(tx)).toViemTransactionSignature(), + }; } -export function createL1TxUtilsFromEthSigner( - client: ViemClient, - signer: EthSigner, +export function createL1TxUtils( + source: L1SignerSource, deps?: { logger?: Logger; dateProvider?: DateProvider; store?: IL1TxStore; metrics?: IL1TxMetrics; + kzg?: BlobKzgInstance; + delayer?: Delayer; }, config?: Partial & { debugMaxGasLimit?: boolean }, ): L1TxUtils { - const callback: SigningCallback = async (transaction: TransactionSerializable, _signingAddress) => { - return (await signer.signTransaction(transaction)).toViemTransactionSignature(); - }; - + const { client, address, signingCallback } = resolveSignerSource(source); return new L1TxUtils( client, - signer.address, - callback, + address, + signingCallback, deps?.logger, deps?.dateProvider, config, config?.debugMaxGasLimit ?? false, deps?.store, deps?.metrics, + deps?.kzg, + deps?.delayer, ); } diff --git a/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.test.ts b/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.test.ts index 66c9d6fd8dce..3a3612efbef1 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.test.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.test.ts @@ -1,3 +1,4 @@ +import { Blob } from '@aztec/blob-lib'; import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { TestDateProvider } from '@aztec/foundation/timer'; @@ -105,6 +106,8 @@ describe('ForwarderL1TxUtils', () => { false, undefined, undefined, + Blob.getViemKzgInstance(), + undefined, forwarderAddress, ); diff --git a/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.ts b/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.ts index ca7811980c95..da2f96ae1c63 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/forwarder_l1_tx_utils.ts @@ -1,25 +1,27 @@ +import type { BlobKzgInstance } from '@aztec/blob-lib/types'; import { EthAddress } from '@aztec/foundation/eth-address'; import type { Logger } from '@aztec/foundation/log'; import type { DateProvider } from '@aztec/foundation/timer'; import { type Hex, encodeFunctionData } from 'viem'; -import type { EthSigner } from '../eth-signer/eth-signer.js'; import { FORWARDER_ABI } from '../forwarder_proxy.js'; -import type { ExtendedViemWalletClient, ViemClient } from '../types.js'; +import type { ViemClient } from '../types.js'; import type { L1TxUtilsConfig } from './config.js'; +import type { L1SignerSource } from './factory.js'; +import { resolveSignerSource } from './factory.js'; import type { IL1TxMetrics, IL1TxStore } from './interfaces.js'; -import { L1TxUtilsWithBlobs } from './l1_tx_utils_with_blobs.js'; -import { createViemSigner } from './signer.js'; +import { L1TxUtils } from './l1_tx_utils.js'; +import { Delayer } from './tx_delayer.js'; import type { L1BlobInputs, L1TxConfig, L1TxRequest, SigningCallback } from './types.js'; /** - * Extends L1TxUtilsWithBlobs to wrap all transactions through a forwarder contract. + * Extends L1TxUtils to wrap all transactions through a forwarder contract. * This is mainly used for testing the archiver's ability to decode transactions that go through proxies. */ -export class ForwarderL1TxUtils extends L1TxUtilsWithBlobs { +export class ForwarderL1TxUtils extends L1TxUtils { constructor( - client: ViemClient | ExtendedViemWalletClient, + client: ViemClient, senderAddress: EthAddress, signingCallback: SigningCallback, logger: Logger | undefined, @@ -28,9 +30,23 @@ export class ForwarderL1TxUtils extends L1TxUtilsWithBlobs { debugMaxGasLimit: boolean, store: IL1TxStore | undefined, metrics: IL1TxMetrics | undefined, + kzg: BlobKzgInstance | undefined, + delayer: Delayer | undefined, private readonly forwarderAddress: EthAddress, ) { - super(client, senderAddress, signingCallback, logger, dateProvider, config, debugMaxGasLimit, store, metrics); + super( + client, + senderAddress, + signingCallback, + logger, + dateProvider, + config, + debugMaxGasLimit, + store, + metrics, + kzg, + delayer, + ); } /** @@ -61,59 +77,32 @@ export class ForwarderL1TxUtils extends L1TxUtilsWithBlobs { } } -export function createForwarderL1TxUtilsFromViemWallet( - client: ExtendedViemWalletClient, +export function createForwarderL1TxUtils( + source: L1SignerSource, forwarderAddress: EthAddress, - deps: { + deps?: { logger?: Logger; dateProvider?: DateProvider; store?: IL1TxStore; metrics?: IL1TxMetrics; - } = {}, - config: Partial = {}, - debugMaxGasLimit: boolean = false, -) { + kzg?: BlobKzgInstance; + delayer?: Delayer; + }, + config?: Partial & { debugMaxGasLimit?: boolean }, +): ForwarderL1TxUtils { + const { client, address, signingCallback } = resolveSignerSource(source); return new ForwarderL1TxUtils( client, - EthAddress.fromString(client.account.address), - createViemSigner(client), - deps.logger, - deps.dateProvider, - config, - debugMaxGasLimit, - deps.store, - deps.metrics, - forwarderAddress, - ); -} - -export function createForwarderL1TxUtilsFromEthSigner( - client: ViemClient, - signer: EthSigner, - forwarderAddress: EthAddress, - deps: { - logger?: Logger; - dateProvider?: DateProvider; - store?: IL1TxStore; - metrics?: IL1TxMetrics; - } = {}, - config: Partial = {}, - debugMaxGasLimit: boolean = false, -) { - const callback: SigningCallback = async (transaction, _signingAddress) => { - return (await signer.signTransaction(transaction)).toViemTransactionSignature(); - }; - - return new ForwarderL1TxUtils( - client, - signer.address, - callback, - deps.logger, - deps.dateProvider, - config, - debugMaxGasLimit, - deps.store, - deps.metrics, + address, + signingCallback, + deps?.logger, + deps?.dateProvider, + config ?? {}, + config?.debugMaxGasLimit ?? false, + deps?.store, + deps?.metrics, + deps?.kzg, + deps?.delayer, forwarderAddress, ); } diff --git a/yarn-project/ethereum/src/l1_tx_utils/index-blobs.ts b/yarn-project/ethereum/src/l1_tx_utils/index-blobs.ts index 9796ce24da1b..bba69654cf89 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/index-blobs.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/index-blobs.ts @@ -1,2 +1,2 @@ -export * from './forwarder_l1_tx_utils.js'; -export * from './l1_tx_utils_with_blobs.js'; +export { createForwarderL1TxUtils, ForwarderL1TxUtils } from './forwarder_l1_tx_utils.js'; +export { createL1TxUtils, type L1SignerSource, resolveSignerSource } from './factory.js'; diff --git a/yarn-project/ethereum/src/l1_tx_utils/index.ts b/yarn-project/ethereum/src/l1_tx_utils/index.ts index 2d51cf6745e2..24605d9d45db 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/index.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/index.ts @@ -8,6 +8,7 @@ export * from './l1_tx_utils.js'; export * from './readonly_l1_tx_utils.js'; export * from './signer.js'; export * from './types.js'; +export * from './tx_delayer.js'; export * from './utils.js'; // Note: We intentionally do not export l1_tx_utils_with_blobs.js diff --git a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.test.ts b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.test.ts index 5d75639dcc20..8670113a7ed7 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.test.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.test.ts @@ -41,10 +41,10 @@ import { ReadOnlyL1TxUtils, TxUtilsState, UnknownMinedTxError, - createL1TxUtilsFromViemWallet, + createL1TxUtils, defaultL1TxUtilsConfig, } from './index.js'; -import { L1TxUtilsWithBlobs } from './l1_tx_utils_with_blobs.js'; +import { L1TxUtils } from './l1_tx_utils.js'; import { createViemSigner } from './signer.js'; const MNEMONIC = 'test test test test test test test test test test test junk'; @@ -96,8 +96,8 @@ describe('L1TxUtils', () => { await anvil.stop().catch(err => createLogger('cleanup').error(err)); }, 5000); - describe('L1TxUtilsWithBlobs', () => { - let gasUtils: TestL1TxUtilsWithBlobs; + describe('L1TxUtils with blobs', () => { + let gasUtils: TestL1TxUtils; let config: Partial; const request = { @@ -107,7 +107,7 @@ describe('L1TxUtils', () => { }; const createL1TxUtils = () => - new TestL1TxUtilsWithBlobs( + new TestL1TxUtils( l1Client, EthAddress.fromString(l1Client.account.address), createViemSigner(l1Client), @@ -117,6 +117,8 @@ describe('L1TxUtils', () => { undefined, undefined, metrics, + Blob.getViemKzgInstance(), + undefined, ); beforeEach(() => { @@ -916,6 +918,21 @@ describe('L1TxUtils', () => { expect(result.receipt.status).toBe('reverted'); }); + it('does not consume nonce when transaction times out before sending', async () => { + // Get the expected nonce before any transaction + const expectedNonce = await l1Client.getTransactionCount({ address: l1Client.account.address }); + + // Try to send with an already-expired timeout (epoch 0 is well in the past) + const pastTimeout = new Date(0); + await expect(gasUtils.sendTransaction(request, { txTimeoutAt: pastTimeout })).rejects.toThrow( + /timed out before sending/, + ); + + // The next transaction should use the same nonce (not skip one due to a leaked consume) + const { state } = await gasUtils.sendTransaction(request); + expect(state.nonce).toBe(expectedNonce); + }, 10_000); + it('stops trying after timeout once block is mined', async () => { await cheatCodes.setAutomine(false); await cheatCodes.setIntervalMining(0); @@ -1777,7 +1794,7 @@ describe('L1TxUtils', () => { }); it('L1TxUtils can be instantiated with wallet client and has write methods', () => { - const l1TxUtils = createL1TxUtilsFromViemWallet(walletClient, { logger }); + const l1TxUtils = createL1TxUtils(walletClient, { logger }); expect(l1TxUtils).toBeDefined(); expect(l1TxUtils.client).toBe(walletClient); @@ -1789,7 +1806,7 @@ describe('L1TxUtils', () => { }); it('L1TxUtils inherits all read-only methods from ReadOnlyL1TxUtils', () => { - const l1TxUtils = createL1TxUtilsFromViemWallet(walletClient, { logger }); + const l1TxUtils = createL1TxUtils(walletClient, { logger }); // Verify all read-only methods are available expect(l1TxUtils.getBlock).toBeDefined(); @@ -1803,13 +1820,13 @@ describe('L1TxUtils', () => { it('L1TxUtils cannot be instantiated with public client', () => { expect(() => { - createL1TxUtilsFromViemWallet(publicClient as any, { logger }); + createL1TxUtils(publicClient as any, { logger }); }).toThrow(); }); }); }); -class TestL1TxUtilsWithBlobs extends L1TxUtilsWithBlobs { +class TestL1TxUtils extends L1TxUtils { declare public txs: L1TxState[]; public setMetrics(metrics: IL1TxMetrics) { diff --git a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts index ab091aa4eba3..5c5c0f776db2 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts @@ -1,8 +1,9 @@ +import type { BlobKzgInstance } from '@aztec/blob-lib/types'; import { maxBigint } from '@aztec/foundation/bigint'; import { merge, pick } from '@aztec/foundation/collection'; import { InterruptError, TimeoutError } from '@aztec/foundation/error'; import { EthAddress } from '@aztec/foundation/eth-address'; -import { type Logger, createLogger } from '@aztec/foundation/log'; +import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { DateProvider } from '@aztec/foundation/timer'; @@ -30,6 +31,7 @@ import { type L1TxUtilsConfig, l1TxUtilsConfigMappings } from './config.js'; import { MAX_L1_TX_LIMIT } from './constants.js'; import type { IL1TxMetrics, IL1TxStore } from './interfaces.js'; import { ReadOnlyL1TxUtils } from './readonly_l1_tx_utils.js'; +import { Delayer, createDelayer, wrapClientWithDelayer } from './tx_delayer.js'; import { DroppedTransactionError, type L1BlobInputs, @@ -47,6 +49,10 @@ const MAX_L1_TX_STATES = 32; export class L1TxUtils extends ReadOnlyL1TxUtils { protected nonceManager: NonceManager; protected txs: L1TxState[] = []; + /** Tx delayer for testing. Only set when enableDelayer config is true. */ + public delayer?: Delayer; + /** KZG instance for blob operations. */ + protected kzg?: BlobKzgInstance; constructor( public override client: ViemClient, @@ -58,9 +64,26 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { debugMaxGasLimit: boolean = false, protected store?: IL1TxStore, protected metrics?: IL1TxMetrics, + kzg?: BlobKzgInstance, + delayer?: Delayer, ) { super(client, logger, dateProvider, config, debugMaxGasLimit); this.nonceManager = createNonceManager({ source: jsonRpc() }); + this.kzg = kzg; + + // Set up delayer: use provided one or create new + if (config?.enableDelayer && config?.ethereumSlotDuration) { + this.delayer = + delayer ?? this.createDelayer({ ethereumSlotDuration: config.ethereumSlotDuration }, logger.getBindings()); + this.client = wrapClientWithDelayer(this.client, this.delayer); + if (config.txDelayerMaxInclusionTimeIntoSlot !== undefined) { + this.delayer.setMaxInclusionTimeIntoSlot(config.txDelayerMaxInclusionTimeIntoSlot); + } + } else if (delayer) { + // Delayer provided but enableDelayer not set — just store it without wrapping + logger.warn('Delayer provided but enableDelayer config is not set; delayer will not be used'); + this.delayer = delayer; + } } public get state() { @@ -221,6 +244,16 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { throw new InterruptError(`Transaction sending is interrupted`); } + // Check timeout before consuming nonce to avoid leaking a nonce that was never sent. + // A leaked nonce creates a gap (e.g. nonce 107 consumed but unsent), so all subsequent + // transactions (108, 109, ...) can never be mined since the chain expects 107 first. + const now = new Date(await this.getL1Timestamp()); + if (gasConfig.txTimeoutAt && now > gasConfig.txTimeoutAt) { + throw new TimeoutError( + `Transaction timed out before sending (now ${now.toISOString()} > timeoutAt ${gasConfig.txTimeoutAt.toISOString()})`, + ); + } + const nonce = await this.nonceManager.consume({ client: this.client, address: account, @@ -230,13 +263,6 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { const baseState = { request, gasLimit, blobInputs, gasPrice, nonce }; const txData = this.makeTxData(baseState, { isCancelTx: false }); - const now = new Date(await this.getL1Timestamp()); - if (gasConfig.txTimeoutAt && now > gasConfig.txTimeoutAt) { - throw new TimeoutError( - `Transaction timed out before sending (now ${now.toISOString()} > timeoutAt ${gasConfig.txTimeoutAt.toISOString()})`, - ); - } - // Send the new tx const signedRequest = await this.prepareSignedTransaction(txData); const txHash = await this.client.sendRawTransaction({ serializedTransaction: signedRequest }); @@ -731,8 +757,17 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { return Number(timestamp) * 1000; } - /** Makes empty blob inputs for the cancellation tx. To be overridden in L1TxUtilsWithBlobs. */ - protected makeEmptyBlobInputs(_maxFeePerBlobGas: bigint): Required { - throw new Error('Cannot make empty blob inputs for cancellation'); + /** Makes empty blob inputs for the cancellation tx. */ + protected makeEmptyBlobInputs(maxFeePerBlobGas: bigint): Required { + if (!this.kzg) { + throw new Error('Cannot make empty blob inputs for cancellation without kzg'); + } + const blobData = new Uint8Array(131072).fill(0); + return { blobs: [blobData], kzg: this.kzg, maxFeePerBlobGas }; + } + + /** Creates a new delayer instance. */ + protected createDelayer(opts: { ethereumSlotDuration: bigint | number }, bindings: LoggerBindings): Delayer { + return createDelayer(this.dateProvider, opts, bindings); } } diff --git a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils_with_blobs.ts b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils_with_blobs.ts deleted file mode 100644 index 4f20c383ee80..000000000000 --- a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils_with_blobs.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Blob } from '@aztec/blob-lib'; -import { EthAddress } from '@aztec/foundation/eth-address'; -import type { Logger } from '@aztec/foundation/log'; -import { DateProvider } from '@aztec/foundation/timer'; - -import type { TransactionSerializable } from 'viem'; - -import type { EthSigner } from '../eth-signer/eth-signer.js'; -import type { ExtendedViemWalletClient, ViemClient } from '../types.js'; -import type { L1TxUtilsConfig } from './config.js'; -import type { IL1TxMetrics, IL1TxStore } from './interfaces.js'; -import { L1TxUtils } from './l1_tx_utils.js'; -import { createViemSigner } from './signer.js'; -import type { L1BlobInputs, SigningCallback } from './types.js'; - -/** Extends L1TxUtils with the capability to cancel blobs. This needs to be a separate class so we don't require a dependency on blob-lib unnecessarily. */ -export class L1TxUtilsWithBlobs extends L1TxUtils { - /** Makes empty blob inputs for the cancellation tx. */ - protected override makeEmptyBlobInputs(maxFeePerBlobGas: bigint): Required { - const blobData = new Uint8Array(131072).fill(0); - const kzg = Blob.getViemKzgInstance(); - return { blobs: [blobData], kzg, maxFeePerBlobGas }; - } -} - -export function createL1TxUtilsWithBlobsFromViemWallet( - client: ExtendedViemWalletClient, - deps: { - logger?: Logger; - dateProvider?: DateProvider; - store?: IL1TxStore; - metrics?: IL1TxMetrics; - } = {}, - config: Partial = {}, - debugMaxGasLimit: boolean = false, -) { - return new L1TxUtilsWithBlobs( - client, - EthAddress.fromString(client.account.address), - createViemSigner(client), - deps.logger, - deps.dateProvider, - config, - debugMaxGasLimit, - deps.store, - deps.metrics, - ); -} - -export function createL1TxUtilsWithBlobsFromEthSigner( - client: ViemClient, - signer: EthSigner, - deps: { - logger?: Logger; - dateProvider?: DateProvider; - store?: IL1TxStore; - metrics?: IL1TxMetrics; - } = {}, - config: Partial = {}, - debugMaxGasLimit: boolean = false, -) { - const callback: SigningCallback = async (transaction: TransactionSerializable, _signingAddress) => { - return (await signer.signTransaction(transaction)).toViemTransactionSignature(); - }; - - return new L1TxUtilsWithBlobs( - client, - signer.address, - callback, - deps.logger, - deps.dateProvider, - config, - debugMaxGasLimit, - deps.store, - deps.metrics, - ); -} diff --git a/yarn-project/ethereum/src/test/tx_delayer.ts b/yarn-project/ethereum/src/l1_tx_utils/tx_delayer.ts similarity index 75% rename from yarn-project/ethereum/src/test/tx_delayer.ts rename to yarn-project/ethereum/src/l1_tx_utils/tx_delayer.ts index 4776f012ec92..a5bb36d63c39 100644 --- a/yarn-project/ethereum/src/test/tx_delayer.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/tx_delayer.ts @@ -1,5 +1,5 @@ import { omit } from '@aztec/foundation/collection'; -import { type Logger, createLogger } from '@aztec/foundation/log'; +import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log'; import { retryUntil } from '@aztec/foundation/retry'; import type { DateProvider } from '@aztec/foundation/timer'; @@ -9,14 +9,16 @@ import { type Hex, type PublicClient, type TransactionSerializableEIP4844, + type TransactionSerialized, keccak256, parseTransaction, publicActions, + recoverTransactionAddress, serializeTransaction, walletActions, } from 'viem'; -import { type ViemClient, isExtendedClient } from '../types.js'; +import type { ExtendedViemWalletClient, ViemClient } from '../types.js'; const MAX_WAIT_TIME_SECONDS = 180; @@ -73,82 +75,98 @@ export function waitUntilL1Timestamp( ); } -export interface Delayer { - /** Returns the hashes of all effectively sent txs. */ - getSentTxHashes(): Hex[]; - /** Returns the raw hex for all cancelled txs. */ - getCancelledTxs(): Hex[]; - /** Delays the next tx to be sent so it lands on the given L1 block number. */ - pauseNextTxUntilBlock(l1BlockNumber: number | bigint | undefined): void; - /** Delays the next tx to be sent so it lands on the given timestamp. */ - pauseNextTxUntilTimestamp(l1Timestamp: number | bigint | undefined): void; - /** Delays the next tx to be sent indefinitely. */ - cancelNextTx(): void; - /** - * Sets max inclusion time into slot. If more than this many seconds have passed - * since the last L1 block was mined, then any tx will not be mined in the current - * L1 slot but will be deferred for the next one. - */ - setMaxInclusionTimeIntoSlot(seconds: number | bigint | undefined): void; -} +/** Manages tx delaying for testing, intercepting sendRawTransaction calls to delay or cancel them. */ +export class Delayer { + private logger: Logger; + + public maxInclusionTimeIntoSlot: number | undefined = undefined; + public ethereumSlotDuration: bigint; + public nextWait: { l1Timestamp: bigint } | { l1BlockNumber: bigint } | { indefinitely: true } | undefined = undefined; + public sentTxHashes: Hex[] = []; + public cancelledTxs: Hex[] = []; -class DelayerImpl implements Delayer { - private logger = createLogger('ethereum:tx_delayer'); constructor( public dateProvider: DateProvider, opts: { ethereumSlotDuration: bigint | number }, + bindings: LoggerBindings, ) { this.ethereumSlotDuration = BigInt(opts.ethereumSlotDuration); + this.logger = createLogger('ethereum:tx_delayer', bindings); } - public maxInclusionTimeIntoSlot: number | undefined = undefined; - public ethereumSlotDuration: bigint; - public nextWait: { l1Timestamp: bigint } | { l1BlockNumber: bigint } | { indefinitely: true } | undefined = undefined; - public sentTxHashes: Hex[] = []; - public cancelledTxs: Hex[] = []; + /** Returns the logger instance used by this delayer. */ + getLogger(): Logger { + return this.logger; + } + /** Returns the hashes of all effectively sent txs. */ getSentTxHashes() { return this.sentTxHashes; } + /** Returns the raw hex for all cancelled txs. */ getCancelledTxs(): Hex[] { return this.cancelledTxs; } + /** Delays the next tx to be sent so it lands on the given L1 block number. */ pauseNextTxUntilBlock(l1BlockNumber: number | bigint) { this.nextWait = { l1BlockNumber: BigInt(l1BlockNumber) }; } + /** Delays the next tx to be sent so it lands on the given timestamp. */ pauseNextTxUntilTimestamp(l1Timestamp: number | bigint) { this.nextWait = { l1Timestamp: BigInt(l1Timestamp) }; } + /** Delays the next tx to be sent indefinitely. */ cancelNextTx() { this.nextWait = { indefinitely: true }; } + /** + * Sets max inclusion time into slot. If more than this many seconds have passed + * since the last L1 block was mined, then any tx will not be mined in the current + * L1 slot but will be deferred for the next one. + */ setMaxInclusionTimeIntoSlot(seconds: number | undefined) { this.maxInclusionTimeIntoSlot = seconds; } } /** - * Returns a new client (without modifying the one passed in) with an injected tx delayer. - * The delayer can be used to hold off the next tx to be sent until a given block number. - * TODO(#10824): This doesn't play along well with blob txs for some reason. + * Creates a new Delayer instance. Exposed so callers can create a single shared delayer + * and pass it to multiple `wrapClientWithDelayer` calls. */ -export function withDelayer( - client: T, +export function createDelayer( dateProvider: DateProvider, opts: { ethereumSlotDuration: bigint | number }, -): { client: T; delayer: Delayer } { - if (!isExtendedClient(client)) { - throw new Error('withDelayer has to be instantiated with a wallet viem client.'); + bindings: LoggerBindings, +): Delayer { + return new Delayer(dateProvider, opts, bindings); +} + +/** Tries to recover the sender address from a serialized signed transaction. */ +async function tryRecoverSender(serializedTransaction: Hex): Promise { + try { + return await recoverTransactionAddress({ + serializedTransaction: serializedTransaction as TransactionSerialized, + }); + } catch { + return undefined; } - const logger = createLogger('ethereum:tx_delayer'); - const delayer = new DelayerImpl(dateProvider, opts); +} - const extended = client +/** + * Wraps a viem client with tx delaying logic. Returns the wrapped client. + * The delayer intercepts sendRawTransaction calls and delays them based on the delayer's state. + */ +export function wrapClientWithDelayer(client: T, delayer: Delayer): T { + const logger = delayer.getLogger(); + + // Cast to ExtendedViemWalletClient for the extend chain since it has sendRawTransaction. + // The sendRawTransaction override is applied to all clients regardless of type. + const withRawTx = (client as unknown as ExtendedViemWalletClient) // Tweak sendRawTransaction so it uses the delay defined in the delayer. // Note that this will only work with local accounts (ie accounts for which we have the private key). // Transactions signed by the node will not be delayed since they use sendTransaction directly, @@ -160,6 +178,7 @@ export function withDelayer( const { serializedTransaction } = args[0]; const publicClient = client as unknown as PublicClient; + const sender = await tryRecoverSender(serializedTransaction); if (delayer.nextWait !== undefined) { // Check if we have been instructed to delay the next tx. @@ -171,7 +190,7 @@ export function withDelayer( // Cancel tx outright if instructed if ('indefinitely' in waitUntil && waitUntil.indefinitely) { - logger.info(`Cancelling tx ${txHash}`); + logger.info(`Cancelling tx ${txHash}`, { sender }); delayer.cancelledTxs.push(serializedTransaction); return Promise.resolve(txHash); } @@ -185,6 +204,7 @@ export function withDelayer( : undefined; logger.info(`Delaying tx ${txHash} until ${inspect(waitUntil)}`, { + sender, argsLen: args.length, ...omit(parseTransaction(serializedTransaction), 'data', 'sidecars'), }); @@ -196,6 +216,7 @@ export function withDelayer( txHash = computeTxHash(serializedTransaction); const logData = { + sender, ...omit(parseTransaction(serializedTransaction), 'data', 'sidecars'), lastBlockTimestamp, now, @@ -225,28 +246,35 @@ export function withDelayer( computedTxHash: txHash, }); } - logger.info(`Sent previously delayed tx ${clientTxHash}`); + logger.info(`Sent previously delayed tx ${clientTxHash}`, { sender }); delayer.sentTxHashes.push(clientTxHash); }) .catch(err => logger.error(`Error sending tx after delay`, err)); return Promise.resolve(txHash!); } else { const txHash = await client.sendRawTransaction(...args); - logger.debug(`Sent tx immediately ${txHash}`); + logger.debug(`Sent tx immediately ${txHash}`, { sender }); delayer.sentTxHashes.push(txHash); return txHash; } }, - })) - // Re-extend with sendTransaction so it uses the modified sendRawTransaction. - .extend(client => ({ sendTransaction: walletActions(client).sendTransaction })) - // And with the actions that depend on the modified sendTransaction - .extend(client => ({ - writeContract: walletActions(client).writeContract, - deployContract: walletActions(client).deployContract, - })) as T; + })); + + // Only re-bind wallet actions (sendTransaction, writeContract, deployContract) for wallet clients. + // This is needed for tests that use wallet actions directly rather than sendRawTransaction. + const isWalletClient = 'account' in client && client.account !== undefined; + const extended = isWalletClient + ? withRawTx + // Re-extend with sendTransaction so it uses the modified sendRawTransaction. + .extend(client => ({ sendTransaction: walletActions(client).sendTransaction })) + // And with the actions that depend on the modified sendTransaction + .extend(client => ({ + writeContract: walletActions(client).writeContract, + deployContract: walletActions(client).deployContract, + })) + : withRawTx; - return { client: extended, delayer }; + return extended as T; } /** diff --git a/yarn-project/ethereum/src/test/delayed_tx_utils.ts b/yarn-project/ethereum/src/test/delayed_tx_utils.ts deleted file mode 100644 index 136f700aa5c4..000000000000 --- a/yarn-project/ethereum/src/test/delayed_tx_utils.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { EthAddress } from '@aztec/foundation/eth-address'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { DateProvider } from '@aztec/foundation/timer'; - -import { type L1TxUtilsConfig, createViemSigner } from '../l1_tx_utils/index.js'; -import { L1TxUtilsWithBlobs } from '../l1_tx_utils/l1_tx_utils_with_blobs.js'; -import type { ExtendedViemWalletClient } from '../types.js'; -import { type Delayer, withDelayer } from './tx_delayer.js'; - -export class DelayedTxUtils extends L1TxUtilsWithBlobs { - public delayer: Delayer | undefined; - - public static fromL1TxUtils( - l1TxUtils: L1TxUtilsWithBlobs, - ethereumSlotDuration: number, - wallet: ExtendedViemWalletClient, - ) { - const { client, delayer } = withDelayer(wallet, l1TxUtils.dateProvider, { - ethereumSlotDuration, - }); - const casted = l1TxUtils as unknown as DelayedTxUtils; - casted.delayer = delayer; - casted.client = client; - return casted; - } - - public enableDelayer(ethereumSlotDuration: number) { - const { client, delayer } = withDelayer(this.client, this.dateProvider, { - ethereumSlotDuration, - }); - this.delayer = delayer; - this.client = client; - } -} - -export function createDelayedL1TxUtilsFromViemWallet( - client: ExtendedViemWalletClient, - logger: Logger = createLogger('L1TxUtils'), - dateProvider: DateProvider = new DateProvider(), - config?: Partial, - debugMaxGasLimit: boolean = false, -) { - return new DelayedTxUtils( - client, - EthAddress.fromString(client.account.address), - createViemSigner(client), - logger, - dateProvider, - config, - debugMaxGasLimit, - ); -} diff --git a/yarn-project/ethereum/src/test/index.ts b/yarn-project/ethereum/src/test/index.ts index ba2c6035af70..1b4ce5a97cfb 100644 --- a/yarn-project/ethereum/src/test/index.ts +++ b/yarn-project/ethereum/src/test/index.ts @@ -1,8 +1,6 @@ -export * from './delayed_tx_utils.js'; export * from './eth_cheat_codes.js'; export * from './eth_cheat_codes_with_state.js'; export * from './start_anvil.js'; -export * from './tx_delayer.js'; export * from './upgrade_utils.js'; export * from './chain_monitor.js'; export * from './rollup_cheat_codes.js'; diff --git a/yarn-project/ethereum/src/test/tx_delayer.test.ts b/yarn-project/ethereum/src/test/tx_delayer.test.ts index f2e03cde63da..00abbb4c0cff 100644 --- a/yarn-project/ethereum/src/test/tx_delayer.test.ts +++ b/yarn-project/ethereum/src/test/tx_delayer.test.ts @@ -10,10 +10,10 @@ import { type PrivateKeyAccount, createWalletClient, fallback, getContract, http import { privateKeyToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; +import { Delayer, createDelayer, waitUntilBlock, wrapClientWithDelayer } from '../l1_tx_utils/tx_delayer.js'; import type { ExtendedViemWalletClient } from '../types.js'; import { EthCheatCodes } from './eth_cheat_codes.js'; import { startAnvil } from './start_anvil.js'; -import { type Delayer, waitUntilBlock, withDelayer } from './tx_delayer.js'; describe('tx_delayer', () => { let anvil: Anvil; @@ -41,7 +41,8 @@ describe('tx_delayer', () => { chain: foundry, account, }).extend(publicActions); - ({ client, delayer } = withDelayer(_client, dateProvider, { ethereumSlotDuration: ETHEREUM_SLOT_DURATION })); + delayer = createDelayer(dateProvider, { ethereumSlotDuration: ETHEREUM_SLOT_DURATION }, {}); + client = wrapClientWithDelayer(_client, delayer); }); const receiptNotFound = expect.objectContaining({ name: 'TransactionReceiptNotFoundError' }); diff --git a/yarn-project/ethereum/src/test/upgrade_utils.ts b/yarn-project/ethereum/src/test/upgrade_utils.ts index 9c4b8511aa73..191002b5aeb4 100644 --- a/yarn-project/ethereum/src/test/upgrade_utils.ts +++ b/yarn-project/ethereum/src/test/upgrade_utils.ts @@ -7,7 +7,7 @@ import { type GetContractReturnType, type PrivateKeyAccount, getContract } from import { extractProposalIdFromLogs } from '../contracts/governance.js'; import type { L1ContractAddresses } from '../l1_contract_addresses.js'; -import { createL1TxUtilsFromViemWallet } from '../l1_tx_utils/index.js'; +import { createL1TxUtils } from '../l1_tx_utils/index.js'; import type { ExtendedViemWalletClient, ViemPublicClient } from '../types.js'; import { EthCheatCodes } from './eth_cheat_codes.js'; @@ -22,7 +22,7 @@ export async function executeGovernanceProposal( ) { const proposal = await governance.read.getProposal([proposalId]); - const l1TxUtils = createL1TxUtilsFromViemWallet(l1Client); + const l1TxUtils = createL1TxUtils(l1Client); const waitL1Block = async () => { await l1TxUtils.sendAndMonitorTransaction({ diff --git a/yarn-project/foundation/eslint-rules/no-async-dispose.js b/yarn-project/foundation/eslint-rules/no-async-dispose.js new file mode 100644 index 000000000000..f8d79ae45ee3 --- /dev/null +++ b/yarn-project/foundation/eslint-rules/no-async-dispose.js @@ -0,0 +1,62 @@ +// @ts-check + +/** + * @fileoverview Rule to disallow async [Symbol.dispose]() methods. + * Use [Symbol.asyncDispose]() with AsyncDisposable instead. + */ + +/** @type {import('eslint').Rule.RuleModule} */ +export default { + meta: { + type: 'problem', + docs: { + description: 'Disallow async [Symbol.dispose]() methods', + category: 'Best Practices', + recommended: true, + }, + messages: { + asyncDispose: + '[Symbol.dispose]() should not be async. Use [Symbol.asyncDispose]() with AsyncDisposable instead.', + }, + schema: [], + }, + + create(context) { + return { + MethodDefinition(node) { + // Match computed property keys like [Symbol.dispose] + if (!node.computed || !node.key || node.key.type !== 'MemberExpression') { + return; + } + + const key = node.key; + if ( + key.object.type !== 'Identifier' || + key.object.name !== 'Symbol' || + key.property.type !== 'Identifier' || + key.property.name !== 'dispose' + ) { + return; + } + + // Check if the method is async + if (node.value.async) { + context.report({ node, messageId: 'asyncDispose' }); + return; + } + + // Check if the return type annotation contains Promise + // @ts-expect-error returnType is a typescript-eslint AST extension + const returnType = node.value.returnType?.typeAnnotation; + if ( + returnType && + returnType.type === 'TSTypeReference' && + returnType.typeName?.type === 'Identifier' && + returnType.typeName.name === 'Promise' + ) { + context.report({ node, messageId: 'asyncDispose' }); + } + }, + }; + }, +}; diff --git a/yarn-project/foundation/eslint.config.js b/yarn-project/foundation/eslint.config.js index a1c756de969d..51b7beb16235 100644 --- a/yarn-project/foundation/eslint.config.js +++ b/yarn-project/foundation/eslint.config.js @@ -8,6 +8,7 @@ import { globalIgnores } from 'eslint/config'; import globals from 'globals'; import tseslint from 'typescript-eslint'; +import noAsyncDispose from './eslint-rules/no-async-dispose.js'; import noNonPrimitiveInCollections from './eslint-rules/no-non-primitive-in-collections.js'; import noUnsafeBrandedTypeConversion from './eslint-rules/no-unsafe-branded-type-conversion.js'; @@ -51,6 +52,7 @@ export default [ importPlugin, 'aztec-custom': { rules: { + 'no-async-dispose': noAsyncDispose, 'no-non-primitive-in-collections': noNonPrimitiveInCollections, 'no-unsafe-branded-type-conversion': noUnsafeBrandedTypeConversion, }, @@ -116,6 +118,7 @@ export default [ 'import-x/no-extraneous-dependencies': 'error', // this unfortunately doesn't block `fit` and `fdescribe` 'no-only-tests/no-only-tests': ['error'], + 'aztec-custom/no-async-dispose': 'error', 'aztec-custom/no-non-primitive-in-collections': 'error', 'aztec-custom/no-unsafe-branded-type-conversion': 'error', }, diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 3ed05d7dbb28..ef1f14bd63b3 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -12,6 +12,9 @@ export type EnvVar = | 'ARCHIVER_VIEM_POLLING_INTERVAL_MS' | 'ARCHIVER_BATCH_SIZE' | 'AZTEC_ADMIN_PORT' + | 'AZTEC_ADMIN_API_KEY_HASH' + | 'AZTEC_NO_ADMIN_API_KEY' + | 'AZTEC_RESET_ADMIN_API_KEY' | 'AZTEC_NODE_ADMIN_URL' | 'AZTEC_NODE_URL' | 'AZTEC_PORT' @@ -67,6 +70,7 @@ export type EnvVar = | 'PUBLIC_DATA_TREE_MAP_SIZE_KB' | 'DEBUG' | 'DEBUG_P2P_DISABLE_COLOCATION_PENALTY' + | 'ENABLE_PROVER_NODE' | 'ETHEREUM_HOSTS' | 'ETHEREUM_DEBUG_HOSTS' | 'ETHEREUM_ALLOW_NO_DEBUG_HOSTS' @@ -215,6 +219,7 @@ export type EnvVar = | 'SEQ_BUILD_CHECKPOINT_IF_EMPTY' | 'SEQ_SECONDS_BEFORE_INVALIDATING_BLOCK_AS_COMMITTEE_MEMBER' | 'SEQ_SECONDS_BEFORE_INVALIDATING_BLOCK_AS_NON_COMMITTEE_MEMBER' + | 'SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT' | 'SLASH_MIN_PENALTY_PERCENTAGE' | 'SLASH_MAX_PENALTY_PERCENTAGE' | 'SLASH_VALIDATORS_ALWAYS' @@ -279,6 +284,7 @@ export type EnvVar = | 'WS_BLOCK_REQUEST_BATCH_SIZE' | 'L1_READER_VIEM_POLLING_INTERVAL_MS' | 'WS_DATA_DIRECTORY' + | 'WS_NUM_HISTORIC_CHECKPOINTS' | 'WS_NUM_HISTORIC_BLOCKS' | 'ETHEREUM_SLOT_DURATION' | 'AZTEC_SLOT_DURATION' diff --git a/yarn-project/foundation/src/json-rpc/client/safe_json_rpc_client.ts b/yarn-project/foundation/src/json-rpc/client/safe_json_rpc_client.ts index 926b227b7cdf..b10c6535f8ed 100644 --- a/yarn-project/foundation/src/json-rpc/client/safe_json_rpc_client.ts +++ b/yarn-project/foundation/src/json-rpc/client/safe_json_rpc_client.ts @@ -24,6 +24,7 @@ export type SafeJsonRpcClientOptions = { batchWindowMS?: number; maxBatchSize?: number; maxRequestBodySize?: number; + extraHeaders?: Record; onResponse?: (res: { response: any; headers: { get: (header: string) => string | null | undefined }; @@ -129,6 +130,7 @@ export function createSafeJsonRpcClient( const { response, headers } = await fetch( host, rpcCalls.map(({ request }) => request), + config.extraHeaders, ); if (config.onResponse) { diff --git a/yarn-project/foundation/src/json-rpc/server/api_key_auth.integration.test.ts b/yarn-project/foundation/src/json-rpc/server/api_key_auth.integration.test.ts new file mode 100644 index 000000000000..fb9e6077480e --- /dev/null +++ b/yarn-project/foundation/src/json-rpc/server/api_key_auth.integration.test.ts @@ -0,0 +1,140 @@ +import type http from 'http'; + +import { makeFetch } from '../client/fetch.js'; +import { createSafeJsonRpcClient } from '../client/safe_json_rpc_client.js'; +import { TestNote, TestState, type TestStateApi, TestStateSchema } from '../fixtures/test_state.js'; +import { getApiKeyAuthMiddleware, sha256Hash } from './api_key_auth.js'; +import { createSafeJsonRpcServer, startHttpRpcServer } from './safe_json_rpc_server.js'; + +describe('API key auth integration', () => { + const RAW_API_KEY = 'integration-test-api-key-0123456789abcdef0123456789abcdef'; + const API_KEY_HASH = sha256Hash(RAW_API_KEY); + + let testState: TestState; + let httpServer: http.Server & { port: number }; + let url: string; + + beforeEach(async () => { + testState = new TestState([new TestNote('a'), new TestNote('b')]); + const rpcServer = createSafeJsonRpcServer(testState, TestStateSchema, { + middlewares: [getApiKeyAuthMiddleware(API_KEY_HASH)], + }); + httpServer = await startHttpRpcServer(rpcServer, { host: '127.0.0.1' }); + url = `http://127.0.0.1:${httpServer.port}`; + }); + + afterEach(() => { + httpServer?.close(); + }); + + const noRetryFetch = makeFetch([], true); + + function createClient(apiKey?: string) { + return createSafeJsonRpcClient(url, TestStateSchema, { + fetch: noRetryFetch, + ...(apiKey ? { extraHeaders: { 'x-api-key': apiKey } } : {}), + }); + } + + function createClientWithBearer(apiKey: string) { + return createSafeJsonRpcClient(url, TestStateSchema, { + fetch: noRetryFetch, + extraHeaders: { Authorization: `Bearer ${apiKey}` }, + }); + } + + describe('with valid API key', () => { + it('allows RPC calls via x-api-key header', async () => { + const client = createClient(RAW_API_KEY); + const count = await client.count(); + expect(count).toBe(2); + }); + + it('allows RPC calls via Authorization: Bearer header', async () => { + const client = createClientWithBearer(RAW_API_KEY); + const note = await client.getNote(0); + expect(note?.toString()).toBe('a'); + }); + + it('allows multiple sequential calls', async () => { + const client = createClient(RAW_API_KEY); + const count1 = await client.count(); + await client.addNotes([new TestNote('c')]); + const count2 = await client.count(); + expect(count1).toBe(2); + expect(count2).toBe(3); + }); + }); + + describe('with invalid API key', () => { + it('rejects RPC calls with wrong key', async () => { + const client = createClient('wrong-api-key'); + await expect(client.count()).rejects.toThrow(); + }); + + it('rejects RPC calls with empty key header', async () => { + const client = createClient(''); + await expect(client.count()).rejects.toThrow(); + }); + }); + + describe('with no API key', () => { + it('rejects RPC calls without any auth header', async () => { + const client = createClient(); // no key + await expect(client.count()).rejects.toThrow(); + }); + }); + + describe('health check bypass', () => { + it('allows GET /status without auth', async () => { + const response = await fetch(`${url}/status`); + expect(response.status).toBe(200); + }); + }); + + describe('full flow: generate key, authenticate, reject bad key', () => { + it('simulates the operator flow end-to-end', async () => { + // 1: "Generate" an API key (simulating what resolveAdminApiKey does) + const { randomBytes } = await import('crypto'); + const generatedKey = randomBytes(32).toString('hex'); + const generatedHash = sha256Hash(generatedKey); + + // 2: Start a NEW server with the generated hash + const freshState = new TestState([new TestNote('x'), new TestNote('y')]); + const freshRpcServer = createSafeJsonRpcServer(freshState, TestStateSchema, { + middlewares: [getApiKeyAuthMiddleware(generatedHash)], + }); + const freshHttpServer = await startHttpRpcServer(freshRpcServer, { host: '127.0.0.1' }); + const freshUrl = `http://127.0.0.1:${freshHttpServer.port}`; + + try { + // 3: Make an authenticated request — should succeed + const goodClient = createSafeJsonRpcClient(freshUrl, TestStateSchema, { + fetch: noRetryFetch, + extraHeaders: { 'x-api-key': generatedKey }, + }); + const count = await goodClient.count(); + expect(count).toBe(2); + + // 4: Make a request with a bad key, should fail + const badClient = createSafeJsonRpcClient(freshUrl, TestStateSchema, { + fetch: noRetryFetch, + extraHeaders: { 'x-api-key': 'definitely-not-the-right-key' }, + }); + await expect(badClient.count()).rejects.toThrow(); + + // 5: Make a request with no key, should fail + const noAuthClient = createSafeJsonRpcClient(freshUrl, TestStateSchema, { + fetch: noRetryFetch, + }); + await expect(noAuthClient.count()).rejects.toThrow(); + + // 6: Health check should still work without auth + const statusResp = await fetch(`${freshUrl}/status`); + expect(statusResp.status).toBe(200); + } finally { + freshHttpServer.close(); + } + }); + }); +}); diff --git a/yarn-project/foundation/src/json-rpc/server/api_key_auth.test.ts b/yarn-project/foundation/src/json-rpc/server/api_key_auth.test.ts new file mode 100644 index 000000000000..5d0ad7e826ba --- /dev/null +++ b/yarn-project/foundation/src/json-rpc/server/api_key_auth.test.ts @@ -0,0 +1,91 @@ +import Koa from 'koa'; +import request from 'supertest'; + +import { getApiKeyAuthMiddleware, sha256Hash } from './api_key_auth.js'; + +describe('getApiKeyAuthMiddleware', () => { + const RAW_API_KEY = 'test-api-key-for-unit-tests-1234567890abcdef'; + const API_KEY_HASH = sha256Hash(RAW_API_KEY); + + let app: Koa; + + beforeEach(() => { + app = new Koa(); + app.use(getApiKeyAuthMiddleware(API_KEY_HASH)); + // A simple handler that returns 200 if middleware passes + app.use((ctx: Koa.Context) => { + ctx.status = 200; + ctx.body = { jsonrpc: '2.0', result: 'ok' }; + }); + }); + + const sendPost = (headers: Record = {}) => + request(app.callback()) + .post('/') + .send({ jsonrpc: '2.0', method: 'test', params: [], id: 1 }) + .set({ 'content-type': 'application/json', ...headers }); + + describe('x-api-key header', () => { + it('allows request with valid API key', async () => { + const response = await sendPost({ 'x-api-key': RAW_API_KEY }); + expect(response.status).toBe(200); + expect(response.body.result).toBe('ok'); + }); + + it('rejects request with invalid API key', async () => { + const response = await sendPost({ 'x-api-key': 'wrong-key' }); + expect(response.status).toBe(401); + expect(response.body.error.message).toContain('Unauthorized'); + }); + }); + + describe('Authorization: Bearer header', () => { + it('allows request with valid Bearer token', async () => { + const response = await sendPost({ Authorization: `Bearer ${RAW_API_KEY}` }); + expect(response.status).toBe(200); + expect(response.body.result).toBe('ok'); + }); + + it('allows case-insensitive Bearer prefix', async () => { + const response = await sendPost({ Authorization: `bearer ${RAW_API_KEY}` }); + expect(response.status).toBe(200); + }); + + it('rejects request with invalid Bearer token', async () => { + const response = await sendPost({ Authorization: 'Bearer wrong-key' }); + expect(response.status).toBe(401); + expect(response.body.error.message).toContain('Unauthorized'); + }); + }); + + describe('missing credentials', () => { + it('rejects request with no auth headers', async () => { + const response = await sendPost(); + expect(response.status).toBe(401); + expect(response.body.error.message).toContain('Unauthorized'); + }); + + it('returns a JSON-RPC error envelope', async () => { + const response = await sendPost(); + expect(response.body).toMatchObject({ + jsonrpc: '2.0', + id: null, + error: { code: -32000 }, + }); + }); + }); + + describe('health check bypass', () => { + it('allows GET /status without any auth', async () => { + const response = await request(app.callback()).get('/status'); + // The status endpoint itself isn't handled by our simple handler, + // but the important thing is the middleware does NOT return 401. + expect(response.status).not.toBe(401); + }); + + it('still requires auth for POST /status', async () => { + const response = await request(app.callback()).post('/status').send({}); + expect(response.status).toBe(401); + }); + }); +}); diff --git a/yarn-project/foundation/src/json-rpc/server/api_key_auth.ts b/yarn-project/foundation/src/json-rpc/server/api_key_auth.ts new file mode 100644 index 000000000000..a22d45730f32 --- /dev/null +++ b/yarn-project/foundation/src/json-rpc/server/api_key_auth.ts @@ -0,0 +1,63 @@ +import { timingSafeEqual } from 'crypto'; +import type Koa from 'koa'; + +import { sha256 } from '../../crypto/sha256/index.js'; +import { createLogger } from '../../log/index.js'; + +const log = createLogger('json-rpc:api-key-auth'); + +/** + * Computes the SHA-256 hash of a string and returns it as a Buffer. + * @param input - The input string to hash. + * @returns The SHA-256 hash as a Buffer. + */ +export function sha256Hash(input: string): Buffer { + return sha256(Buffer.from(input)); +} + +/** + * Creates a Koa middleware that enforces API key authentication on all requests + * except the health check endpoint (GET /status). + * + * The API key can be provided via the `x-api-key` header or the `Authorization: Bearer ` header. + * Comparison is done by hashing the provided key with SHA-256 and comparing against the stored hash. + * + * @param apiKeyHash - The SHA-256 hash of the expected API key as a Buffer. + * @returns A Koa middleware that rejects requests without a valid API key. + */ +export function getApiKeyAuthMiddleware( + apiKeyHash: Buffer, +): (ctx: Koa.Context, next: () => Promise) => Promise { + return async (ctx: Koa.Context, next: () => Promise) => { + // Allow health check through without auth + if (ctx.path === '/status' && ctx.method === 'GET') { + return next(); + } + + const providedKey = ctx.get('x-api-key') || ctx.get('authorization')?.replace(/^Bearer\s+/i, ''); + if (!providedKey) { + log.warn(`Rejected admin RPC request from ${ctx.ip}: missing API key`); + ctx.status = 401; + ctx.body = { + jsonrpc: '2.0', + id: null, + error: { code: -32000, message: 'Unauthorized: invalid or missing API key' }, + }; + return; + } + + const providedHashBuf = sha256Hash(providedKey); + if (!timingSafeEqual(apiKeyHash, providedHashBuf)) { + log.warn(`Rejected admin RPC request from ${ctx.ip}: invalid API key`); + ctx.status = 401; + ctx.body = { + jsonrpc: '2.0', + id: null, + error: { code: -32000, message: 'Unauthorized: invalid or missing API key' }, + }; + return; + } + + await next(); + }; +} diff --git a/yarn-project/foundation/src/json-rpc/server/index.ts b/yarn-project/foundation/src/json-rpc/server/index.ts index 048e53af1fcc..2e35821006ed 100644 --- a/yarn-project/foundation/src/json-rpc/server/index.ts +++ b/yarn-project/foundation/src/json-rpc/server/index.ts @@ -1 +1,2 @@ +export * from './api_key_auth.js'; export * from './safe_json_rpc_server.js'; diff --git a/yarn-project/foundation/src/log/bigint-utils.ts b/yarn-project/foundation/src/log/bigint-utils.ts new file mode 100644 index 000000000000..6cc94101ac2f --- /dev/null +++ b/yarn-project/foundation/src/log/bigint-utils.ts @@ -0,0 +1,22 @@ +/** + * Converts bigint values to strings recursively in a log object to avoid serialization issues. + */ +export function convertBigintsToStrings(obj: unknown): unknown { + if (typeof obj === 'bigint') { + return String(obj); + } + + if (Array.isArray(obj)) { + return obj.map(item => convertBigintsToStrings(item)); + } + + if (obj !== null && typeof obj === 'object') { + const result: Record = {}; + for (const key in obj) { + result[key] = convertBigintsToStrings((obj as Record)[key]); + } + return result; + } + + return obj; +} diff --git a/yarn-project/foundation/src/log/gcloud-logger-config.ts b/yarn-project/foundation/src/log/gcloud-logger-config.ts index db3b331141df..2e036212af71 100644 --- a/yarn-project/foundation/src/log/gcloud-logger-config.ts +++ b/yarn-project/foundation/src/log/gcloud-logger-config.ts @@ -1,5 +1,7 @@ import type { pino } from 'pino'; +import { convertBigintsToStrings } from './bigint-utils.js'; + /* eslint-disable camelcase */ const GOOGLE_CLOUD_TRACE_ID = 'logging.googleapis.com/trace'; @@ -15,6 +17,9 @@ export const GoogleCloudLoggerConfig = { messageKey: 'message', formatters: { log(object: Record): Record { + // Convert bigints to strings recursively to avoid serialization issues + object = convertBigintsToStrings(object) as Record; + // Add trace context attributes following Cloud Logging structured log format described // in https://cloud.google.com/logging/docs/structured-logging#special-payload-fields const { trace_id, span_id, trace_flags, ...rest } = object; diff --git a/yarn-project/foundation/src/log/pino-logger.test.ts b/yarn-project/foundation/src/log/pino-logger.test.ts index 1efdcf038f18..9881535d4f58 100644 --- a/yarn-project/foundation/src/log/pino-logger.test.ts +++ b/yarn-project/foundation/src/log/pino-logger.test.ts @@ -188,6 +188,91 @@ describe('pino-logger', () => { }); }); + it('converts bigints to strings recursively ', () => { + const testLogger = createLogger('bigint-test'); + capturingStream.clear(); + + testLogger.info('comprehensive bigint conversion', { + // Top-level bigints + amount: 123456789012345678901234n, + slot: 42n, + // Nested objects + nested: { + value: 999999999999999999n, + deepNested: { + id: 12345678901234567890n, + }, + }, + // Arrays with bigints + array: [1n, 2n, 3n], + mixedArray: [{ id: 999n }, { id: 888n }], + // Mixed types + numberValue: 123, + stringValue: 'test', + boolValue: true, + nullValue: null, + }); + + const entries = capturingStream.getJsonLines(); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + module: 'bigint-test', + msg: 'comprehensive bigint conversion', + // All bigints converted to strings + amount: '123456789012345678901234', + slot: '42', + nested: { + value: '999999999999999999', + deepNested: { + id: '12345678901234567890', + }, + }, + array: ['1', '2', '3'], + mixedArray: [{ id: '999' }, { id: '888' }], + // Other types preserved + numberValue: 123, + stringValue: 'test', + boolValue: true, + nullValue: null, + }); + }); + + it('does not mutate the original log data object', () => { + const testLogger = createLogger('mutation-test'); + capturingStream.clear(); + + const originalData = { + amount: 123456789012345678901234n, + nested: { + value: 999n, + }, + array: [1n, 2n, 3n], + }; + + // Keep references to verify mutation + const originalAmount = originalData.amount; + const originalNestedValue = originalData.nested.value; + const originalArrayItem = originalData.array[0]; + + testLogger.info('mutation test', originalData); + + // Verify the original object was NOT mutated + expect(originalData.amount).toBe(originalAmount); + expect(typeof originalData.amount).toBe('bigint'); + expect(originalData.nested.value).toBe(originalNestedValue); + expect(typeof originalData.nested.value).toBe('bigint'); + expect(originalData.array[0]).toBe(originalArrayItem); + expect(typeof originalData.array[0]).toBe('bigint'); + + // But the logged version should have strings + const entries = capturingStream.getJsonLines(); + expect(entries[0]).toMatchObject({ + amount: '123456789012345678901234', + nested: { value: '999' }, + array: ['1', '2', '3'], + }); + }); + it('returns bindings via getBindings', () => { const testLogger = createLogger('bindings-test', { actor: 'main', instanceId: 'id-123' }); const bindings = testLogger.getBindings(); diff --git a/yarn-project/foundation/src/log/pino-logger.ts b/yarn-project/foundation/src/log/pino-logger.ts index 21427d154b22..f5d97c22f1b8 100644 --- a/yarn-project/foundation/src/log/pino-logger.ts +++ b/yarn-project/foundation/src/log/pino-logger.ts @@ -7,6 +7,7 @@ import { inspect } from 'util'; import { compactArray } from '../collection/array.js'; import type { EnvVar } from '../config/index.js'; import { parseBooleanEnv } from '../config/parse-env.js'; +import { convertBigintsToStrings } from './bigint-utils.js'; import { GoogleCloudLoggerConfig } from './gcloud-logger-config.js'; import { getLogLevelFromFilters, parseLogLevelEnvVar } from './log-filters.js'; import type { LogLevel } from './log-levels.js'; @@ -165,6 +166,9 @@ const pinoOpts: pino.LoggerOptions = { ...redactedPaths.map(p => `opts.${p}`), ], }, + formatters: { + log: obj => convertBigintsToStrings(obj) as Record, + }, ...(useGcloudLogging ? GoogleCloudLoggerConfig : {}), }; diff --git a/yarn-project/node-lib/src/factories/l1_tx_utils.ts b/yarn-project/node-lib/src/factories/l1_tx_utils.ts index 7e1a54c3cfb0..d3a441c66bb9 100644 --- a/yarn-project/node-lib/src/factories/l1_tx_utils.ts +++ b/yarn-project/node-lib/src/factories/l1_tx_utils.ts @@ -1,17 +1,11 @@ +import type { BlobKzgInstance } from '@aztec/blob-lib/types'; import type { EthSigner } from '@aztec/ethereum/eth-signer'; -import { - createL1TxUtilsFromEthSigner as createL1TxUtilsFromEthSignerBase, - createL1TxUtilsFromViemWallet as createL1TxUtilsFromViemWalletBase, -} from '@aztec/ethereum/l1-tx-utils'; +import { createDelayer, createL1TxUtils as createL1TxUtilsBase } from '@aztec/ethereum/l1-tx-utils'; import type { L1TxUtilsConfig } from '@aztec/ethereum/l1-tx-utils'; -import { - createForwarderL1TxUtilsFromEthSigner as createForwarderL1TxUtilsFromEthSignerBase, - createForwarderL1TxUtilsFromViemWallet as createForwarderL1TxUtilsFromViemWalletBase, - createL1TxUtilsWithBlobsFromEthSigner as createL1TxUtilsWithBlobsFromEthSignerBase, - createL1TxUtilsWithBlobsFromViemWallet as createL1TxUtilsWithBlobsFromViemWalletBase, -} from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import { createForwarderL1TxUtils as createForwarderL1TxUtilsBase } from '@aztec/ethereum/l1-tx-utils-with-blobs'; import type { ExtendedViemWalletClient, ViemClient } from '@aztec/ethereum/types'; import { omit } from '@aztec/foundation/collection'; +import type { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import type { DateProvider } from '@aztec/foundation/timer'; import type { DataStoreConfig } from '@aztec/kv-store/config'; @@ -25,14 +19,15 @@ import { L1TxStore } from '../stores/l1_tx_store.js'; const L1_TX_STORE_NAME = 'l1-tx-utils'; /** - * Creates shared dependencies (logger, store, metrics) for L1TxUtils instances. + * Creates shared dependencies (logger, store, metrics, delayer) for L1TxUtils instances. + * When enableDelayer is set in config, a single shared delayer is created and passed to all instances. */ async function createSharedDeps( - config: DataStoreConfig & { scope?: L1TxScope }, + config: DataStoreConfig & Partial & { scope?: L1TxScope }, deps: { telemetry: TelemetryClient; logger?: ReturnType; - dateProvider?: DateProvider; + dateProvider: DateProvider; }, ) { const logger = deps.logger ?? createLogger('l1-tx-utils'); @@ -46,96 +41,48 @@ async function createSharedDeps( const meter = deps.telemetry.getMeter('L1TxUtils'); const metrics = new L1TxMetrics(meter, config.scope ?? 'other', logger); - return { logger, store, metrics, dateProvider: deps.dateProvider }; -} - -/** - * Creates L1TxUtils with blobs from multiple Viem wallets, sharing store and metrics. - */ -export async function createL1TxUtilsWithBlobsFromViemWallet( - clients: ExtendedViemWalletClient[], - config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, - deps: { - telemetry: TelemetryClient; - logger?: ReturnType; - dateProvider?: DateProvider; - }, -) { - const sharedDeps = await createSharedDeps(config, deps); + // Create a single shared delayer for all L1TxUtils instances in this group + const delayer = + config.enableDelayer && config.ethereumSlotDuration !== undefined + ? createDelayer(deps.dateProvider, { ethereumSlotDuration: config.ethereumSlotDuration }, logger.getBindings()) + : undefined; - return clients.map(client => - createL1TxUtilsWithBlobsFromViemWalletBase(client, sharedDeps, config, config.debugMaxGasLimit), - ); -} - -/** - * Creates L1TxUtils with blobs from multiple EthSigners, sharing store and metrics. Removes duplicates - */ -export async function createL1TxUtilsWithBlobsFromEthSigner( - client: ViemClient, - signers: EthSigner[], - config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, - deps: { - telemetry: TelemetryClient; - logger?: ReturnType; - dateProvider?: DateProvider; - }, -) { - const sharedDeps = await createSharedDeps(config, deps); - - // Deduplicate signers by address to avoid creating multiple L1TxUtils instances - // for the same publisher address (e.g., when multiple attesters share the same publisher key) - const signersByAddress = new Map(); - for (const signer of signers) { - const addressKey = signer.address.toString().toLowerCase(); - if (!signersByAddress.has(addressKey)) { - signersByAddress.set(addressKey, signer); - } - } - - const uniqueSigners = Array.from(signersByAddress.values()); - - if (uniqueSigners.length < signers.length) { - sharedDeps.logger.info( - `Deduplicated ${signers.length} signers to ${uniqueSigners.length} unique publisher addresses`, - ); - } - - return uniqueSigners.map(signer => - createL1TxUtilsWithBlobsFromEthSignerBase(client, signer, sharedDeps, config, config.debugMaxGasLimit), - ); + return { logger, store, metrics, dateProvider: deps.dateProvider, delayer }; } /** - * Creates L1TxUtils (without blobs) from multiple Viem wallets, sharing store and metrics. + * Creates L1TxUtils from multiple Viem wallet clients, sharing store, metrics, and delayer. + * When kzg is provided in deps, blob support is enabled. */ -export async function createL1TxUtilsFromViemWalletWithStore( +export async function createL1TxUtilsFromWallets( clients: ExtendedViemWalletClient[], config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, deps: { telemetry: TelemetryClient; logger?: ReturnType; - dateProvider?: DateProvider; - scope?: L1TxScope; + dateProvider: DateProvider; + kzg?: BlobKzgInstance; }, ) { const sharedDeps = await createSharedDeps(config, deps); - return clients.map(client => createL1TxUtilsFromViemWalletBase(client, sharedDeps, config)); + return clients.map(client => createL1TxUtilsBase(client, { ...sharedDeps, kzg: deps.kzg }, config)); } /** - * Creates L1TxUtils (without blobs) from multiple EthSigners, sharing store and metrics. Removes duplicates. + * Creates L1TxUtils from multiple EthSigners, sharing store, metrics, and delayer. + * When kzg is provided in deps, blob support is enabled. + * Deduplicates signers by address to avoid creating multiple instances for the same publisher. */ -export async function createL1TxUtilsFromEthSignerWithStore( +export async function createL1TxUtilsFromSigners( client: ViemClient, signers: EthSigner[], config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, deps: { telemetry: TelemetryClient; logger?: ReturnType; - dateProvider?: DateProvider; - scope?: L1TxScope; + dateProvider: DateProvider; + kzg?: BlobKzgInstance; }, ) { const sharedDeps = await createSharedDeps(config, deps); @@ -158,55 +105,52 @@ export async function createL1TxUtilsFromEthSignerWithStore( ); } - return uniqueSigners.map(signer => createL1TxUtilsFromEthSignerBase(client, signer, sharedDeps, config)); + return uniqueSigners.map(signer => createL1TxUtilsBase({ client, signer }, { ...sharedDeps, kzg: deps.kzg }, config)); } /** - * Creates ForwarderL1TxUtils from multiple Viem wallets, sharing store and metrics. - * This wraps all transactions through a forwarder contract for testing purposes. + * Creates ForwarderL1TxUtils from multiple Viem wallet clients, sharing store, metrics, and delayer. + * Wraps all transactions through a forwarder contract for testing purposes. + * When kzg is provided in deps, blob support is enabled. */ -export async function createForwarderL1TxUtilsFromViemWallet( +export async function createForwarderL1TxUtilsFromWallets( clients: ExtendedViemWalletClient[], - forwarderAddress: import('@aztec/foundation/eth-address').EthAddress, + forwarderAddress: EthAddress, config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, deps: { telemetry: TelemetryClient; logger?: ReturnType; - dateProvider?: DateProvider; + dateProvider: DateProvider; + kzg?: BlobKzgInstance; }, ) { const sharedDeps = await createSharedDeps(config, deps); return clients.map(client => - createForwarderL1TxUtilsFromViemWalletBase(client, forwarderAddress, sharedDeps, config, config.debugMaxGasLimit), + createForwarderL1TxUtilsBase(client, forwarderAddress, { ...sharedDeps, kzg: deps.kzg }, config), ); } /** - * Creates ForwarderL1TxUtils from multiple EthSigners, sharing store and metrics. - * This wraps all transactions through a forwarder contract for testing purposes. + * Creates ForwarderL1TxUtils from multiple EthSigners, sharing store, metrics, and delayer. + * Wraps all transactions through a forwarder contract for testing purposes. + * When kzg is provided in deps, blob support is enabled. */ -export async function createForwarderL1TxUtilsFromEthSigner( +export async function createForwarderL1TxUtilsFromSigners( client: ViemClient, signers: EthSigner[], - forwarderAddress: import('@aztec/foundation/eth-address').EthAddress, + forwarderAddress: EthAddress, config: DataStoreConfig & Partial & { debugMaxGasLimit?: boolean; scope?: L1TxScope }, deps: { telemetry: TelemetryClient; logger?: ReturnType; - dateProvider?: DateProvider; + dateProvider: DateProvider; + kzg?: BlobKzgInstance; }, ) { const sharedDeps = await createSharedDeps(config, deps); return signers.map(signer => - createForwarderL1TxUtilsFromEthSignerBase( - client, - signer, - forwarderAddress, - sharedDeps, - config, - config.debugMaxGasLimit, - ), + createForwarderL1TxUtilsBase({ client, signer }, forwarderAddress, { ...sharedDeps, kzg: deps.kzg }, config), ); } diff --git a/yarn-project/node-lib/src/factories/l1_tx_utils_integration.test.ts b/yarn-project/node-lib/src/factories/l1_tx_utils_integration.test.ts index 1d68d0b35aae..abbf3259ef21 100644 --- a/yarn-project/node-lib/src/factories/l1_tx_utils_integration.test.ts +++ b/yarn-project/node-lib/src/factories/l1_tx_utils_integration.test.ts @@ -1,7 +1,9 @@ +import { Blob } from '@aztec/blob-lib'; import { getAddressFromPrivateKey } from '@aztec/ethereum/account'; import type { ViemClient } from '@aztec/ethereum/types'; import { times } from '@aztec/foundation/collection'; import { EthAddress } from '@aztec/foundation/eth-address'; +import type { DateProvider } from '@aztec/foundation/timer'; import type { AztecAsyncKVStore } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { KeystoreManager } from '@aztec/node-keystore'; @@ -11,12 +13,13 @@ import type { TelemetryClient } from '@aztec/telemetry-client'; import { generatePrivateKey } from 'viem/accounts'; -import { createL1TxUtilsWithBlobsFromEthSigner } from './l1_tx_utils.js'; +import { createL1TxUtilsFromSigners } from './l1_tx_utils.js'; describe('L1TxUtils Integration - Publisher Deduplication', () => { let kvStore: AztecAsyncKVStore; let mockClient: ViemClient; let mockTelemetry: TelemetryClient; + let mockDateProvider: DateProvider; let count = 0; const mockConfig = { @@ -44,6 +47,9 @@ describe('L1TxUtils Integration - Publisher Deduplication', () => { createUpDownCounter: () => ({ add: () => {} }), }), } as any; + + // Mock DateProvider + mockDateProvider = { now: () => Date.now(), nowInSeconds: () => Math.floor(Date.now() / 1000) } as DateProvider; }); afterEach(async () => { @@ -80,8 +86,10 @@ describe('L1TxUtils Integration - Publisher Deduplication', () => { // we should have publishers for each validator expect(allPublisherSigners).toHaveLength(keystore.validators!.length); - const l1TxUtils = await createL1TxUtilsWithBlobsFromEthSigner(mockClient, allPublisherSigners, mockConfig, { + const l1TxUtils = await createL1TxUtilsFromSigners(mockClient, allPublisherSigners, mockConfig, { telemetry: mockTelemetry, + dateProvider: mockDateProvider, + kzg: Blob.getViemKzgInstance(), }); // all of the publisherSigners should deduplicate to one L1TxUtils instance @@ -139,8 +147,10 @@ describe('L1TxUtils Integration - Publisher Deduplication', () => { expect(allPublisherSigners).toHaveLength(keystore.validators!.length); - const l1TxUtils = await createL1TxUtilsWithBlobsFromEthSigner(mockClient, allPublisherSigners, mockConfig, { + const l1TxUtils = await createL1TxUtilsFromSigners(mockClient, allPublisherSigners, mockConfig, { telemetry: mockTelemetry, + dateProvider: mockDateProvider, + kzg: Blob.getViemKzgInstance(), }); expect(l1TxUtils).toHaveLength(3); diff --git a/yarn-project/noir-protocol-circuits-types/src/utils/server/foreign_call_handler.ts b/yarn-project/noir-protocol-circuits-types/src/utils/server/foreign_call_handler.ts index ccc3d73ee1c5..33e40d554ee6 100644 --- a/yarn-project/noir-protocol-circuits-types/src/utils/server/foreign_call_handler.ts +++ b/yarn-project/noir-protocol-circuits-types/src/utils/server/foreign_call_handler.ts @@ -94,7 +94,7 @@ export async function foreignCallHandler(name: string, args: ForeignCallInput[]) ); } - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); blobs.forEach((blob, i) => { const injected = kzgCommitments[i]; const calculated = BLS12Point.decompress(blob.commitment); diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 1a4a813c5fa5..08d1dd209343 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -7,7 +7,7 @@ import { AztecLMDBStoreV2, createStore } from '@aztec/kv-store/lmdb-v2'; import type { BlockHash, L2BlockSource } from '@aztec/stdlib/block'; import type { ChainConfig } from '@aztec/stdlib/config'; import type { ContractDataSource } from '@aztec/stdlib/contract'; -import type { ClientProtocolCircuitVerifier, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; +import type { AztecNode, ClientProtocolCircuitVerifier, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; import { P2PClientType } from '@aztec/stdlib/p2p'; import { MerkleTreeId } from '@aztec/stdlib/trees'; import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client'; @@ -26,7 +26,7 @@ import { DummyP2PService } from '../services/dummy_service.js'; import { LibP2PService } from '../services/index.js'; import { createFileStoreTxSources } from '../services/tx_collection/file_store_tx_source.js'; import { TxCollection } from '../services/tx_collection/tx_collection.js'; -import { type TxSource, createNodeRpcTxSources } from '../services/tx_collection/tx_source.js'; +import { NodeRpcTxSource, type TxSource, createNodeRpcTxSources } from '../services/tx_collection/tx_source.js'; import { TxFileStore } from '../services/tx_file_store/tx_file_store.js'; import { configureP2PClientAddresses, createLibP2PPeerIdFromPrivateKey, getPeerIdPrivateKey } from '../util.js'; @@ -36,6 +36,7 @@ export type P2PClientDeps = { attestationPool?: AttestationPoolApi; logger?: Logger; txCollectionNodeSources?: TxSource[]; + rpcTxProviders?: AztecNode[]; p2pServiceFactory?: (...args: Parameters<(typeof LibP2PService)['new']>) => Promise>; }; @@ -147,6 +148,7 @@ export async function createP2PClient( const nodeSources = [ ...createNodeRpcTxSources(config.txCollectionNodeRpcUrls, config), + ...(deps.rpcTxProviders ?? []).map((node, i) => new NodeRpcTxSource(node, `node-rpc-provider-${i}`)), ...(deps.txCollectionNodeSources ?? []), ]; if (nodeSources.length > 0) { @@ -159,6 +161,7 @@ export async function createP2PClient( config.txCollectionFileStoreUrls, txFileStoreBasePath, logger.createChild('file-store-tx-source'), + telemetry, ); if (fileStoreSources.length > 0) { logger.info(`Using ${fileStoreSources.length} file store sources for tx collection.`, { diff --git a/yarn-project/p2p/src/client/interface.ts b/yarn-project/p2p/src/client/interface.ts index f70316f88715..9585dd6b89e8 100644 --- a/yarn-project/p2p/src/client/interface.ts +++ b/yarn-project/p2p/src/client/interface.ts @@ -1,6 +1,6 @@ import type { SlotNumber } from '@aztec/foundation/branded-types'; import type { EthAddress, L2BlockId } from '@aztec/stdlib/block'; -import type { P2PApiFull } from '@aztec/stdlib/interfaces/server'; +import type { ITxProvider, P2PApiFull } from '@aztec/stdlib/interfaces/server'; import type { BlockProposal, CheckpointAttestation, CheckpointProposal, P2PClientType } from '@aztec/stdlib/p2p'; import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx'; @@ -107,13 +107,6 @@ export type P2P = P2PApiFull & **/ sendTx(tx: Tx): Promise; - /** - * Adds transactions to the pool. Does not send to peers or validate the tx. - * @param txs - The transactions. - * @returns The number of txs added to the pool. Note if the transaction already exists, it will not be added again. - **/ - addTxsToPool(txs: Tx[]): Promise; - /** * Handles failed transaction execution by removing txs from the pool. * @param txHashes - Hashes of the transactions that failed execution. @@ -220,6 +213,9 @@ export type P2P = P2PApiFull & /** Identifies a p2p client. */ isP2PClient(): true; + /** Returns the tx provider used for fetching transactions. */ + getTxProvider(): ITxProvider; + updateP2PConfig(config: Partial): Promise; /** Validates a set of txs. */ diff --git a/yarn-project/p2p/src/client/p2p_client.test.ts b/yarn-project/p2p/src/client/p2p_client.test.ts index 25200df67014..c4ffc3fa35e6 100644 --- a/yarn-project/p2p/src/client/p2p_client.test.ts +++ b/yarn-project/p2p/src/client/p2p_client.test.ts @@ -133,6 +133,29 @@ describe('P2P Client', () => { await client.stop(); }); + it('throws TxPoolError with structured reason when pool rejects tx', async () => { + await client.start(); + const tx1 = await mockTx(); + const txHashStr = tx1.getTxHash().toString(); + const errors = new Map(); + errors.set(txHashStr, { + code: 'LOW_PRIORITY_FEE', + message: 'Tx does not meet minimum priority fee', + minimumPriorityFee: 101n, + txPriorityFee: 50n, + }); + txPool.addPendingTxs.mockResolvedValueOnce({ + accepted: [], + ignored: [tx1.getTxHash()], + rejected: [], + errors, + }); + + await expect(client.sendTx(tx1)).rejects.toThrow('Tx does not meet minimum priority fee'); + expect(p2pService.propagate).not.toHaveBeenCalled(); + await client.stop(); + }); + it('rejects txs after being stopped', async () => { await client.start(); const tx1 = await mockTx(); diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 588eccbc2269..4851b7bf7bbd 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -32,6 +32,7 @@ import type { PeerId } from '@libp2p/interface'; import type { ENR } from '@nethermindeth/enr'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; +import { TxPoolError } from '../errors/tx-pool.error.js'; import type { AttestationPoolApi } from '../mem_pools/attestation_pool/attestation_pool.js'; import type { MemPools } from '../mem_pools/interface.js'; import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js'; @@ -582,23 +583,22 @@ export class P2PClient **/ public async sendTx(tx: Tx): Promise { this.#assertIsReady(); - const result = await this.txPool.addPendingTxs([tx]); + const result = await this.txPool.addPendingTxs([tx], { feeComparisonOnly: true }); if (result.accepted.length === 1) { await this.p2pService.propagate(tx); - } else { - this.log.warn( - `Tx ${tx.getTxHash()} not propagated: accepted=${result.accepted.length} ignored=${result.ignored.length} rejected=${result.rejected.length}`, - ); + return; } - } - /** - * Adds transactions to the pool. Does not send to peers or validate the txs. - * @param txs - The transactions. - **/ - public async addTxsToPool(txs: Tx[]): Promise { - this.#assertIsReady(); - return (await this.txPool.addPendingTxs(txs)).accepted.length; + const txHashStr = tx.getTxHash().toString(); + const reason = result.errors?.get(txHashStr); + if (reason) { + this.log.warn(`Tx ${txHashStr} not added to pool: ${reason.message}`); + throw new TxPoolError(reason); + } + + this.log.warn( + `Tx ${txHashStr} not propagated: accepted=${result.accepted.length} ignored=${result.ignored.length} rejected=${result.rejected.length}`, + ); } /** diff --git a/yarn-project/p2p/src/errors/tx-pool.error.ts b/yarn-project/p2p/src/errors/tx-pool.error.ts new file mode 100644 index 000000000000..1b59099757e4 --- /dev/null +++ b/yarn-project/p2p/src/errors/tx-pool.error.ts @@ -0,0 +1,12 @@ +import type { TxPoolRejectionError } from '../mem_pools/tx_pool_v2/eviction/interfaces.js'; + +/** Error thrown when a transaction is not added to the mempool. */ +export class TxPoolError extends Error { + public readonly data: TxPoolRejectionError; + + constructor(public readonly reason: TxPoolRejectionError) { + super(reason.message); + this.name = 'TxPoolError'; + this.data = reason; + } +} diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts index 9227e0b32809..b1aca0208f12 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts @@ -301,6 +301,17 @@ export class DeletedPool { return this.#state.size; } + /** Gets the count of soft-deleted transactions (both prune-based and slot-based). */ + getSoftDeletedCount(): number { + let count = this.#slotDeletedTxs.size; + for (const state of this.#state.values()) { + if (state.softDeleted) { + count++; + } + } + return count; + } + /** * Gets all transaction hashes from pruned blocks. */ diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts index 70cab64727c7..9ffb86919685 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts @@ -12,6 +12,7 @@ import { type PoolOperations, type PreAddPoolAccess, type PreAddRule, + TxPoolRejectionCode, } from './interfaces.js'; describe('EvictionManager', () => { @@ -183,6 +184,7 @@ describe('EvictionManager', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -204,16 +206,36 @@ describe('EvictionManager', () => { expect(result.shouldIgnore).toBe(false); expect(result.txHashesToEvict).toContain('0x2222'); - expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess); + expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess, undefined); + }); + + it('forwards PreAddContext to rules', async () => { + preAddRule.check.mockResolvedValue({ + shouldIgnore: false, + txHashesToEvict: [], + }); + + evictionManager.registerPreAddRule(preAddRule); + const incomingMeta = createMeta('0x1111', 100n); + const context = { feeComparisonOnly: true }; + + await evictionManager.runPreAddRules(incomingMeta, poolAccess, context); + + expect(preAddRule.check).toHaveBeenCalledWith(incomingMeta, poolAccess, context); }); it('returns ignore result immediately when a rule says to ignore', async () => { const preAddRule2 = mock({ name: 'preAddRule2' }); + const testReason = { + code: 'NULLIFIER_CONFLICT' as const, + message: 'test reason', + conflictingTxHash: '0x9999', + }; preAddRule.check.mockResolvedValue({ shouldIgnore: true, txHashesToEvict: [], - reason: 'test reason', + reason: testReason, }); preAddRule2.check.mockResolvedValue({ shouldIgnore: false, @@ -227,7 +249,7 @@ describe('EvictionManager', () => { const result = await evictionManager.runPreAddRules(incomingMeta, poolAccess); expect(result.shouldIgnore).toBe(true); - expect(result.reason).toBe('test reason'); + expect(result.reason).toEqual(testReason); expect(preAddRule.check).toHaveBeenCalledTimes(1); // Second rule should not be called since first rule ignored expect(preAddRule2.check).not.toHaveBeenCalled(); @@ -318,6 +340,7 @@ describe('EvictionManager', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -334,7 +357,9 @@ describe('EvictionManager', () => { const result = await evictionManager.runPreAddRules(incomingMeta, poolAccess); expect(result.shouldIgnore).toBe(true); - expect(result.reason).toContain('failingRule'); + expect(result.reason).toBeDefined(); + expect(result.reason!.code).toBe(TxPoolRejectionCode.INTERNAL_ERROR); + expect(result.reason!.message).toContain('failingRule'); expect(result.txHashesToEvict).toHaveLength(0); // Second rule should not be called since first rule threw expect(preAddRule2.check).not.toHaveBeenCalled(); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts index 476f40eb0be7..f8245cf4614e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts @@ -9,9 +9,12 @@ import { EvictionEvent, type EvictionRule, type PoolOperations, + type PreAddContext, type PreAddPoolAccess, type PreAddResult, type PreAddRule, + type TaggedEviction, + TxPoolRejectionCode, } from './interfaces.js'; /** @@ -47,21 +50,27 @@ export class EvictionManager { * Runs all pre-add rules for an incoming transaction. * Returns combined result of all rules. */ - async runPreAddRules(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise { - const allTxHashesToEvict: string[] = []; + async runPreAddRules( + incomingMeta: TxMetaData, + poolAccess: PreAddPoolAccess, + context?: PreAddContext, + ): Promise { + const evictions: TaggedEviction[] = []; + const seen = new Set(); for (const rule of this.preAddRules) { try { - const result = await rule.check(incomingMeta, poolAccess); + const result = await rule.check(incomingMeta, poolAccess, context); if (result.shouldIgnore) { return result; } - // Collect txs to evict from all rules + // Collect txs to evict from all rules, tagged with the rule name for (const txHash of result.txHashesToEvict) { - if (!allTxHashesToEvict.includes(txHash)) { - allTxHashesToEvict.push(txHash); + if (!seen.has(txHash)) { + seen.add(txHash); + evictions.push({ txHash, reason: rule.name }); } } } catch (err) { @@ -70,14 +79,18 @@ export class EvictionManager { return { shouldIgnore: true, txHashesToEvict: [], - reason: `pre-add rule ${rule.name} error: ${err}`, + reason: { + code: TxPoolRejectionCode.INTERNAL_ERROR, + message: `Pre-add rule ${rule.name} error: ${err}`, + }, }; } } return { shouldIgnore: false, - txHashesToEvict: allTxHashesToEvict, + txHashesToEvict: evictions.map(e => e.txHash), + evictions, }; } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts index db41d206f772..ae2b886818a9 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts @@ -43,6 +43,7 @@ describe('FeePayerBalanceEvictionRule', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -144,7 +145,7 @@ describe('FeePayerBalanceEvictionRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x1111']); // Low priority evicted - expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'FeePayerBalanceEviction'); }); it('evicts multiple low-priority txs when balance is insufficient', async () => { @@ -193,7 +194,7 @@ describe('FeePayerBalanceEvictionRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0xaaaa']); // Only lowest priority evicted - expect(deleteTxsMock).toHaveBeenCalledWith(['0xaaaa']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0xaaaa'], 'FeePayerBalanceEviction'); }); it('considers claim amount when calculating available balance', async () => { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts index 32a5db700677..969fd127b1d1 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts @@ -67,8 +67,8 @@ export class FeePayerBalanceEvictionRule implements EvictionRule { ).flat(); if (txsToEvict.length > 0) { - await pool.deleteTxs(txsToEvict); - this.log.verbose(`Evicted ${txsToEvict.length} txs due to insufficient fee payer balance`, { + await pool.deleteTxs(txsToEvict, this.name); + this.log.debug(`Evicted ${txsToEvict.length} txs due to insufficient fee payer balance`, { txHashes: txsToEvict, }); } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts index 8079390b9d0f..af21423f39e2 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts @@ -1,6 +1,6 @@ import { type TxMetaData, stubTxMetaValidationData } from '../tx_metadata.js'; import { FeePayerBalancePreAddRule } from './fee_payer_balance_pre_add_rule.js'; -import type { PreAddPoolAccess } from './interfaces.js'; +import { type PreAddPoolAccess, TxPoolRejectionCode } from './interfaces.js'; describe('FeePayerBalancePreAddRule', () => { let rule: FeePayerBalancePreAddRule; @@ -24,6 +24,7 @@ describe('FeePayerBalancePreAddRule', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -62,7 +63,12 @@ describe('FeePayerBalancePreAddRule', () => { expect(result.shouldIgnore).toBe(true); expect(result.txHashesToEvict).toHaveLength(0); - expect(result.reason).toContain('insufficient balance'); + expect(result.reason).toBeDefined(); + expect(result.reason!.code).toBe(TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE); + if (result.reason!.code === TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE) { + expect(result.reason!.currentBalance).toBe(50n); + expect(result.reason!.feeLimit).toBe(100n); + } }); it('accepts tx when balance exactly equals fee limit', async () => { @@ -107,7 +113,8 @@ describe('FeePayerBalancePreAddRule', () => { const result = await rule.check(incomingMeta, poolAccess); expect(result.shouldIgnore).toBe(true); - expect(result.reason).toContain('insufficient balance'); + expect(result.reason).toBeDefined(); + expect(result.reason!.code).toBe(TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE); }); it('evicts lower-priority existing tx when high-priority tx is added', async () => { @@ -262,7 +269,8 @@ describe('FeePayerBalancePreAddRule', () => { expect(result.shouldIgnore).toBe(true); expect(result.reason).toBeDefined(); - expect(result.reason).toContain('insufficient balance'); + expect(result.reason!.code).toBe(TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE); + expect(result.reason!.message).toContain('insufficient balance'); }); }); }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts index ae2ba0006058..0cdebb94948d 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts @@ -1,7 +1,13 @@ import { createLogger } from '@aztec/foundation/log'; import { type TxMetaData, comparePriority } from '../tx_metadata.js'; -import type { PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js'; +import { + type PreAddContext, + type PreAddPoolAccess, + type PreAddResult, + type PreAddRule, + TxPoolRejectionCode, +} from './interfaces.js'; /** * Pre-add rule that checks if a fee payer has sufficient balance to cover the incoming transaction. @@ -19,7 +25,7 @@ export class FeePayerBalancePreAddRule implements PreAddRule { private log = createLogger('p2p:tx_pool_v2:fee_payer_balance_pre_add_rule'); - async check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise { + async check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, _context?: PreAddContext): Promise { // Get fee payer's on-chain balance const initialBalance = await poolAccess.getFeePayerBalance(incomingMeta.feePayer); @@ -78,7 +84,13 @@ export class FeePayerBalancePreAddRule implements PreAddRule { return { shouldIgnore: true, txHashesToEvict: [], - reason: `fee payer ${incomingMeta.feePayer} has insufficient balance`, + reason: { + code: TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE, + message: `Fee payer ${incomingMeta.feePayer} has insufficient balance. Balance at transaction: ${available}, required: ${incomingMeta.feeLimit}`, + currentBalance: initialBalance, + availableBalance: available, + feeLimit: incomingMeta.feeLimit, + }, }; } else { // Existing tx cannot be covered after adding incoming - mark for eviction @@ -93,7 +105,6 @@ export class FeePayerBalancePreAddRule implements PreAddRule { return { shouldIgnore: true, txHashesToEvict: [], - reason: 'internal error: tx coverage not determined', }; } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/index.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/index.ts index faf1cc5b9615..79abcdc12812 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/index.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/index.ts @@ -6,9 +6,13 @@ export { type EvictionResult, type EvictionRule, type PoolOperations, + type PreAddContext, type PreAddPoolAccess, type PreAddResult, type PreAddRule, + type TaggedEviction, + TxPoolRejectionCode, + type TxPoolRejectionError, } from './interfaces.js'; // Pre-add rules diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/interfaces.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/interfaces.ts index 1d0ba416013c..32135758973d 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/interfaces.ts @@ -67,6 +67,42 @@ export interface PreAddPoolAccess { getLowestPriorityPendingTx(): TxMetaData | undefined; } +/** A single eviction tagged with the rule that caused it. */ +export interface TaggedEviction { + readonly txHash: string; + readonly reason: string; +} + +/** + * Machine-readable rejection codes for pre-add rule rejections. + */ +export const TxPoolRejectionCode = { + LOW_PRIORITY_FEE: 'LOW_PRIORITY_FEE', + INSUFFICIENT_FEE_PAYER_BALANCE: 'INSUFFICIENT_FEE_PAYER_BALANCE', + NULLIFIER_CONFLICT: 'NULLIFIER_CONFLICT', + INTERNAL_ERROR: 'INTERNAL_ERROR', +} as const; + +export type TxPoolRejectionCode = (typeof TxPoolRejectionCode)[keyof typeof TxPoolRejectionCode]; + +/** Structured rejection reason returned by pre-add rules. */ +export type TxPoolRejectionError = + | { + code: typeof TxPoolRejectionCode.LOW_PRIORITY_FEE; + message: string; + minimumPriorityFee: bigint; + txPriorityFee: bigint; + } + | { + code: typeof TxPoolRejectionCode.INSUFFICIENT_FEE_PAYER_BALANCE; + message: string; + currentBalance: bigint; + availableBalance: bigint; + feeLimit: bigint; + } + | { code: typeof TxPoolRejectionCode.NULLIFIER_CONFLICT; message: string; conflictingTxHash: string } + | { code: typeof TxPoolRejectionCode.INTERNAL_ERROR; message: string }; + /** * Result of a pre-add check for a single transaction. */ @@ -75,8 +111,16 @@ export interface PreAddResult { readonly shouldIgnore: boolean; /** Tx hashes (as strings) that should be evicted if this tx is added */ readonly txHashesToEvict: string[]; + /** Evictions tagged with the rule name that produced them. Populated by EvictionManager. */ + readonly evictions?: TaggedEviction[]; /** Optional reason for ignoring */ - readonly reason?: string; + readonly reason?: TxPoolRejectionError; +} + +/** Context passed to pre-add rules from addPendingTxs. */ +export interface PreAddContext { + /** If true, compare priority fee only (no tx hash tiebreaker). Used for RPC submissions. */ + feeComparisonOnly?: boolean; } /** @@ -90,9 +134,10 @@ export interface PreAddRule { * Check if incoming tx should be added and which existing txs to evict. * @param incomingMeta - Metadata for the incoming transaction * @param poolAccess - Read-only access to current pool state + * @param context - Optional context from addPendingTxs caller * @returns Result indicating whether to ignore and what to evict */ - check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise; + check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, context?: PreAddContext): Promise; /** * Updates the configuration for this rule. @@ -120,8 +165,8 @@ export interface PoolOperations { /** Get the N lowest priority pending tx hashes */ getLowestPriorityPending(limit: number): string[]; - /** Delete transactions by hash */ - deleteTxs(txHashes: string[]): Promise; + /** Delete transactions by hash, with an optional reason for metrics */ + deleteTxs(txHashes: string[], reason?: string): Promise; } /** diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts index 833028245ae3..8c16d90d9c00 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts @@ -37,6 +37,7 @@ describe('InvalidTxsAfterMiningRule', () => { nullifiers, expirationTimestamp, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData({ expirationTimestamp }), }; }; @@ -122,7 +123,7 @@ describe('InvalidTxsAfterMiningRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x1111']); // Only tx1 has duplicate nullifier - expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'InvalidTxsAfterMining'); }); it('evicts transactions with expired timestamps', async () => { @@ -142,7 +143,7 @@ describe('InvalidTxsAfterMiningRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x1111']); // Only tx1 is expired - expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'InvalidTxsAfterMining'); }); it('evicts transactions with timestamp equal to block timestamp', async () => { @@ -162,7 +163,7 @@ describe('InvalidTxsAfterMiningRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x1111']); // tx1 has timestamp <= block timestamp - expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'InvalidTxsAfterMining'); }); it('handles transactions with both duplicate nullifiers and expired timestamps', async () => { @@ -182,7 +183,7 @@ describe('InvalidTxsAfterMiningRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x1111']); - expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x1111'], 'InvalidTxsAfterMining'); }); it('handles empty pending transactions list', async () => { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts index dfecfc86524c..51559176827b 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts @@ -35,7 +35,6 @@ export class InvalidTxsAfterMiningRule implements EvictionRule { for (const meta of pendingTxs) { // Evict pending txs that share nullifiers with mined txs if (meta.nullifiers.some(nullifier => minedNullifiers.has(nullifier))) { - this.log.verbose(`Evicting tx ${meta.txHash} from pool due to a duplicate nullifier with a mined tx`); txsToEvict.push(meta.txHash); continue; } @@ -51,7 +50,8 @@ export class InvalidTxsAfterMiningRule implements EvictionRule { } if (txsToEvict.length > 0) { - await pool.deleteTxs(txsToEvict); + this.log.info(`Evicted ${txsToEvict.length} invalid txs after block mined`); + await pool.deleteTxs(txsToEvict, this.name); } this.log.debug(`Evicted ${txsToEvict.length} invalid txs after block mined`, { txHashes: txsToEvict }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts index 4d36d5a8be44..0d02b3431f88 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts @@ -31,6 +31,7 @@ describe('InvalidTxsAfterReorgRule', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -160,7 +161,7 @@ describe('InvalidTxsAfterReorgRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted.length).toBe(pendingTxs.length); - expect(deleteTxsMock).toHaveBeenCalledWith(result.txsEvicted); + expect(deleteTxsMock).toHaveBeenCalledWith(result.txsEvicted, 'InvalidTxsAfterReorg'); }); it('handles error from deleteTxs operation', async () => { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts index f76c23eb7b1e..72462a8a687f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts @@ -72,8 +72,8 @@ export class InvalidTxsAfterReorgRule implements EvictionRule { } if (txsToEvict.length > 0) { - this.log.verbose(`Evicting ${txsToEvict.length} txs from pool due to referencing pruned blocks`); - await pool.deleteTxs(txsToEvict); + this.log.info(`Evicting ${txsToEvict.length} txs from pool due to referencing pruned blocks`); + await pool.deleteTxs(txsToEvict, this.name); } const keptCount = pendingTxs.length - txsToEvict.length; @@ -81,7 +81,7 @@ export class InvalidTxsAfterReorgRule implements EvictionRule { this.log.verbose(`Kept ${keptCount} txs that did not reference pruned blocks`); } - this.log.info(`Evicted ${txsToEvict.length} invalid txs after reorg`, { txHashes: txsToEvict }); + this.log.debug(`Evicted ${txsToEvict.length} invalid txs after reorg`, { txHashes: txsToEvict }); return { reason: 'reorg_invalid_txs', diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.test.ts index 75aabcd10439..93744abc603b 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.test.ts @@ -132,7 +132,7 @@ describe('LowPriorityEvictionRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x3333', '0x4444']); - expect(deleteTxsMock).toHaveBeenCalledWith(['0x3333', '0x4444']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x3333', '0x4444'], 'LowPriorityEviction'); }); it('tracks newly added transactions that were evicted', async () => { @@ -148,7 +148,7 @@ describe('LowPriorityEvictionRule', () => { expect(result.success).toBe(true); expect(result.txsEvicted).toEqual(['0x3333', '0x1111']); - expect(deleteTxsMock).toHaveBeenCalledWith(['0x3333', '0x1111']); + expect(deleteTxsMock).toHaveBeenCalledWith(['0x3333', '0x1111'], 'LowPriorityEviction'); }); it('handles all transactions being non-evictable', async () => { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts index c9854aeb6a17..047695aaf681 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts @@ -48,19 +48,18 @@ export class LowPriorityEvictionRule implements EvictionRule { }; } - this.log.verbose( - `Evicting low priority txs. Pending tx count above limit: ${currentTxCount} > ${this.maxPoolSize}`, - ); + this.log.info(`Evicting low priority txs. Pending tx count above limit: ${currentTxCount} > ${this.maxPoolSize}`); const numberToEvict = currentTxCount - this.maxPoolSize; const txsToEvict = pool.getLowestPriorityPending(numberToEvict); + const toEvictSet = new Set(txsToEvict); + const numNewTxsEvicted = context.newTxHashes.filter(newTxHash => toEvictSet.has(newTxHash)).length; if (txsToEvict.length > 0) { - await pool.deleteTxs(txsToEvict); + this.log.info(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`); + await pool.deleteTxs(txsToEvict, this.name); } - const numNewTxsEvicted = context.newTxHashes.filter(newTxHash => txsToEvict.includes(newTxHash)).length; - - this.log.verbose(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`, { + this.log.debug(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`, { txHashes: txsToEvict, }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts index 85c9eea11eff..b7173ba8bdbe 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts @@ -1,5 +1,5 @@ -import { type TxMetaData, stubTxMetaValidationData } from '../tx_metadata.js'; -import type { PreAddPoolAccess } from './interfaces.js'; +import { type TxMetaData, comparePriority, stubTxMetaValidationData } from '../tx_metadata.js'; +import { type PreAddContext, type PreAddPoolAccess, TxPoolRejectionCode } from './interfaces.js'; import { LowPriorityPreAddRule } from './low_priority_pre_add_rule.js'; describe('LowPriorityPreAddRule', () => { @@ -16,6 +16,7 @@ describe('LowPriorityPreAddRule', () => { nullifiers: [`0x${txHash.slice(2)}null1`], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -100,7 +101,12 @@ describe('LowPriorityPreAddRule', () => { expect(result.shouldIgnore).toBe(true); expect(result.txHashesToEvict).toHaveLength(0); - expect(result.reason).toContain('lower priority'); + expect(result.reason).toBeDefined(); + expect(result.reason!.code).toBe(TxPoolRejectionCode.LOW_PRIORITY_FEE); + if (result.reason!.code === TxPoolRejectionCode.LOW_PRIORITY_FEE) { + expect(result.reason!.minimumPriorityFee).toBe(101n); + expect(result.reason!.txPriorityFee).toBe(50n); + } }); it('ignores tx when incoming has equal priority to lowest', async () => { @@ -147,5 +153,73 @@ describe('LowPriorityPreAddRule', () => { expect(result.txHashesToEvict).toHaveLength(0); }); }); + + describe('feeOnly context', () => { + it('uses comparePriority (default): same fee, higher-priority hash evicts existing', async () => { + // Pick two hashes with the same fee, where incoming has higher priority by hash tiebreaker + const existing = createMeta('0x1111', 100n); + const incoming = createMeta('0x2222', 100n); + + // Determine which direction the tiebreaker goes and swap if needed + const cmp = comparePriority(incoming, existing); + const [incomingMeta, lowestPriorityMeta] = cmp > 0 ? [incoming, existing] : [existing, incoming]; + + const poolAccess = createPoolAccess(100, lowestPriorityMeta); + + // Default context (no feeOnly) — uses full comparePriority + const result = await rule.check(incomingMeta, poolAccess); + + expect(result.shouldIgnore).toBe(false); + expect(result.txHashesToEvict).toContain(lowestPriorityMeta.txHash); + }); + + it('uses feeComparisonOnly: same fee, incoming is ignored even if it wins hash tiebreaker', async () => { + const existing = createMeta('0x1111', 100n); + const incoming = createMeta('0x2222', 100n); + + // Determine which has higher hash priority and use that as incoming + const cmp = comparePriority(incoming, existing); + const [incomingMeta, lowestPriorityMeta] = cmp > 0 ? [incoming, existing] : [existing, incoming]; + + const poolAccess = createPoolAccess(100, lowestPriorityMeta); + const context: PreAddContext = { feeComparisonOnly: true }; + + // feeOnly mode: same fee means ignored (no hash tiebreaker) + const result = await rule.check(incomingMeta, poolAccess, context); + + expect(result.shouldIgnore).toBe(true); + expect(result.txHashesToEvict).toHaveLength(0); + }); + + it('higher fee evicts regardless of feeOnly flag', async () => { + const lowestPriorityMeta = createMeta('0x2222', 50n); + const poolAccess = createPoolAccess(100, lowestPriorityMeta); + const incomingMeta = createMeta('0x1111', 100n); + + // Without feeOnly + const result1 = await rule.check(incomingMeta, poolAccess); + expect(result1.shouldIgnore).toBe(false); + expect(result1.txHashesToEvict).toContain('0x2222'); + + // With feeOnly + const result2 = await rule.check(incomingMeta, poolAccess, { feeComparisonOnly: true }); + expect(result2.shouldIgnore).toBe(false); + expect(result2.txHashesToEvict).toContain('0x2222'); + }); + + it('lower fee is always ignored regardless of feeOnly flag', async () => { + const lowestPriorityMeta = createMeta('0x2222', 100n); + const poolAccess = createPoolAccess(100, lowestPriorityMeta); + const incomingMeta = createMeta('0x1111', 50n); + + // Without feeOnly + const result1 = await rule.check(incomingMeta, poolAccess); + expect(result1.shouldIgnore).toBe(true); + + // With feeOnly + const result2 = await rule.check(incomingMeta, poolAccess, { feeComparisonOnly: true }); + expect(result2.shouldIgnore).toBe(true); + }); + }); }); }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts index a086cd64fc85..b4d5ef8382db 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts @@ -1,7 +1,14 @@ import { createLogger } from '@aztec/foundation/log'; -import type { TxMetaData } from '../tx_metadata.js'; -import type { EvictionConfig, PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js'; +import { type TxMetaData, comparePriority } from '../tx_metadata.js'; +import { + type EvictionConfig, + type PreAddContext, + type PreAddPoolAccess, + type PreAddResult, + type PreAddRule, + TxPoolRejectionCode, +} from './interfaces.js'; /** * Pre-add rule that checks if the pool is at capacity and handles low-priority eviction. @@ -20,7 +27,7 @@ export class LowPriorityPreAddRule implements PreAddRule { this.maxPoolSize = config.maxPoolSize; } - check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise { + check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, context?: PreAddContext): Promise { // Skip if max pool size is disabled (0 = unlimited) if (this.maxPoolSize === 0) { return Promise.resolve({ shouldIgnore: false, txHashesToEvict: [] }); @@ -40,8 +47,14 @@ export class LowPriorityPreAddRule implements PreAddRule { return Promise.resolve({ shouldIgnore: false, txHashesToEvict: [] }); } - // If incoming tx has strictly higher priority, evict the lowest priority tx - if (incomingMeta.priorityFee > lowestPriorityMeta.priorityFee) { + // Compare incoming tx against lowest priority tx. + // feeOnly mode (RPC): use strict fee comparison only — avoids churn from hash ordering + // Default (gossip): use full comparePriority (fee + tx hash tiebreaker) for determinism + const isHigherPriority = context?.feeComparisonOnly + ? incomingMeta.priorityFee > lowestPriorityMeta.priorityFee + : comparePriority(incomingMeta, lowestPriorityMeta) > 0; + + if (isHigherPriority) { this.log.debug( `Pool at capacity (${currentCount}/${this.maxPoolSize}), evicting ${lowestPriorityMeta.txHash} ` + `(priority ${lowestPriorityMeta.priorityFee}) for ${incomingMeta.txHash} (priority ${incomingMeta.priorityFee})`, @@ -60,7 +73,12 @@ export class LowPriorityPreAddRule implements PreAddRule { return Promise.resolve({ shouldIgnore: true, txHashesToEvict: [], - reason: `pool at capacity and tx has lower priority than existing transactions`, + reason: { + code: TxPoolRejectionCode.LOW_PRIORITY_FEE, + message: `Tx does not meet minimum priority fee. Required: ${lowestPriorityMeta.priorityFee + 1n}, got: ${incomingMeta.priorityFee}`, + minimumPriorityFee: lowestPriorityMeta.priorityFee + 1n, + txPriorityFee: incomingMeta.priorityFee, + }, }); } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts index 353231e0a807..f5f08c1ece36 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts @@ -21,6 +21,7 @@ describe('NullifierConflictRule', () => { nullifiers, expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts index 6eecac930709..9b638e13e83d 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts @@ -1,7 +1,7 @@ import { createLogger } from '@aztec/foundation/log'; import { type TxMetaData, checkNullifierConflict } from '../tx_metadata.js'; -import type { PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js'; +import type { PreAddContext, PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js'; /** * Pre-add rule that checks for nullifier conflicts between incoming and existing transactions. @@ -15,7 +15,7 @@ export class NullifierConflictRule implements PreAddRule { private log = createLogger('p2p:tx_pool_v2:nullifier_conflict_rule'); - check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise { + check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, _context?: PreAddContext): Promise { const result = checkNullifierConflict( incomingMeta, nullifier => poolAccess.getTxHashByNullifier(nullifier), @@ -23,7 +23,7 @@ export class NullifierConflictRule implements PreAddRule { ); if (result.shouldIgnore) { - this.log.debug(`Ignoring tx ${incomingMeta.txHash}: ${result.reason}`); + this.log.debug(`Ignoring tx ${incomingMeta.txHash}: ${result.reason?.message}`); } return Promise.resolve(result); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/instrumentation.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/instrumentation.ts new file mode 100644 index 000000000000..6ec711bb826c --- /dev/null +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/instrumentation.ts @@ -0,0 +1,69 @@ +import { + Attributes, + type Meter, + Metrics, + type ObservableGauge, + type ObservableResult, + type TelemetryClient, + type UpDownCounter, + createUpDownCounterWithDefault, +} from '@aztec/telemetry-client'; + +/** Callback that returns the current estimated metadata memory in bytes. */ +export type MetadataMemoryCallback = () => number; + +/** Instrumentation for TxPoolV2Impl internal operations. */ +export class TxPoolV2Instrumentation { + #evictedCounter: UpDownCounter; + #ignoredCounter: UpDownCounter; + #rejectedCounter: UpDownCounter; + #softDeletedHitsCounter: UpDownCounter; + #missingOnProtectCounter: UpDownCounter; + #missingPreviouslyEvictedCounter: UpDownCounter; + #metadataMemoryGauge: ObservableGauge; + + constructor(telemetry: TelemetryClient, metadataMemoryCallback: MetadataMemoryCallback) { + const meter: Meter = telemetry.getMeter('TxPoolV2Impl'); + + this.#evictedCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_EVICTED_COUNT); + this.#ignoredCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_IGNORED_COUNT); + this.#rejectedCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_REJECTED_COUNT); + this.#softDeletedHitsCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_SOFT_DELETED_HITS); + this.#missingOnProtectCounter = createUpDownCounterWithDefault( + meter, + Metrics.MEMPOOL_TX_POOL_V2_MISSING_ON_PROTECT, + ); + this.#missingPreviouslyEvictedCounter = createUpDownCounterWithDefault( + meter, + Metrics.MEMPOOL_TX_POOL_V2_MISSING_PREVIOUSLY_EVICTED, + ); + this.#metadataMemoryGauge = meter.createObservableGauge(Metrics.MEMPOOL_TX_POOL_V2_METADATA_MEMORY); + this.#metadataMemoryGauge.addCallback((result: ObservableResult) => { + result.observe(metadataMemoryCallback()); + }); + } + + recordEvictions(count: number, reason: string) { + this.#evictedCounter.add(count, { [Attributes.TX_POOL_EVICTION_REASON]: reason }); + } + + recordIgnored(count: number) { + this.#ignoredCounter.add(count); + } + + recordRejected(count: number) { + this.#rejectedCounter.add(count); + } + + recordSoftDeletedHits(count: number) { + this.#softDeletedHitsCounter.add(count); + } + + recordMissingOnProtect(count: number) { + this.#missingOnProtectCounter.add(count); + } + + recordMissingPreviouslyEvicted(count: number) { + this.#missingPreviouslyEvictedCounter.add(count); + } +} diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts index 1057f57e3954..ca23965e117e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts @@ -4,6 +4,7 @@ import type { L2Block, L2BlockId, L2BlockSource } from '@aztec/stdlib/block'; import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; import type { BlockHeader, Tx, TxHash, TxValidator } from '@aztec/stdlib/tx'; +import type { TxPoolRejectionError } from './eviction/interfaces.js'; import type { TxMetaData, TxState } from './tx_metadata.js'; /** @@ -17,6 +18,8 @@ export type AddTxsResult = { ignored: TxHash[]; /** Transactions rejected because they failed validation (e.g., invalid proof, expired timestamp) */ rejected: TxHash[]; + /** Optional rejection errors, only present when there are rejections with structured errors. */ + errors?: Map; }; /** @@ -39,6 +42,8 @@ export type TxPoolV2Config = { archivedTxLimit: number; /** Minimum age (ms) a transaction must have been in the pool before it's eligible for block building */ minTxPoolAgeMs: number; + /** Maximum number of evicted tx hashes to remember for metrics tracking */ + evictedTxCacheSize: number; }; /** @@ -48,6 +53,7 @@ export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = { maxPendingTxCount: 0, // 0 = disabled archivedTxLimit: 0, // 0 = disabled minTxPoolAgeMs: 2_000, + evictedTxCacheSize: 10_000, }; /** @@ -98,7 +104,7 @@ export interface TxPoolV2 extends TypedEventEmitter { * @param opts - Optional metadata (e.g., source for logging) * @returns Result categorizing each transaction as accepted, rejected, or ignored */ - addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise; + addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise; /** * Checks if a transaction can be added without modifying the pool. diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts index bc384e8ba029..63417fa8559a 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts @@ -1,5 +1,6 @@ import { mockTx } from '@aztec/stdlib/testing'; +import { TxPoolRejectionCode } from './eviction/interfaces.js'; import { type TxMetaData, buildTxMetaData, @@ -48,6 +49,7 @@ describe('TxMetaData', () => { nullifiers: [], expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -81,6 +83,7 @@ describe('TxMetaData', () => { nullifiers, expirationTimestamp: 0n, receivedAt: 0, + estimatedSizeBytes: 0, data: stubTxMetaValidationData(), }); @@ -129,7 +132,11 @@ describe('TxMetaData', () => { expect(result.shouldIgnore).toBe(true); expect(result.txHashesToEvict).toEqual([]); - expect(result.reason).toContain(existing.txHash); + expect(result.reason).toBeDefined(); + expect(result.reason!.code).toBe(TxPoolRejectionCode.NULLIFIER_CONFLICT); + if (result.reason!.code === TxPoolRejectionCode.NULLIFIER_CONFLICT) { + expect(result.reason!.conflictingTxHash).toBe(existing.txHash); + } }); it('ignores incoming tx when existing has equal priority (tie goes to existing)', () => { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts index 3bdf207167cf..1649075062a1 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts @@ -6,7 +6,7 @@ import type { Tx } from '@aztec/stdlib/tx'; import { getFeePayerBalanceDelta } from '../../msg_validators/tx_validator/fee_payer_balance.js'; import { getTxPriorityFee } from '../tx_pool/priority.js'; -import type { PreAddResult } from './eviction/interfaces.js'; +import { type PreAddResult, TxPoolRejectionCode } from './eviction/interfaces.js'; /** Validator-compatible data interface, mirroring the subset of PrivateKernelTailCircuitPublicInputs used by validators. */ export type TxMetaValidationData = { @@ -63,6 +63,9 @@ export type TxMetaData = { /** Timestamp (ms) when the tx was received into the pool. 0 for hydrated txs (always eligible). */ receivedAt: number; + + /** Estimated memory footprint of this metadata object in bytes */ + readonly estimatedSizeBytes: number; }; /** Transaction state derived from TxMetaData fields and pool protection status */ @@ -86,6 +89,8 @@ export async function buildTxMetaData(tx: Tx): Promise { const { feeLimit, claimAmount } = await getFeePayerBalanceDelta(tx, ProtocolContractAddress.FeeJuice); + const estimatedSizeBytes = estimateTxMetaDataSize(nullifiers.length); + return { txHash, anchorBlockHeaderHash, @@ -96,6 +101,7 @@ export async function buildTxMetaData(tx: Tx): Promise { nullifiers, expirationTimestamp, receivedAt: 0, + estimatedSizeBytes, data: { getNonEmptyNullifiers: () => nullifierFrs, expirationTimestamp, @@ -109,6 +115,27 @@ export async function buildTxMetaData(tx: Tx): Promise { }; } +// V8 JS object overhead (~64 bytes for a plain object with hidden class). +// String overhead: ~32 bytes header + 1 byte per ASCII char (V8 one-byte strings). +// Hex string (0x + 64 hex chars = 66 chars): ~98 bytes per string. +// bigint: ~32 bytes. number: 8 bytes. Fr: ~80 bytes (32 data + object overhead). +const OBJECT_OVERHEAD = 64; +const HEX_STRING_BYTES = 98; +const BIGINT_BYTES = 32; +const FR_BYTES = 80; +// Fixed cost: object shell + txHash + anchorBlockHeaderHash + feePayer (3 hex strings) +// + priorityFee + claimAmount + feeLimit + includeByTimestamp (4 bigints) +// + receivedAt (number, 8 bytes) + estimatedSizeBytes (number, 8 bytes) +// + data closure object (~OBJECT_OVERHEAD + anchorBlockHeaderHashFr Fr + anchorBlockNumber number) +const FIXED_METADATA_BYTES = + OBJECT_OVERHEAD + 3 * HEX_STRING_BYTES + 4 * BIGINT_BYTES + 8 + 8 + OBJECT_OVERHEAD + FR_BYTES + 8; + +/** Estimates the in-memory size of a TxMetaData object based on the number of nullifiers. */ +function estimateTxMetaDataSize(nullifierCount: number): number { + // Per nullifier: one hex string in nullifiers[] + one Fr in the captured nullifierFrs[] + return FIXED_METADATA_BYTES + nullifierCount * (HEX_STRING_BYTES + FR_BYTES); +} + /** Minimal fields required for priority comparison. */ type PriorityComparable = Pick; @@ -188,7 +215,11 @@ export function checkNullifierConflict( return { shouldIgnore: true, txHashesToEvict: [], - reason: `nullifier conflict with ${conflictingHashStr}`, + reason: { + code: TxPoolRejectionCode.NULLIFIER_CONFLICT, + message: `Nullifier conflict with existing tx ${conflictingHashStr}`, + conflictingTxHash: conflictingHashStr, + }, }; } } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts index cf8291a17bad..a9a368dce37c 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts @@ -348,13 +348,15 @@ export class TxPoolIndices { // METRICS // ============================================================================ - /** Counts transactions by state */ - countTxs(): { pending: number; protected: number; mined: number } { + /** Counts transactions by state and estimates total metadata memory usage */ + countTxs(): { pending: number; protected: number; mined: number; totalMetadataBytes: number } { let pending = 0; let protected_ = 0; let mined = 0; + let totalMetadataBytes = 0; for (const meta of this.#metadata.values()) { + totalMetadataBytes += meta.estimatedSizeBytes; const state = this.getTxState(meta); if (state === 'pending') { pending++; @@ -365,7 +367,16 @@ export class TxPoolIndices { } } - return { pending, protected: protected_, mined }; + return { pending, protected: protected_, mined, totalMetadataBytes }; + } + + /** Returns the estimated total memory consumed by all metadata objects */ + getTotalMetadataBytes(): number { + let total = 0; + for (const meta of this.#metadata.values()) { + total += meta.estimatedSizeBytes; + } + return total; } /** Gets all mined transactions with their block IDs */ diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts index 278cc846f162..200d8e16709c 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts @@ -1158,6 +1158,144 @@ describe('TxPoolV2', () => { }); }); }); + + describe('soft-deleted tx resurrection', () => { + let mockValidator: MockProxy>; + let poolWithValidator: AztecKVTxPoolV2; + let validatorStore: Awaited>; + let validatorArchiveStore: Awaited>; + + beforeEach(async () => { + mockValidator = mock>(); + mockValidator.validateTx.mockResolvedValue({ result: 'valid' }); + + validatorStore = await openTmpStore('p2p-protect-soft-delete'); + validatorArchiveStore = await openTmpStore('archive-protect-soft-delete'); + poolWithValidator = new AztecKVTxPoolV2(validatorStore, validatorArchiveStore, { + l2BlockSource: mockL2BlockSource, + worldStateSynchronizer: mockWorldState, + createTxValidator: () => Promise.resolve(mockValidator), + }); + await poolWithValidator.start(); + }); + + afterEach(async () => { + await poolWithValidator.stop(); + await validatorStore.delete(); + await validatorArchiveStore.delete(); + }); + + /** Helper: add tx, mine it, prune it, fail validation -> soft-deleted */ + const softDeleteTx = async (tx: Tx) => { + await poolWithValidator.addPendingTxs([tx]); + await poolWithValidator.handleMinedBlock(makeBlock([tx], slot1Header)); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('mined'); + + // Make validator reject so tx is soft-deleted on prune + mockValidator.validateTx.mockResolvedValue({ + result: 'invalid', + reason: ['timestamp expired'], + }); + await poolWithValidator.handlePrunedBlocks(block0Id); + + // Verify soft-deleted + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeDefined(); + + // Restore validator for subsequent operations + mockValidator.validateTx.mockResolvedValue({ result: 'valid' }); + }; + + it('resurrects a soft-deleted tx as protected instead of reporting it missing', async () => { + const tx = await mockTx(1); + await softDeleteTx(tx); + + // protectTxs should find the soft-deleted tx and resurrect it + const missing = await poolWithValidator.protectTxs([tx.getTxHash()], slot2Header); + + expect(missing).toHaveLength(0); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('protected'); + }); + + it('resurrected soft-deleted tx is retrievable and in indices', async () => { + const tx = await mockTx(1); + await softDeleteTx(tx); + + await poolWithValidator.protectTxs([tx.getTxHash()], slot2Header); + + // Should be retrievable + const retrieved = await poolWithValidator.getTxByHash(tx.getTxHash()); + expect(retrieved).toBeDefined(); + expect(retrieved!.getTxHash().toString()).toEqual(tx.getTxHash().toString()); + + // hasTxs should return true (in indices, not just soft-deleted) + const [hasTx] = await poolWithValidator.hasTxs([tx.getTxHash()]); + expect(hasTx).toBe(true); + }); + + it('resurrected tx is unprotected on the next slot', async () => { + const tx = await mockTx(1); + await softDeleteTx(tx); + + await poolWithValidator.protectTxs([tx.getTxHash()], slot1Header); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('protected'); + + // Advance to slot 2 — protection from slot 1 expires + await poolWithValidator.prepareForSlot(SlotNumber(2)); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('pending'); + }); + + it('mix of existing, soft-deleted, and truly missing txs', async () => { + const txExisting = await mockTx(1); + const txSoftDeleted = await mockTx(2); + const txMissing = await mockTx(3); + + // Add txExisting as a regular pending tx + await poolWithValidator.addPendingTxs([txExisting]); + expect(await poolWithValidator.getTxStatus(txExisting.getTxHash())).toBe('pending'); + + // Soft-delete txSoftDeleted + await softDeleteTx(txSoftDeleted); + + // Protect all three + const missing = await poolWithValidator.protectTxs( + [txExisting.getTxHash(), txSoftDeleted.getTxHash(), txMissing.getTxHash()], + slot2Header, + ); + + // Only txMissing should be reported as missing + expect(toStrings(missing)).toEqual([hashOf(txMissing)]); + + // txExisting: protected (was pending, now protected) + expect(await poolWithValidator.getTxStatus(txExisting.getTxHash())).toBe('protected'); + // txSoftDeleted: protected (resurrected from soft-deleted) + expect(await poolWithValidator.getTxStatus(txSoftDeleted.getTxHash())).toBe('protected'); + // txMissing: pre-recorded protection, not in pool yet + expect(await poolWithValidator.getTxStatus(txMissing.getTxHash())).toBeUndefined(); + }); + + it('resurrected tx survives a second protectTxs call', async () => { + const tx = await mockTx(1); + await softDeleteTx(tx); + + // Resurrect via protectTxs at slot 1 + await poolWithValidator.protectTxs([tx.getTxHash()], slot1Header); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('protected'); + + // Re-protect at slot 2 — should update slot, not report missing + const missing = await poolWithValidator.protectTxs([tx.getTxHash()], slot2Header); + expect(missing).toHaveLength(0); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('protected'); + + // Should survive prepareForSlot(2) + await poolWithValidator.prepareForSlot(SlotNumber(2)); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('protected'); + + // Should unprotect at slot 3 + await poolWithValidator.prepareForSlot(SlotNumber(3)); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('pending'); + }); + }); }); describe('handleMinedBlock', () => { @@ -3326,6 +3464,111 @@ describe('TxPoolV2', () => { }); }); + describe('feeOnly priority comparison', () => { + it('default (gossip): same-fee tx can evict via hash tiebreaker at capacity', async () => { + await pool.updateConfig({ maxPendingTxCount: 2 }); + + const tx1 = await mockTxWithFee(1, 10); + const tx2 = await mockTxWithFee(2, 20); + await pool.addPendingTxs([tx1, tx2]); + expect(await pool.getPendingTxCount()).toBe(2); + clearCallbackTracking(); + + // Create a tx with the same fee as the lowest (tx1, fee=10). + // Without feeOnly, comparePriority uses hash tiebreaker and may evict. + const tx3 = await mockTxWithFee(3, 10); + + // Determine tiebreaker direction + const tx3HashFr = Fr.fromHexString(tx3.getTxHash().toString()); + const tx1HashFr = Fr.fromHexString(tx1.getTxHash().toString()); + const tx3WinsTiebreaker = tx3HashFr.cmp(tx1HashFr) > 0; + + // Default: no feeOnly flag (gossip path) + const result = await pool.addPendingTxs([tx3]); + + if (tx3WinsTiebreaker) { + expect(toStrings(result.accepted)).toContain(hashOf(tx3)); + expect(await pool.getPendingTxCount()).toBe(2); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); + expect(await pool.getTxStatus(tx3.getTxHash())).toBe('pending'); + } else { + expect(toStrings(result.ignored)).toContain(hashOf(tx3)); + expect(await pool.getPendingTxCount()).toBe(2); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('pending'); + } + }); + + it('feeOnly (RPC): same-fee tx is ignored at capacity regardless of hash', async () => { + await pool.updateConfig({ maxPendingTxCount: 2 }); + + const tx1 = await mockTxWithFee(1, 10); + const tx2 = await mockTxWithFee(2, 20); + await pool.addPendingTxs([tx1, tx2]); + expect(await pool.getPendingTxCount()).toBe(2); + clearCallbackTracking(); + + // Same fee as the lowest — with feeOnly, no hash tiebreaker, always ignored + const tx3 = await mockTxWithFee(3, 10); + const result = await pool.addPendingTxs([tx3], { feeComparisonOnly: true }); + + expect(toStrings(result.ignored)).toContain(hashOf(tx3)); + expect(result.accepted).toHaveLength(0); + expect(await pool.getPendingTxCount()).toBe(2); + expectNoCallbacks(); + }); + + it('feeOnly (RPC): higher-fee tx still evicts at capacity', async () => { + await pool.updateConfig({ maxPendingTxCount: 2 }); + + const tx1 = await mockTxWithFee(1, 10); + const tx2 = await mockTxWithFee(2, 20); + await pool.addPendingTxs([tx1, tx2]); + expect(await pool.getPendingTxCount()).toBe(2); + clearCallbackTracking(); + + const tx3 = await mockTxWithFee(3, 15); + const result = await pool.addPendingTxs([tx3], { feeComparisonOnly: true }); + + expect(toStrings(result.accepted)).toContain(hashOf(tx3)); + expect(await pool.getPendingTxCount()).toBe(2); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); // fee=10 evicted + expect(await pool.getTxStatus(tx3.getTxHash())).toBe('pending'); + }); + + it('feeOnly (RPC): lower-fee tx is ignored at capacity', async () => { + await pool.updateConfig({ maxPendingTxCount: 2 }); + + const tx1 = await mockTxWithFee(1, 10); + const tx2 = await mockTxWithFee(2, 20); + await pool.addPendingTxs([tx1, tx2]); + expect(await pool.getPendingTxCount()).toBe(2); + clearCallbackTracking(); + + const tx3 = await mockTxWithFee(3, 5); + const result = await pool.addPendingTxs([tx3], { feeComparisonOnly: true }); + + expect(toStrings(result.ignored)).toContain(hashOf(tx3)); + expect(await pool.getPendingTxCount()).toBe(2); + expectNoCallbacks(); + }); + + it('feeOnly has no effect when pool is not at capacity', async () => { + await pool.updateConfig({ maxPendingTxCount: 10 }); + + const tx1 = await mockTxWithFee(1, 10); + + // Both modes accept when below capacity + const result1 = await pool.addPendingTxs([tx1], { feeComparisonOnly: true }); + expect(result1.accepted).toHaveLength(1); + + const tx2 = await mockTxWithFee(2, 10); + const result2 = await pool.addPendingTxs([tx2]); + expect(result2.accepted).toHaveLength(1); + + expect(await pool.getPendingTxCount()).toBe(2); + }); + }); + describe('multiple nullifier conflicts', () => { it('handles tx with multiple nullifiers conflicting with different txs', async () => { const tx1 = await mockPublicTx(1, 5); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts index c37702c77a8f..42d8a39406be 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts @@ -61,7 +61,7 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte }; // Create the implementation - this.#impl = new TxPoolV2Impl(store, archiveStore, deps, callbacks, config, dateProvider, log); + this.#impl = new TxPoolV2Impl(store, archiveStore, deps, callbacks, telemetry, config, dateProvider, log); } // ============================================================================ @@ -70,7 +70,7 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte // === Core Operations === - addPendingTxs(txs: Tx[], opts: { source?: string } = {}): Promise { + addPendingTxs(txs: Tx[], opts: { source?: string; feeComparisonOnly?: boolean } = {}): Promise { return this.#queue.put(() => this.#impl.addPendingTxs(txs, opts)); } @@ -83,7 +83,7 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte } protectTxs(txHashes: TxHash[], block: BlockHeader): Promise { - return this.#queue.put(() => Promise.resolve(this.#impl.protectTxs(txHashes, block))); + return this.#queue.put(() => this.#impl.protectTxs(txHashes, block)); } addMinedTxs(txs: Tx[], block: BlockHeader, opts: { source?: string } = {}): Promise { @@ -195,7 +195,12 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte this.#queue.put(() => { const counts = this.#impl.countTxs(); return Promise.resolve({ - itemCount: { pending: counts.pending, protected: counts.protected, mined: counts.mined }, + itemCount: { + pending: counts.pending, + protected: counts.protected, + mined: counts.mined, + softDeleted: counts.softDeleted, + }, }); }), () => this.#store.estimateSize(), diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 31979f3ea1a1..4b149185e63b 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -9,6 +9,7 @@ import type { L2Block, L2BlockId, L2BlockSource } from '@aztec/stdlib/block'; import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; import { DatabasePublicStateSource } from '@aztec/stdlib/trees'; import { BlockHeader, Tx, TxHash, type TxValidator } from '@aztec/stdlib/tx'; +import type { TelemetryClient } from '@aztec/telemetry-client'; import { TxArchive } from './archive/index.js'; import { DeletedPool } from './deleted_pool.js'; @@ -22,8 +23,12 @@ import { LowPriorityPreAddRule, NullifierConflictRule, type PoolOperations, + type PreAddContext, type PreAddPoolAccess, + TxPoolRejectionCode, + type TxPoolRejectionError, } from './eviction/index.js'; +import { TxPoolV2Instrumentation } from './instrumentation.js'; import { type AddTxsResult, DEFAULT_TX_POOL_V2_CONFIG, @@ -66,6 +71,8 @@ export class TxPoolV2Impl { #deletedPool: DeletedPool; #evictionManager: EvictionManager; #dateProvider: DateProvider; + #instrumentation: TxPoolV2Instrumentation; + #evictedTxHashes: Set = new Set(); #log: Logger; #callbacks: TxPoolV2Callbacks; @@ -74,6 +81,7 @@ export class TxPoolV2Impl { archiveStore: AztecAsyncKVStore, deps: TxPoolV2Dependencies, callbacks: TxPoolV2Callbacks, + telemetry: TelemetryClient, config: Partial = {}, dateProvider: DateProvider, log: Logger, @@ -89,6 +97,7 @@ export class TxPoolV2Impl { this.#archive = new TxArchive(archiveStore, this.#config.archivedTxLimit, log); this.#deletedPool = new DeletedPool(store, this.#txsDB, log); this.#dateProvider = dateProvider; + this.#instrumentation = new TxPoolV2Instrumentation(telemetry, () => this.#indices.getTotalMetadataBytes()); this.#log = log; this.#callbacks = callbacks; @@ -171,13 +180,16 @@ export class TxPoolV2Impl { this.#log.info(`Deleted ${toDelete.length} invalid/rejected transactions on startup`, { txHashes: toDelete }); } - async addPendingTxs(txs: Tx[], opts: { source?: string }): Promise { + async addPendingTxs(txs: Tx[], opts: { source?: string; feeComparisonOnly?: boolean }): Promise { const accepted: TxHash[] = []; const ignored: TxHash[] = []; const rejected: TxHash[] = []; + const errors = new Map(); const acceptedPending = new Set(); const poolAccess = this.#createPreAddPoolAccess(); + const preAddContext: PreAddContext | undefined = + opts.feeComparisonOnly !== undefined ? { feeComparisonOnly: opts.feeComparisonOnly } : undefined; await this.#store.transactionAsync(async () => { for (const tx of txs) { @@ -204,7 +216,15 @@ export class TxPoolV2Impl { accepted.push(txHash); } else { // Regular pending tx - validate and run pre-add rules - const result = await this.#tryAddRegularPendingTx(tx, opts, poolAccess, acceptedPending, ignored); + const result = await this.#tryAddRegularPendingTx( + tx, + opts, + poolAccess, + acceptedPending, + ignored, + errors, + preAddContext, + ); if (result.status === 'accepted') { acceptedPending.add(txHashStr); } else if (result.status === 'rejected') { @@ -221,6 +241,14 @@ export class TxPoolV2Impl { accepted.push(TxHash.fromString(txHashStr)); } + // Record metrics + if (ignored.length > 0) { + this.#instrumentation.recordIgnored(ignored.length); + } + if (rejected.length > 0) { + this.#instrumentation.recordRejected(rejected.length); + } + // Run post-add eviction rules for pending txs if (acceptedPending.size > 0) { const feePayers = Array.from(acceptedPending).map(txHash => this.#indices.getMetadata(txHash)!.feePayer); @@ -228,7 +256,7 @@ export class TxPoolV2Impl { await this.#evictionManager.evictAfterNewTxs(Array.from(acceptedPending), [...uniqueFeePayers]); } - return { accepted, ignored, rejected }; + return { accepted, ignored, rejected, ...(errors.size > 0 ? { errors } : {}) }; } /** Validates and adds a regular pending tx. Returns status. */ @@ -238,6 +266,8 @@ export class TxPoolV2Impl { poolAccess: PreAddPoolAccess, acceptedPending: Set, ignored: TxHash[], + errors: Map, + preAddContext?: PreAddContext, ): Promise<{ status: 'accepted' | 'ignored' | 'rejected' }> { const txHash = tx.getTxHash(); const txHashStr = txHash.toString(); @@ -249,24 +279,40 @@ export class TxPoolV2Impl { } // Run pre-add rules - const preAddResult = await this.#evictionManager.runPreAddRules(meta, poolAccess); + const preAddResult = await this.#evictionManager.runPreAddRules(meta, poolAccess, preAddContext); if (preAddResult.shouldIgnore) { - this.#log.debug(`Ignoring tx ${txHashStr}: ${preAddResult.reason}`); + this.#log.debug(`Ignoring tx ${txHashStr}: ${preAddResult.reason?.message ?? 'unknown reason'}`); + if (preAddResult.reason && preAddResult.reason.code !== TxPoolRejectionCode.INTERNAL_ERROR) { + errors.set(txHashStr, preAddResult.reason); + } return { status: 'ignored' }; } - // Evict conflicts - for (const evictHashStr of preAddResult.txHashesToEvict) { - await this.#deleteTx(evictHashStr); - this.#log.debug(`Evicted tx ${evictHashStr} due to higher-fee tx ${txHashStr}`, { - evictedTxHash: evictHashStr, - replacementTxHash: txHashStr, - }); - if (acceptedPending.has(evictHashStr)) { - // Evicted tx was from this batch - mark as ignored in result - acceptedPending.delete(evictHashStr); - ignored.push(TxHash.fromString(evictHashStr)); + // Evict conflicts, grouped by rule name for metrics + if (preAddResult.evictions && preAddResult.evictions.length > 0) { + const byReason = new Map(); + for (const { txHash: evictHash, reason } of preAddResult.evictions) { + const group = byReason.get(reason); + if (group) { + group.push(evictHash); + } else { + byReason.set(reason, [evictHash]); + } + } + for (const [reason, hashes] of byReason) { + await this.#evictTxs(hashes, reason); + } + for (const evictHashStr of preAddResult.txHashesToEvict) { + this.#log.debug(`Evicted tx ${evictHashStr} due to higher-fee tx ${txHashStr}`, { + evictedTxHash: evictHashStr, + replacementTxHash: txHashStr, + }); + if (acceptedPending.has(evictHashStr)) { + // Evicted tx was from this batch - mark as ignored in result + acceptedPending.delete(evictHashStr); + ignored.push(TxHash.fromString(evictHashStr)); + } } } @@ -327,9 +373,11 @@ export class TxPoolV2Impl { }); } - protectTxs(txHashes: TxHash[], block: BlockHeader): TxHash[] { + async protectTxs(txHashes: TxHash[], block: BlockHeader): Promise { const slotNumber = block.globalVariables.slotNumber; const missing: TxHash[] = []; + let softDeletedHits = 0; + let missingPreviouslyEvicted = 0; for (const txHash of txHashes) { const txHashStr = txHash.toString(); @@ -337,13 +385,44 @@ export class TxPoolV2Impl { if (this.#indices.has(txHashStr)) { // Update protection for existing tx this.#indices.updateProtection(txHashStr, slotNumber); + } else if (this.#deletedPool.isSoftDeleted(txHashStr)) { + // Resurrect soft-deleted tx as protected + const buffer = await this.#txsDB.getAsync(txHashStr); + if (buffer) { + const tx = Tx.fromBuffer(buffer); + await this.#addTx(tx, { protected: slotNumber }); + softDeletedHits++; + } else { + // Data missing despite soft-delete flag — treat as truly missing + this.#indices.setProtection(txHashStr, slotNumber); + missing.push(txHash); + } } else { - // Pre-record protection for tx we don't have yet + // Truly missing — pre-record protection for tx we don't have yet this.#indices.setProtection(txHashStr, slotNumber); missing.push(txHash); + if (this.#evictedTxHashes.has(txHashStr)) { + missingPreviouslyEvicted++; + } } } + // Record metrics + if (softDeletedHits > 0) { + this.#instrumentation.recordSoftDeletedHits(softDeletedHits); + } + if (missing.length > 0) { + this.#log.debug(`protectTxs missing tx hashes: ${missing.map(h => h.toString()).join(', ')}`); + this.#instrumentation.recordMissingOnProtect(missing.length); + } + if (missingPreviouslyEvicted > 0) { + this.#instrumentation.recordMissingPreviouslyEvicted(missingPreviouslyEvicted); + } + + this.#log.info( + `Protected ${txHashes.length} txs, missing: ${missing.length}, soft-deleted hits: ${softDeletedHits}`, + ); + return missing; } @@ -412,6 +491,7 @@ export class TxPoolV2Impl { // Step 3: Filter to only txs that have metadata and are not mined const txsToRestore = this.#indices.filterRestorable(expiredProtected); if (txsToRestore.length === 0) { + this.#log.debug(`Preparing for slot ${slotNumber}, no txs to unprotect`); return; } @@ -423,8 +503,9 @@ export class TxPoolV2Impl { // Step 5: Resolve nullifier conflicts and add winners to pending indices const { added, toEvict } = this.#applyNullifierConflictResolution(valid); - // Step 6: Delete invalid and evicted txs - await this.#deleteTxsBatch([...invalid, ...toEvict]); + // Step 6: Delete invalid txs and evict conflict losers + await this.#deleteTxsBatch(invalid); + await this.#evictTxs(toEvict, 'NullifierConflict'); // Step 7: Run eviction rules (enforce pool size limit) if (added.length > 0) { @@ -471,8 +552,9 @@ export class TxPoolV2Impl { // Step 6: Resolve nullifier conflicts and add winners to pending indices const { toEvict } = this.#applyNullifierConflictResolution(valid); - // Step 7: Delete invalid and evicted txs - await this.#deleteTxsBatch([...invalid, ...toEvict]); + // Step 7: Delete invalid txs and evict conflict losers + await this.#deleteTxsBatch(invalid); + await this.#evictTxs(toEvict, 'NullifierConflict'); this.#log.info( `Handled prune to block ${latestBlock.number}: ${valid.length} txs restored to pending, ${invalid.length} invalid, ${toEvict.length} evicted due to nullifier conflicts`, @@ -637,8 +719,17 @@ export class TxPoolV2Impl { // === Metrics === - countTxs(): { pending: number; protected: number; mined: number } { - return this.#indices.countTxs(); + countTxs(): { + pending: number; + protected: number; + mined: number; + softDeleted: number; + totalMetadataBytes: number; + } { + return { + ...this.#indices.countTxs(), + softDeleted: this.#deletedPool.getSoftDeletedCount(), + }; } // ============================================================================ @@ -672,9 +763,11 @@ export class TxPoolV2Impl { } const stateStr = typeof state === 'string' ? state : Object.keys(state)[0]; - this.#log.verbose(`Added ${stateStr} tx ${txHashStr}`, { + this.#log.debug(`Added tx ${txHashStr} as ${stateStr}`, { eventName: 'tx-added-to-pool', + txHash: txHashStr, state: stateStr, + source: opts.source, }); return meta; @@ -702,6 +795,29 @@ export class TxPoolV2Impl { } } + /** Evicts transactions: records eviction metric with reason, caches hashes, then deletes. */ + async #evictTxs(txHashes: string[], reason: string): Promise { + if (txHashes.length === 0) { + return; + } + this.#instrumentation.recordEvictions(txHashes.length, reason); + for (const txHashStr of txHashes) { + this.#log.debug(`Evicting tx ${txHashStr}`, { txHash: txHashStr, reason }); + this.#addToEvictedCache(txHashStr); + } + await this.#deleteTxsBatch(txHashes); + } + + /** Adds a tx hash to the bounded evicted cache, evicting the oldest entry if at capacity. */ + #addToEvictedCache(txHashStr: string): void { + if (this.#evictedTxHashes.size >= this.#config.evictedTxCacheSize) { + // FIFO eviction: remove the first (oldest) entry + const oldest = this.#evictedTxHashes.values().next().value!; + this.#evictedTxHashes.delete(oldest); + } + this.#evictedTxHashes.add(txHashStr); + } + // ============================================================================ // PRIVATE HELPERS - Validation & Conflict Resolution // ============================================================================ @@ -857,7 +973,9 @@ export class TxPoolV2Impl { if (preAddResult.shouldIgnore) { // Transaction rejected - mark for deletion from DB rejected.push(meta.txHash); - this.#log.debug(`Rejected tx ${meta.txHash} during rebuild: ${preAddResult.reason}`); + this.#log.debug( + `Rejected tx ${meta.txHash} during rebuild: ${preAddResult.reason?.message ?? 'unknown reason'}`, + ); continue; } @@ -893,7 +1011,7 @@ export class TxPoolV2Impl { getFeePayerPendingTxs: (feePayer: string) => this.#indices.getFeePayerPendingTxs(feePayer), getPendingTxCount: () => this.#indices.getPendingTxCount(), getLowestPriorityPending: (limit: number) => this.#indices.getLowestPriorityPending(limit), - deleteTxs: (txHashes: string[]) => this.#deleteTxsBatch(txHashes), + deleteTxs: (txHashes: string[], reason?: string) => this.#evictTxs(txHashes, reason ?? 'unknown'), }; } diff --git a/yarn-project/p2p/src/services/encoding.ts b/yarn-project/p2p/src/services/encoding.ts index c2a5e6dc5a87..9a4d610c4fa5 100644 --- a/yarn-project/p2p/src/services/encoding.ts +++ b/yarn-project/p2p/src/services/encoding.ts @@ -58,7 +58,8 @@ const DefaultMaxSizesKb: Record = { // Proposals may carry some tx objects, so we allow a larger size capped at 10mb // Note this may not be enough for carrying all tx objects in a block [TopicType.block_proposal]: 1024 * 10, - // TODO(palla/mbps): Check size for checkpoint proposal + // Checkpoint proposals carry almost the same data as a block proposal (see the lastBlockProposal) + // Only diff is an additional header, which is pretty small compared to the 10mb limit [TopicType.checkpoint_proposal]: 1024 * 10, }; diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts index ec8381d2d6cf..d9762a955d43 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts @@ -1,28 +1,51 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store'; import { Tx, type TxHash } from '@aztec/stdlib/tx'; +import { + type Histogram, + Metrics, + type TelemetryClient, + type UpDownCounter, + getTelemetryClient, +} from '@aztec/telemetry-client'; import type { TxSource } from './tx_source.js'; /** TxSource implementation that downloads txs from a file store. */ export class FileStoreTxSource implements TxSource { + private downloadsSuccess: UpDownCounter; + private downloadsFailed: UpDownCounter; + private downloadDuration: Histogram; + private downloadSize: Histogram; + private constructor( private readonly fileStore: ReadOnlyFileStore, private readonly baseUrl: string, private readonly basePath: string, private readonly log: Logger, - ) {} + telemetry: TelemetryClient, + ) { + const meter = telemetry.getMeter('file-store-tx-source'); + this.downloadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_SUCCESS); + this.downloadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_FAILED); + this.downloadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_DURATION); + this.downloadSize = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_SIZE); + } /** * Creates a FileStoreTxSource from a URL. * @param url - The file store URL (s3://, gs://, file://, http://, https://). + * @param basePath - Base path for tx files within the store. * @param log - Optional logger. + * @param telemetry - Optional telemetry client. * @returns The FileStoreTxSource instance, or undefined if creation fails. */ public static async create( url: string, basePath: string, log: Logger = createLogger('p2p:file_store_tx_source'), + telemetry: TelemetryClient = getTelemetryClient(), ): Promise { try { const fileStore = await createReadOnlyFileStore(url, log); @@ -30,7 +53,7 @@ export class FileStoreTxSource implements TxSource { log.warn(`Failed to create file store for URL: ${url}`); return undefined; } - return new FileStoreTxSource(fileStore, url, basePath, log); + return new FileStoreTxSource(fileStore, url, basePath, log, telemetry); } catch (err) { log.warn(`Error creating file store for URL: ${url}`, { error: err }); return undefined; @@ -45,10 +68,15 @@ export class FileStoreTxSource implements TxSource { return Promise.all( txHashes.map(async txHash => { const path = `${this.basePath}/txs/${txHash.toString()}.bin`; + const timer = new Timer(); try { const buffer = await this.fileStore.read(path); + this.downloadsSuccess.add(1); + this.downloadDuration.record(Math.ceil(timer.ms())); + this.downloadSize.record(buffer.length); return Tx.fromBuffer(buffer); } catch { + this.downloadsFailed.add(1); // Tx not found or error reading - return undefined return undefined; } @@ -60,14 +88,17 @@ export class FileStoreTxSource implements TxSource { /** * Creates FileStoreTxSource instances from URLs. * @param urls - Array of file store URLs. + * @param basePath - Base path for tx files within each store. * @param log - Optional logger. + * @param telemetry - Optional telemetry client. * @returns Array of successfully created FileStoreTxSource instances. */ export async function createFileStoreTxSources( urls: string[], basePath: string, log: Logger = createLogger('p2p:file_store_tx_source'), + telemetry: TelemetryClient = getTelemetryClient(), ): Promise { - const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log))); + const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry))); return sources.filter((s): s is FileStoreTxSource => s !== undefined); } diff --git a/yarn-project/p2p/src/services/tx_collection/instrumentation.ts b/yarn-project/p2p/src/services/tx_collection/instrumentation.ts index 780ee1b043f4..16068bb7045f 100644 --- a/yarn-project/p2p/src/services/tx_collection/instrumentation.ts +++ b/yarn-project/p2p/src/services/tx_collection/instrumentation.ts @@ -18,7 +18,13 @@ export class TxCollectionInstrumentation { const meter = client.getMeter(name); this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, { - [Attributes.TX_COLLECTION_METHOD]: ['fast-req-resp', 'fast-node-rpc', 'slow-req-resp', 'slow-node-rpc'], + [Attributes.TX_COLLECTION_METHOD]: [ + 'fast-req-resp', + 'fast-node-rpc', + 'slow-req-resp', + 'slow-node-rpc', + 'file-store', + ], }); this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX); diff --git a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts index e14f47a2b6dd..1a0ab6b19fde 100644 --- a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts +++ b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts @@ -9,6 +9,7 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { InMemoryTxPool } from '../../test-helpers/testbench-utils.js'; +import { FileStoreTxSource } from '../tx_collection/file_store_tx_source.js'; import type { TxFileStoreConfig } from './config.js'; import { TxFileStore } from './tx_file_store.js'; @@ -103,7 +104,7 @@ describe('TxFileStore', () => { await txFileStore!.flush(); expect(spy).toHaveBeenCalledWith(`${basePath}/txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { - compress: false, + compress: true, }); spy.mockRestore(); @@ -148,7 +149,7 @@ describe('TxFileStore', () => { await txFileStore!.flush(); expect(spy).toHaveBeenCalledWith(`${basePath}/txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { - compress: false, + compress: true, }); spy.mockRestore(); @@ -328,4 +329,24 @@ describe('TxFileStore', () => { expect(txFileStore!.getPendingUploadCount()).toBe(0); }); }); + + describe('compression round-trip', () => { + it('uploads compressed tx and reads it back via FileStoreTxSource', async () => { + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); + txFileStore!.start(); + + const tx = await makeTx(); + await txPool.addPendingTxs([tx]); + await txFileStore!.flush(); + + // Read back via FileStoreTxSource using the same local file store + const txSource = await FileStoreTxSource.create(`file://${tmpDir}`, basePath, log); + expect(txSource).toBeDefined(); + + const results = await txSource!.getTxsByHash([tx.getTxHash()]); + expect(results).toHaveLength(1); + expect(results[0]).toBeDefined(); + expect(results[0]!.toBuffer()).toEqual(tx.toBuffer()); + }); + }); }); diff --git a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts index 672bdd6d40cc..063c6256680f 100644 --- a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts +++ b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts @@ -146,7 +146,7 @@ export class TxFileStore { } await retry( - () => this.fileStore.save(path, tx.toBuffer(), { compress: false }), + () => this.fileStore.save(path, tx.toBuffer(), { compress: true }), `Uploading tx ${txHash}`, makeBackoff([0.1, 0.5, 2]), this.log, diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index d0b8ce352fde..cfa6a923fcb8 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -59,7 +59,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 { // === Core Operations (TxPoolV2) === - addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise { + addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise { const accepted: TxHash[] = []; const newTxs: Tx[] = []; for (const tx of txs) { diff --git a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.test.ts b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.test.ts index a2e426847320..8c791e955acc 100644 --- a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.test.ts +++ b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.test.ts @@ -3,17 +3,16 @@ import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/bra import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; -import { ProtocolContractsList, protocolContractsHash } from '@aztec/protocol-contracts'; +import { ProtocolContractsList } from '@aztec/protocol-contracts'; import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice'; import { PublicDataWrite } from '@aztec/stdlib/avm'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { EthAddress } from '@aztec/stdlib/block'; import { GasFees } from '@aztec/stdlib/gas'; import { accumulateCheckpointOutHashes } from '@aztec/stdlib/messaging'; -import { CheckpointConstantData } from '@aztec/stdlib/rollup'; import { mockProcessedTx } from '@aztec/stdlib/testing'; import { PublicDataTreeLeaf } from '@aztec/stdlib/trees'; -import type { ProcessedTx } from '@aztec/stdlib/tx'; +import type { CheckpointGlobalVariables, ProcessedTx } from '@aztec/stdlib/tx'; import { GlobalVariables } from '@aztec/stdlib/tx'; import { NativeWorldStateService } from '@aztec/world-state/native'; @@ -41,18 +40,16 @@ describe('LightweightCheckpointBuilder', () => { await worldState.close(); }); - const makeCheckpointConstants = (slotNumber: SlotNumber): CheckpointConstantData => { - return CheckpointConstantData.from({ + const makeCheckpointConstants = (slotNumber: SlotNumber): CheckpointGlobalVariables => { + return { chainId: Fr.ZERO, version: Fr.ZERO, - vkTreeRoot: getVKTreeRoot(), - protocolContractsHash, - proverId: Fr.ZERO, slotNumber, + timestamp: BigInt(slotNumber) * 123n, coinbase: EthAddress.ZERO, feeRecipient: AztecAddress.ZERO, gasFees: GasFees.empty(), - }); + }; }; const makeGlobalVariables = (blockNumber: BlockNumber, slotNumber: SlotNumber): GlobalVariables => { diff --git a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts index 188a2af864ee..cb789075c3f8 100644 --- a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts +++ b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts @@ -242,7 +242,7 @@ export class LightweightCheckpointBuilder { const newArchive = this.lastArchives[this.lastArchives.length - 1]; - const blobs = getBlobsPerL1Block(this.blobFields); + const blobs = await getBlobsPerL1Block(this.blobFields); const blobsHash = computeBlobsHashFromBlobs(blobs); const inHash = computeInHashFromL1ToL2Messages(this.l1ToL2Messages); @@ -253,8 +253,7 @@ export class LightweightCheckpointBuilder { ); const epochOutHash = accumulateCheckpointOutHashes([...this.previousCheckpointOutHashes, checkpointOutHash]); - // TODO(palla/mbps): Should we source this from the constants instead? - // timestamp of a checkpoint is the timestamp of the last block in the checkpoint. + // All blocks in the checkpoint have the same timestamp const timestamp = blocks[blocks.length - 1].timestamp; const totalManaUsed = blocks.reduce((acc, block) => acc.add(block.header.totalManaUsed), Fr.ZERO); diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 8234ce61ecf9..b27adcec0d97 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -250,7 +250,7 @@ export class TestContext { const previousCheckpointOutHashes = this.checkpointOutHashes; const builder = await LightweightCheckpointBuilder.startNewCheckpoint( checkpointNumber, - constants, + { ...constants, timestamp }, l1ToL2Messages, previousCheckpointOutHashes, cleanFork, diff --git a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts index 13d29d31bc98..01c0726d70bb 100644 --- a/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts +++ b/yarn-project/prover-client/src/orchestrator/block-building-helpers.ts @@ -253,8 +253,8 @@ export function getPublicChonkVerifierPrivateInputsFromTx(tx: Tx | ProcessedTx, // Build "hints" as the private inputs for the checkpoint root rollup circuit. // The `blobCommitments` will be accumulated and checked in the root rollup against the `finalBlobChallenges`. // The `blobsHash` will be validated on L1 against the submitted blob data. -export const buildBlobHints = (blobFields: Fr[]) => { - const blobs = getBlobsPerL1Block(blobFields); +export const buildBlobHints = async (blobFields: Fr[]) => { + const blobs = await getBlobsPerL1Block(blobFields); const blobCommitments = getBlobCommitmentsFromBlobs(blobs); const blobsHash = computeBlobsHashFromBlobs(blobs); return { blobCommitments, blobs, blobsHash }; diff --git a/yarn-project/prover-client/src/orchestrator/block-proving-state.ts b/yarn-project/prover-client/src/orchestrator/block-proving-state.ts index fb49b4be2d0e..53c7a594db82 100644 --- a/yarn-project/prover-client/src/orchestrator/block-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/block-proving-state.ts @@ -55,6 +55,7 @@ export class BlockProvingState { | ProofState | undefined; private builtBlockHeader: BlockHeader | undefined; + private builtArchive: AppendOnlyTreeSnapshot | undefined; private endState: StateReference | undefined; private endSpongeBlob: SpongeBlob | undefined; private txs: TxProvingState[] = []; @@ -232,6 +233,14 @@ export class BlockProvingState { return this.builtBlockHeader; } + public setBuiltArchive(archive: AppendOnlyTreeSnapshot) { + this.builtArchive = archive; + } + + public getBuiltArchive() { + return this.builtArchive; + } + public getStartSpongeBlob() { return this.startSpongeBlob; } diff --git a/yarn-project/prover-client/src/orchestrator/block_building_helpers.test.ts b/yarn-project/prover-client/src/orchestrator/block_building_helpers.test.ts index e38d60e8ccb6..5f1755f192fe 100644 --- a/yarn-project/prover-client/src/orchestrator/block_building_helpers.test.ts +++ b/yarn-project/prover-client/src/orchestrator/block_building_helpers.test.ts @@ -16,7 +16,7 @@ describe('buildBlobHints', () => { encodeCheckpointEndMarker({ numBlobFields: blobFieldsWithoutEndMarker.length + 1 }), ]); - const { blobCommitments, blobsHash, blobs } = buildBlobHints(blobFields); + const { blobCommitments, blobsHash, blobs } = await buildBlobHints(blobFields); expect(blobs.length).toBe(1); const onlyBlob = blobs[0]; @@ -37,7 +37,7 @@ describe('buildBlobHints', () => { const zStr = challengeZ.toString(); expect(zStr).toMatchInlineSnapshot(`"0x11d6daed56531bd5c5acf341663d21089bb96913f4e716dca3cdb01b8d5735a3"`); - const proof = onlyBlob.evaluate(challengeZ, true /* verifyProof */); + const proof = await onlyBlob.evaluate(challengeZ, true /* verifyProof */); const yStr = proof.y.toString(); expect(yStr).toMatchInlineSnapshot(`"0x6033e46c697b3de1a5ddedb940ae6ccdb6efc0adeb255336b0220d3fd4b76720"`); diff --git a/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts b/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts index 7b92edcd2d6e..32a813c6bc25 100644 --- a/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts @@ -85,7 +85,7 @@ export class CheckpointProvingState { typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH >, public parentEpoch: EpochProvingState, - private onBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => void, + private onBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise, ) { this.blockProofs = new UnbalancedTreeStore(totalNumBlocks); this.firstBlockNumber = BlockNumber(headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber + 1); @@ -245,7 +245,7 @@ export class CheckpointProvingState { this.endBlobAccumulator = await accumulateBlobs(this.blobFields!, startBlobAccumulator); this.startBlobAccumulator = startBlobAccumulator; - this.onBlobAccumulatorSet(this); + await this.onBlobAccumulatorSet(this); return this.endBlobAccumulator; } @@ -271,7 +271,7 @@ export class CheckpointProvingState { return this.totalNumBlocks === 1 ? 'rollup-checkpoint-root-single-block' : 'rollup-checkpoint-root'; } - public getCheckpointRootRollupInputs() { + public async getCheckpointRootRollupInputs() { const proofs = this.#getChildProofsForRoot(); const nonEmptyProofs = proofs.filter(p => !!p); if (proofs.length !== nonEmptyProofs.length) { @@ -287,7 +287,7 @@ export class CheckpointProvingState { // `blobFields` must've been set if `startBlobAccumulator` is set (in `accumulateBlobs`). const blobFields = this.blobFields!; - const { blobCommitments, blobsHash } = buildBlobHints(blobFields); + const { blobCommitments, blobsHash } = await buildBlobHints(blobFields); const hints = CheckpointRootRollupHints.from({ previousBlockHeader: this.headerOfLastBlockInPreviousCheckpoint, diff --git a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts index 97cbc1013827..a551082873c6 100644 --- a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts @@ -76,7 +76,7 @@ export class EpochProvingState { public readonly epochNumber: EpochNumber, public readonly totalNumCheckpoints: number, private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges, - private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => void, + private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise, private completionCallback: (result: ProvingResult) => void, private rejectionCallback: (reason: string) => void, ) { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 4ac203a6c271..001e91e0de34 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -71,11 +71,6 @@ import { EpochProvingState, type ProvingResult, type TreeSnapshots } from './epo import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js'; import { TxProvingState } from './tx-proving-state.js'; -type WorldStateFork = { - fork: MerkleTreeWriteOperations; - cleanupPromise: Promise | undefined; -}; - /** * Implements an event driven proving scheduler to build the recursive proof tree. The idea being: * 1. Transactions are provided to the scheduler post simulation. @@ -97,7 +92,7 @@ export class ProvingOrchestrator implements EpochProver { private provingPromise: Promise | undefined = undefined; private metrics: ProvingOrchestratorMetrics; // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections - private dbs: Map = new Map(); + private dbs: Map = new Map(); private logger: Logger; constructor( @@ -182,7 +177,7 @@ export class ProvingOrchestrator implements EpochProver { const db = await this.dbProvider.fork(lastBlockNumber); const firstBlockNumber = BlockNumber(lastBlockNumber + 1); - this.dbs.set(firstBlockNumber, { fork: db, cleanupPromise: undefined }); + this.dbs.set(firstBlockNumber, db); // Get archive sibling path before any block in this checkpoint lands. const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db); @@ -240,9 +235,9 @@ export class ProvingOrchestrator implements EpochProver { if (!this.dbs.has(blockNumber)) { // Fork world state at the end of the immediately previous block const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1)); - this.dbs.set(blockNumber, { fork: db, cleanupPromise: undefined }); + this.dbs.set(blockNumber, db); } - const db = this.dbs.get(blockNumber)!.fork; + const db = this.getDbForBlock(blockNumber); // Get archive snapshot and sibling path before any txs in this block lands. const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); @@ -317,7 +312,7 @@ export class ProvingOrchestrator implements EpochProver { this.logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`); - const db = this.dbs.get(blockNumber)!.fork; + const db = this.getDbForBlock(blockNumber); const lastArchive = provingState.lastArchiveTreeSnapshot; const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot; const spongeBlobState = provingState.getStartSpongeBlob().clone(); @@ -445,14 +440,20 @@ export class ProvingOrchestrator implements EpochProver { throw new Error('Block header mismatch'); } - // Get db for this block - const db = this.dbs.get(provingState.blockNumber)!.fork; + // Get db for this block and remove from map — no other code should use it after this point. + const db = this.getDbForBlock(provingState.blockNumber); + this.dbs.delete(provingState.blockNumber); - // Update the archive tree, so we're ready to start processing the next block: - this.logger.verbose( - `Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`, - ); - await db.updateArchive(header); + // Update the archive tree, capture the snapshot, and close the fork deterministically. + try { + this.logger.verbose( + `Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`, + ); + await db.updateArchive(header); + provingState.setBuiltArchive(await getTreeSnapshot(MerkleTreeId.ARCHIVE, db)); + } finally { + await db.close(); + } await this.verifyBuiltBlockAgainstSyncedState(provingState); @@ -472,6 +473,13 @@ export class ProvingOrchestrator implements EpochProver { this.logger.debug('Block root rollup proof not built yet, skipping header check.'); return; } + + const newArchive = provingState.getBuiltArchive(); + if (!newArchive) { + this.logger.debug('Archive snapshot not yet captured, skipping header check.'); + return; + } + const header = await buildHeaderFromCircuitOutputs(output); if (!(await header.hash()).equals(await builtBlockHeader.hash())) { @@ -480,11 +488,7 @@ export class ProvingOrchestrator implements EpochProver { return; } - // Get db for this block const blockNumber = provingState.blockNumber; - const db = this.dbs.get(blockNumber)!.fork; - - const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber)); if (!syncedArchive.equals(newArchive)) { this.logger.error( @@ -502,12 +506,6 @@ export class ProvingOrchestrator implements EpochProver { provingState.reject(`New archive mismatch.`); return; } - - // TODO(palla/prover): This closes the fork only on the happy path. If this epoch orchestrator - // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup, - // but have to make sure it only runs once all operations are completed, otherwise some function here - // will attempt to access the fork after it was closed. - void this.cleanupDBFork(blockNumber); } /** @@ -523,6 +521,19 @@ export class ProvingOrchestrator implements EpochProver { } this.provingState?.cancel(); + + for (const [blockNumber, db] of this.dbs.entries()) { + void db.close().catch(err => this.logger.error(`Error closing db for block ${blockNumber}`, err)); + } + this.dbs.clear(); + } + + private getDbForBlock(blockNumber: BlockNumber): MerkleTreeWriteOperations { + const db = this.dbs.get(blockNumber); + if (!db) { + throw new Error(`World state fork for block ${blockNumber} not found.`); + } + return db; } /** @@ -554,24 +565,6 @@ export class ProvingOrchestrator implements EpochProver { return epochProofResult; } - private async cleanupDBFork(blockNumber: BlockNumber): Promise { - this.logger.debug(`Cleaning up world state fork for ${blockNumber}`); - const fork = this.dbs.get(blockNumber); - if (!fork) { - return; - } - - try { - if (!fork.cleanupPromise) { - fork.cleanupPromise = fork.fork.close(); - } - await fork.cleanupPromise; - this.dbs.delete(blockNumber); - } catch (err) { - this.logger.error(`Error closing db for block ${blockNumber}`, err); - } - } - /** * Enqueue a job to be scheduled * @param provingState - The proving state object being operated on @@ -894,17 +887,15 @@ export class ProvingOrchestrator implements EpochProver { const leafLocation = provingState.setBlockRootRollupProof(result); const checkpointProvingState = provingState.parentCheckpoint; - // If the proofs were slower than the block header building, then we need to try validating the block header hashes here. + // Verification is called from both here and setBlockCompleted. Whichever runs last + // will be the first to see all three pieces (header, proof output, archive) and run the checks. await this.verifyBuiltBlockAgainstSyncedState(provingState); if (checkpointProvingState.totalNumBlocks === 1) { - this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState); + await this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState); } else { - this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation); + await this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation); } - - // We are finished with the block at this point, ensure the fork is cleaned up - void this.cleanupDBFork(provingState.blockNumber); }, ); } @@ -1009,14 +1000,14 @@ export class ProvingOrchestrator implements EpochProver { }, signal => this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber), ), - result => { + async result => { provingState.setBlockMergeRollupProof(location, result); - this.checkAndEnqueueNextBlockMergeRollup(provingState, location); + await this.checkAndEnqueueNextBlockMergeRollup(provingState, location); }, ); } - private enqueueCheckpointRootRollup(provingState: CheckpointProvingState) { + private async enqueueCheckpointRootRollup(provingState: CheckpointProvingState) { if (!provingState.verifyState()) { this.logger.debug('Not running checkpoint root rollup. State no longer valid.'); return; @@ -1031,7 +1022,7 @@ export class ProvingOrchestrator implements EpochProver { this.logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`); - const inputs = provingState.getCheckpointRootRollupInputs(); + const inputs = await provingState.getCheckpointRootRollupInputs(); this.deferredProving( provingState, @@ -1191,25 +1182,28 @@ export class ProvingOrchestrator implements EpochProver { this.enqueueBlockRootRollup(provingState); } - private checkAndEnqueueNextBlockMergeRollup(provingState: CheckpointProvingState, currentLocation: TreeNodeLocation) { + private async checkAndEnqueueNextBlockMergeRollup( + provingState: CheckpointProvingState, + currentLocation: TreeNodeLocation, + ) { if (!provingState.isReadyForBlockMerge(currentLocation)) { return; } const parentLocation = provingState.getParentLocation(currentLocation); if (parentLocation.level === 0) { - this.checkAndEnqueueCheckpointRootRollup(provingState); + await this.checkAndEnqueueCheckpointRootRollup(provingState); } else { this.enqueueBlockMergeRollup(provingState, parentLocation); } } - private checkAndEnqueueCheckpointRootRollup(provingState: CheckpointProvingState) { + private async checkAndEnqueueCheckpointRootRollup(provingState: CheckpointProvingState) { if (!provingState.isReadyForCheckpointRoot()) { return; } - this.enqueueCheckpointRootRollup(provingState); + await this.enqueueCheckpointRootRollup(provingState); } private checkAndEnqueueNextCheckpointMergeRollup(provingState: EpochProvingState, currentLocation: TreeNodeLocation) { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts index 6f5e642e54fd..3d148f6beca7 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts @@ -151,9 +151,7 @@ describe('prover/orchestrator/errors', () => { await orchestrator.startNewBlock(blockNumber, timestamp, 1); orchestrator.cancel(); - await expect(async () => await orchestrator.addTxs(block.txs)).rejects.toThrow( - 'Invalid proving state when adding a tx', - ); + await expect(async () => await orchestrator.addTxs(block.txs)).rejects.toThrow('World state fork for block'); }); it('rejects if too many l1 to l2 messages are provided', async () => { diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index 2ac46e75b7fc..0ada8a3c4267 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -183,8 +183,8 @@ describe('prover/orchestrator', () => { const result = await orchestrator.finalizeEpoch(); expect(result.proof).toBeDefined(); - const numForks = orchestrator.getNumActiveForks(); - expect(numForks).toEqual(0); + // Forks are closed deterministically in setBlockCompleted, so no cancel() needed. + expect(orchestrator.getNumActiveForks()).toEqual(0); }); it('can start chonk verifier proofs before adding processed txs', async () => { diff --git a/yarn-project/prover-node/src/bin/run-failed-epoch.ts b/yarn-project/prover-node/src/bin/run-failed-epoch.ts index 2eb584adad9b..9406faef4fc1 100644 --- a/yarn-project/prover-node/src/bin/run-failed-epoch.ts +++ b/yarn-project/prover-node/src/bin/run-failed-epoch.ts @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; import { EthAddress } from '@aztec/foundation/eth-address'; import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc'; @@ -22,8 +23,10 @@ async function rerunFailedEpoch(provingJobUrl: string, baseLocalDir: string) { const dataDir = join(localDir, 'state'); const env = getProverNodeConfigFromEnv(); + const l1Config = getL1ContractsConfigEnvVars(); const config = { - ...getProverNodeConfigFromEnv(), + ...env, + ...l1Config, dataDirectory: dataDir, dataStoreMapSizeKb: env.dataStoreMapSizeKb ?? 1024 * 1024, proverId: env.proverId ?? EthAddress.random(), diff --git a/yarn-project/prover-node/src/config.test.ts b/yarn-project/prover-node/src/config.test.ts index e50f6082715e..da92da3a332b 100644 --- a/yarn-project/prover-node/src/config.test.ts +++ b/yarn-project/prover-node/src/config.test.ts @@ -20,9 +20,9 @@ describe('createKeyStoreForProver', () => { ): ProverNodeConfig => { const mockValue = (val: string) => ({ getValue: () => val }); return { - publisherPrivateKeys: publisherPrivateKeys.map(mockValue), + proverPublisherPrivateKeys: publisherPrivateKeys.map(mockValue), proverId, - publisherAddresses, + proverPublisherAddresses: publisherAddresses, web3SignerUrl, } as ProverNodeConfig; }; @@ -115,4 +115,17 @@ describe('createKeyStoreForProver', () => { validators: undefined, }); }); + + it('should fall through to publisher keys when web3SignerUrl is set but proverId is missing', () => { + const config = createMockConfig([mockKey1], undefined, [], mockSignerUrl); + const result = createKeyStoreForProver(config); + + expect(result).toEqual({ + schemaVersion: 1, + slasher: undefined, + prover: mockKey1, + remoteSigner: undefined, + validators: undefined, + }); + }); }); diff --git a/yarn-project/prover-node/src/config.ts b/yarn-project/prover-node/src/config.ts index dac85d6f3f45..9ba657d77f0a 100644 --- a/yarn-project/prover-node/src/config.ts +++ b/yarn-project/prover-node/src/config.ts @@ -1,18 +1,16 @@ -import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/config'; import type { ACVMConfig, BBConfig } from '@aztec/bb-prover/config'; -import { type GenesisStateConfig, genesisStateConfigMappings } from '@aztec/ethereum/config'; import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings, numberConfigHelper, + pickConfigMappings, } from '@aztec/foundation/config'; import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config'; import { type KeyStoreConfig, keyStoreConfigMappings } from '@aztec/node-keystore/config'; import { ethPrivateKeySchema } from '@aztec/node-keystore/schemas'; import type { KeyStore } from '@aztec/node-keystore/types'; import { type SharedNodeConfig, sharedNodeConfigMappings } from '@aztec/node-lib/config'; -import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p/config'; import { type ProverAgentConfig, type ProverBrokerConfig, @@ -21,24 +19,19 @@ import { } from '@aztec/prover-client/broker/config'; import { type ProverClientUserConfig, bbConfigMappings, proverClientConfigMappings } from '@aztec/prover-client/config'; import { - type PublisherConfig, - type TxSenderConfig, - getPublisherConfigMappings, - getTxSenderConfigMappings, + type ProverPublisherConfig, + type ProverTxSenderConfig, + proverPublisherConfigMappings, + proverTxSenderConfigMappings, } from '@aztec/sequencer-client/config'; -import { type WorldStateConfig, worldStateConfigMappings } from '@aztec/world-state/config'; - -export type ProverNodeConfig = ArchiverConfig & - ProverClientUserConfig & - P2PConfig & - WorldStateConfig & - PublisherConfig & - TxSenderConfig & + +export type ProverNodeConfig = ProverClientUserConfig & + ProverPublisherConfig & + ProverTxSenderConfig & DataStoreConfig & KeyStoreConfig & - SharedNodeConfig & SpecificProverNodeConfig & - GenesisStateConfig; + Pick; export type SpecificProverNodeConfig = { proverNodeMaxPendingJobs: number; @@ -53,7 +46,7 @@ export type SpecificProverNodeConfig = { txGatheringMaxParallelRequestsPerNode: number; }; -const specificProverNodeConfigMappings: ConfigMappingsType = { +export const specificProverNodeConfigMappings: ConfigMappingsType = { proverNodeMaxPendingJobs: { env: 'PROVER_NODE_MAX_PENDING_JOBS', description: 'The maximum number of pending jobs for the prover node', @@ -108,15 +101,11 @@ const specificProverNodeConfigMappings: ConfigMappingsType = { ...dataConfigMappings, ...keyStoreConfigMappings, - ...archiverConfigMappings, ...proverClientConfigMappings, - ...p2pConfigMappings, - ...worldStateConfigMappings, - ...getPublisherConfigMappings('PROVER'), - ...getTxSenderConfigMappings('PROVER'), + ...proverPublisherConfigMappings, + ...proverTxSenderConfigMappings, ...specificProverNodeConfigMappings, - ...genesisStateConfigMappings, - ...sharedNodeConfigMappings, + ...pickConfigMappings(sharedNodeConfigMappings, ['web3SignerUrl']), }; export function getProverNodeConfigFromEnv(): ProverNodeConfig { @@ -143,7 +132,7 @@ function createKeyStoreFromWeb3Signer(config: ProverNodeConfig): KeyStore | unde } // Also, we need at least one publisher address. - const publishers = config.publisherAddresses ?? []; + const publishers = config.proverPublisherAddresses ?? []; if (publishers.length === 0) { return undefined; @@ -164,8 +153,8 @@ function createKeyStoreFromWeb3Signer(config: ProverNodeConfig): KeyStore | unde function createKeyStoreFromPublisherKeys(config: ProverNodeConfig): KeyStore | undefined { // Extract the publisher keys from the provided config. - const publisherKeys = config.publisherPrivateKeys - ? config.publisherPrivateKeys.map((k: { getValue: () => string }) => ethPrivateKeySchema.parse(k.getValue())) + const publisherKeys = config.proverPublisherPrivateKeys + ? config.proverPublisherPrivateKeys.map((k: { getValue: () => string }) => ethPrivateKeySchema.parse(k.getValue())) : []; // There must be at least 1. @@ -194,7 +183,10 @@ function createKeyStoreFromPublisherKeys(config: ProverNodeConfig): KeyStore | u export function createKeyStoreForProver(config: ProverNodeConfig): KeyStore | undefined { if (config.web3SignerUrl !== undefined && config.web3SignerUrl.length > 0) { - return createKeyStoreFromWeb3Signer(config); + const keyStore = createKeyStoreFromWeb3Signer(config); + if (keyStore) { + return keyStore; + } } return createKeyStoreFromPublisherKeys(config); diff --git a/yarn-project/prover-node/src/factory.ts b/yarn-project/prover-node/src/factory.ts index dd8b755985f1..c0d3660b09dd 100644 --- a/yarn-project/prover-node/src/factory.ts +++ b/yarn-project/prover-node/src/factory.ts @@ -1,7 +1,7 @@ -import { type Archiver, createArchiver } from '@aztec/archiver'; -import { BBCircuitVerifier, QueuedIVCVerifier, TestCircuitVerifier } from '@aztec/bb-prover'; -import { createBlobClientWithFileStores } from '@aztec/blob-client/client'; -import { EpochCache } from '@aztec/epoch-cache'; +import type { Archiver } from '@aztec/archiver'; +import type { BlobClientInterface } from '@aztec/blob-client/client'; +import { Blob } from '@aztec/blob-lib'; +import type { EpochCacheInterface } from '@aztec/epoch-cache'; import { createEthereumChain } from '@aztec/ethereum/chain'; import { RollupContract } from '@aztec/ethereum/contracts'; import { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; @@ -9,26 +9,28 @@ import { PublisherManager } from '@aztec/ethereum/publisher-manager'; import { pick } from '@aztec/foundation/collection'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; -import type { DataStoreConfig } from '@aztec/kv-store/config'; -import { type KeyStoreConfig, KeystoreManager, loadKeystores, mergeKeystores } from '@aztec/node-keystore'; -import { trySnapshotSync } from '@aztec/node-lib/actions'; -import { - createForwarderL1TxUtilsFromEthSigner, - createL1TxUtilsFromEthSignerWithStore, -} from '@aztec/node-lib/factories'; -import { NodeRpcTxSource, type P2PClientDeps, createP2PClient } from '@aztec/p2p'; -import { type ProverClientConfig, createProverClient } from '@aztec/prover-client'; +import { KeystoreManager } from '@aztec/node-keystore'; +import { createForwarderL1TxUtilsFromSigners, createL1TxUtilsFromSigners } from '@aztec/node-lib/factories'; +import { type ProverClientConfig, type ProverClientUserConfig, createProverClient } from '@aztec/prover-client'; import { createAndStartProvingBroker } from '@aztec/prover-client/broker'; -import type { AztecNode, ProvingJobBroker } from '@aztec/stdlib/interfaces/server'; -import { P2PClientType } from '@aztec/stdlib/p2p'; -import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees'; -import { getPackageVersion } from '@aztec/stdlib/update-checker'; +import { + type ProverPublisherConfig, + type ProverTxSenderConfig, + getPublisherConfigFromProverConfig, +} from '@aztec/sequencer-client'; +import type { + AztecNode, + ITxProvider, + ProverConfig, + ProvingJobBroker, + Service, + WorldStateSynchronizer, +} from '@aztec/stdlib/interfaces/server'; import { L1Metrics, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client'; -import { createWorldStateSynchronizer } from '@aztec/world-state'; import { createPublicClient, fallback, http } from 'viem'; -import { type ProverNodeConfig, createKeyStoreForProver } from './config.js'; +import type { SpecificProverNodeConfig } from './config.js'; import { EpochMonitor } from './monitors/epoch-monitor.js'; import { ProverNode } from './prover-node.js'; import { ProverPublisherFactory } from './prover-publisher-factory.js'; @@ -37,54 +39,42 @@ export type ProverNodeDeps = { telemetry?: TelemetryClient; log?: Logger; aztecNodeTxProvider?: Pick; - archiver?: Archiver; + archiver: Archiver; publisherFactory?: ProverPublisherFactory; broker?: ProvingJobBroker; l1TxUtils?: L1TxUtils; dateProvider?: DateProvider; - p2pClientDeps?: P2PClientDeps; + worldStateSynchronizer: WorldStateSynchronizer; + p2pClient: { getTxProvider(): ITxProvider } & Partial; + epochCache: EpochCacheInterface; + blobClient: BlobClientInterface; + keyStoreManager?: KeystoreManager; }; -/** Creates a new prover node given a config. */ +/** Creates a new prover node subsystem given a config and dependencies */ export async function createProverNode( - userConfig: ProverNodeConfig & DataStoreConfig & KeyStoreConfig, - deps: ProverNodeDeps = {}, - options: { - prefilledPublicData?: PublicDataTreeLeaf[]; - } = {}, + userConfig: SpecificProverNodeConfig & + ProverConfig & + ProverClientUserConfig & + ProverPublisherConfig & + ProverTxSenderConfig, + deps: ProverNodeDeps, ) { const config = { ...userConfig }; const telemetry = deps.telemetry ?? getTelemetryClient(); const dateProvider = deps.dateProvider ?? new DateProvider(); - const blobClient = await createBlobClientWithFileStores(config, createLogger('prover-node:blob-client:client')); - const log = deps.log ?? createLogger('prover-node'); - - // Build a key store from file if given or from environment otherwise - let keyStoreManager: KeystoreManager | undefined; - const keyStoreProvided = config.keyStoreDirectory !== undefined && config.keyStoreDirectory.length > 0; - if (keyStoreProvided) { - const keyStores = loadKeystores(config.keyStoreDirectory!); - keyStoreManager = new KeystoreManager(mergeKeystores(keyStores)); - } else { - const keyStore = createKeyStoreForProver(config); - if (keyStore) { - keyStoreManager = new KeystoreManager(keyStore); - } - } + const log = deps.log ?? createLogger('prover'); - await keyStoreManager?.validateSigners(); + const { p2pClient, archiver, keyStoreManager, worldStateSynchronizer } = deps; // Extract the prover signers from the key store and verify that we have one. + await keyStoreManager?.validateSigners(); const proverSigners = keyStoreManager?.createProverSigners(); if (proverSigners === undefined) { throw new Error('Failed to create prover key store configuration'); } else if (proverSigners.signers.length === 0) { throw new Error('No prover signers found in the key store'); - } else if (!keyStoreProvided) { - log.warn( - 'KEY STORE CREATED FROM ENVIRONMENT, IT IS RECOMMENDED TO USE A FILE-BASED KEY STORE IN PRODUCTION ENVIRONMENTS', - ); } log.info(`Creating prover with publishers ${proverSigners.signers.map(signer => signer.address.toString()).join()}`); @@ -96,27 +86,7 @@ export async function createProverNode( const proverId = proverSigners.id ?? proverIdInUserConfig ?? proverSigners.signers[0].address; // Now create the prover client configuration from this. - const proverClientConfig: ProverClientConfig = { - ...config, - proverId, - }; - - await trySnapshotSync(config, log); - - const epochCache = await EpochCache.create(config.l1Contracts.rollupAddress, config); - - const archiver = - deps.archiver ?? - (await createArchiver(config, { blobClient, epochCache, telemetry, dateProvider }, { blockUntilSync: true })); - log.verbose(`Created archiver and synced to block ${await archiver.getBlockNumber()}`); - - const worldStateSynchronizer = await createWorldStateSynchronizer( - config, - archiver, - options.prefilledPublicData, - telemetry, - ); - await worldStateSynchronizer.start(); + const proverClientConfig: ProverClientConfig = { ...config, proverId }; const broker = deps.broker ?? (await createAndStartProvingBroker(config, telemetry)); @@ -135,15 +105,15 @@ export async function createProverNode( const l1TxUtils = deps.l1TxUtils ? [deps.l1TxUtils] - : config.publisherForwarderAddress - ? await createForwarderL1TxUtilsFromEthSigner( + : config.proverPublisherForwarderAddress + ? await createForwarderL1TxUtilsFromSigners( publicClient, proverSigners.signers, - config.publisherForwarderAddress, + config.proverPublisherForwarderAddress, { ...config, scope: 'prover' }, - { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider }, + { telemetry, logger: log.createChild('l1-tx-utils'), dateProvider, kzg: Blob.getViemKzgInstance() }, ) - : await createL1TxUtilsFromEthSignerWithStore( + : await createL1TxUtilsFromSigners( publicClient, proverSigners.signers, { ...config, scope: 'prover' }, @@ -154,37 +124,12 @@ export async function createProverNode( deps.publisherFactory ?? new ProverPublisherFactory(config, { rollupContract, - publisherManager: new PublisherManager(l1TxUtils, config, log.getBindings()), + publisherManager: new PublisherManager(l1TxUtils, getPublisherConfigFromProverConfig(config), log.getBindings()), telemetry, }); - const proofVerifier = new QueuedIVCVerifier( - config, - config.realProofs || config.debugForceTxProofVerification - ? await BBCircuitVerifier.new(config) - : new TestCircuitVerifier(config.proverTestVerificationDelayMs), - ); - - const p2pClient = await createP2PClient( - P2PClientType.Prover, - config, - archiver, - proofVerifier, - worldStateSynchronizer, - epochCache, - getPackageVersion() ?? '', - dateProvider, - telemetry, - { - ...deps.p2pClientDeps, - txCollectionNodeSources: [ - ...(deps.p2pClientDeps?.txCollectionNodeSources ?? []), - ...(deps.aztecNodeTxProvider ? [new NodeRpcTxSource(deps.aztecNodeTxProvider, 'TestNode')] : []), - ], - }, - ); - - await p2pClient.start(); + // TODO(#20393): Check that the tx collection node sources are properly injected + // See aztecNodeTxProvider const proverNodeConfig = { ...pick( @@ -216,6 +161,9 @@ export async function createProverNode( l1TxUtils.map(utils => utils.getSenderAddress()), ); + // Extract the shared delayer from the first L1TxUtils instance (all instances share the same delayer) + const delayer = l1TxUtils[0]?.delayer; + return new ProverNode( prover, publisherFactory, @@ -229,5 +177,7 @@ export async function createProverNode( l1Metrics, proverNodeConfig, telemetry, + delayer, + dateProvider, ); } diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index 8983a5047bdb..499049701e89 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -149,7 +149,9 @@ export class EpochProvingJob implements Traceable { try { const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields()); + this.log.info(`Blob fields per checkpoint: ${timer.ms()}ms`); const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint); + this.log.info(`Final blob batching challeneger: ${timer.ms()}ms`); this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges); await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values())); diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index 06d61cb3f080..8adf039d7651 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -1,5 +1,6 @@ import type { Archiver } from '@aztec/archiver'; import type { RollupContract } from '@aztec/ethereum/contracts'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types'; import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection'; import type { Fr } from '@aztec/foundation/curves/bn254'; @@ -7,7 +8,6 @@ import { memoize } from '@aztec/foundation/decorators'; import { createLogger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; import type { DataStoreConfig } from '@aztec/kv-store/config'; -import type { P2PClient } from '@aztec/p2p'; import { PublicProcessorFactory } from '@aztec/simulator/server'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; @@ -17,6 +17,7 @@ import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers import { type EpochProverManager, EpochProvingJobTerminalState, + type ITxProvider, type ProverNodeApi, type Service, type WorldStateSyncStatus, @@ -24,7 +25,6 @@ import { tryStop, } from '@aztec/stdlib/interfaces/server'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; -import type { P2PClientType } from '@aztec/stdlib/p2p'; import type { Tx } from '@aztec/stdlib/tx'; import { Attributes, @@ -55,7 +55,6 @@ type DataStoreOptions = Pick & Pick = new Map(); private config: ProverNodeOptions; @@ -73,12 +72,14 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable protected readonly l1ToL2MessageSource: L1ToL2MessageSource, protected readonly contractDataSource: ContractDataSource, protected readonly worldState: WorldStateSynchronizer, - protected readonly p2pClient: Pick, 'getTxProvider'> & Partial, + protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial, protected readonly epochsMonitor: EpochMonitor, protected readonly rollupContract: RollupContract, protected readonly l1Metrics: L1Metrics, config: Partial = {}, protected readonly telemetryClient: TelemetryClient = getTelemetryClient(), + private delayer?: Delayer, + private readonly dateProvider: DateProvider = new DateProvider(), ) { this.config = { proverNodePollingIntervalMs: 1_000, @@ -111,6 +112,11 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable return this.p2pClient; } + /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */ + public getDelayer(): Delayer | undefined { + return this.delayer; + } + /** * Handles an epoch being completed by starting a proof for it if there are no active jobs for it. * @param epochNumber - The epoch number that was just completed. @@ -155,17 +161,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable /** * Stops the prover node and all its dependencies. + * Resources not owned by this node (shared with the parent aztec-node) are skipped. */ async stop() { this.log.info('Stopping ProverNode'); await this.epochsMonitor.stop(); await this.prover.stop(); - await tryStop(this.p2pClient); - await tryStop(this.l2BlockSource); await tryStop(this.publisherFactory); this.publisher?.interrupt(); await Promise.all(Array.from(this.jobs.values()).map(job => job.stop())); - await this.worldState.stop(); this.rewardsMetrics.stop(); this.l1Metrics.stop(); await this.telemetryClient.stop(); diff --git a/yarn-project/prover-node/src/prover-publisher-factory.ts b/yarn-project/prover-node/src/prover-publisher-factory.ts index 8bcf72b321ed..8e1b88d1560e 100644 --- a/yarn-project/prover-node/src/prover-publisher-factory.ts +++ b/yarn-project/prover-node/src/prover-publisher-factory.ts @@ -2,14 +2,14 @@ import type { RollupContract } from '@aztec/ethereum/contracts'; import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { PublisherManager } from '@aztec/ethereum/publisher-manager'; import type { LoggerBindings } from '@aztec/foundation/log'; -import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client'; +import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client'; import type { TelemetryClient } from '@aztec/telemetry-client'; import { ProverNodePublisher } from './prover-node-publisher.js'; export class ProverPublisherFactory { constructor( - private config: TxSenderConfig & PublisherConfig, + private config: ProverTxSenderConfig & ProverPublisherConfig, private deps: { rollupContract: RollupContract; publisherManager: PublisherManager; diff --git a/yarn-project/sequencer-client/src/client/sequencer-client.ts b/yarn-project/sequencer-client/src/client/sequencer-client.ts index 9f1d93d36067..c55521d7b233 100644 --- a/yarn-project/sequencer-client/src/client/sequencer-client.ts +++ b/yarn-project/sequencer-client/src/client/sequencer-client.ts @@ -3,7 +3,7 @@ import { EpochCache } from '@aztec/epoch-cache'; import { isAnvilTestChain } from '@aztec/ethereum/chain'; import { getPublicClient } from '@aztec/ethereum/client'; import { GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; -import { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import { type Delayer, L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { PublisherManager } from '@aztec/ethereum/publisher-manager'; import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; @@ -18,7 +18,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import { L1Metrics, type TelemetryClient } from '@aztec/telemetry-client'; import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client'; -import type { SequencerClientConfig } from '../config.js'; +import { type SequencerClientConfig, getPublisherConfigFromSequencerConfig } from '../config.js'; import { GlobalVariableBuilder } from '../global_variable_builder/index.js'; import { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js'; import { Sequencer, type SequencerConfig } from '../sequencer/index.js'; @@ -28,11 +28,12 @@ import { Sequencer, type SequencerConfig } from '../sequencer/index.js'; */ export class SequencerClient { constructor( - protected publisherManager: PublisherManager, + protected publisherManager: PublisherManager, protected sequencer: Sequencer, protected checkpointsBuilder: FullNodeCheckpointsBuilder, protected validatorClient?: ValidatorClient, private l1Metrics?: L1Metrics, + private delayer_?: Delayer, ) {} /** @@ -62,7 +63,7 @@ export class SequencerClient { blobClient: BlobClientInterface; dateProvider: DateProvider; epochCache?: EpochCache; - l1TxUtils: L1TxUtilsWithBlobs[]; + l1TxUtils: L1TxUtils[]; nodeKeyStore: KeystoreManager; }, ) { @@ -85,7 +86,11 @@ export class SequencerClient { publicClient, l1TxUtils.map(x => x.getSenderAddress()), ); - const publisherManager = new PublisherManager(l1TxUtils, config, log.getBindings()); + const publisherManager = new PublisherManager( + l1TxUtils, + getPublisherConfigFromSequencerConfig(config), + log.getBindings(), + ); const rollupContract = new RollupContract(publicClient, config.l1Contracts.rollupAddress.toString()); const [l1GenesisTime, slotDuration, rollupVersion, rollupManaLimit] = await Promise.all([ rollupContract.getL1GenesisTime(), @@ -171,9 +176,12 @@ export class SequencerClient { log, ); - await sequencer.init(); + sequencer.init(); + + // Extract the shared delayer from the first L1TxUtils instance (all instances share the same delayer) + const delayer = l1TxUtils[0]?.delayer; - return new SequencerClient(publisherManager, sequencer, checkpointsBuilder, validatorClient, l1Metrics); + return new SequencerClient(publisherManager, sequencer, checkpointsBuilder, validatorClient, l1Metrics, delayer); } /** @@ -208,6 +216,16 @@ export class SequencerClient { return this.sequencer; } + /** Updates the publisher factory's node keystore adapter after a keystore reload. */ + public updatePublisherNodeKeyStore(adapter: NodeKeystoreAdapter): void { + this.sequencer.updatePublisherNodeKeyStore(adapter); + } + + /** Returns the shared tx delayer for sequencer L1 txs, if enabled. Test-only. */ + getDelayer(): Delayer | undefined { + return this.delayer_; + } + get validatorAddresses(): EthAddress[] | undefined { return this.sequencer.getValidatorAddresses(); } diff --git a/yarn-project/sequencer-client/src/config.ts b/yarn-project/sequencer-client/src/config.ts index 60ce42919779..469651fba387 100644 --- a/yarn-project/sequencer-client/src/config.ts +++ b/yarn-project/sequencer-client/src/config.ts @@ -22,10 +22,10 @@ import { DEFAULT_P2P_PROPAGATION_TIME } from '@aztec/stdlib/timetable'; import { type ValidatorClientConfig, validatorClientConfigMappings } from '@aztec/validator-client/config'; import { - type PublisherConfig, - type TxSenderConfig, - getPublisherConfigMappings, - getTxSenderConfigMappings, + type SequencerPublisherConfig, + type SequencerTxSenderConfig, + sequencerPublisherConfigMappings, + sequencerTxSenderConfigMappings, } from './publisher/config.js'; export * from './publisher/config.js'; @@ -55,15 +55,16 @@ export const DefaultSequencerConfig: ResolvedSequencerConfig = { fishermanMode: false, shuffleAttestationOrdering: false, skipPushProposedBlocksToArchiver: false, + skipPublishingCheckpointsPercent: 0, }; /** * Configuration settings for the SequencerClient. */ -export type SequencerClientConfig = PublisherConfig & +export type SequencerClientConfig = SequencerPublisherConfig & KeyStoreConfig & ValidatorClientConfig & - TxSenderConfig & + SequencerTxSenderConfig & SequencerConfig & L1ReaderConfig & ChainConfig & @@ -205,6 +206,14 @@ export const sequencerConfigMappings: ConfigMappingsType = { description: 'Skip pushing proposed blocks to archiver (default: true)', ...booleanConfigHelper(DefaultSequencerConfig.skipPushProposedBlocksToArchiver), }, + minBlocksForCheckpoint: { + description: 'Minimum number of blocks required for a checkpoint proposal (test only)', + }, + skipPublishingCheckpointsPercent: { + env: 'SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT', + description: 'Percent probability (0 - 100) of sequencer skipping checkpoint publishing (testing only)', + ...numberConfigHelper(DefaultSequencerConfig.skipPublishingCheckpointsPercent), + }, ...pickConfigMappings(p2pConfigMappings, ['txPublicSetupAllowList']), }; @@ -213,8 +222,8 @@ export const sequencerClientConfigMappings: ConfigMappingsType { + ): Promise { const { chainId, version } = this; const timestamp = getTimestampForSlot(slotNumber, { diff --git a/yarn-project/sequencer-client/src/publisher/config.ts b/yarn-project/sequencer-client/src/publisher/config.ts index b663971ebe06..50d4cd8ff16b 100644 --- a/yarn-project/sequencer-client/src/publisher/config.ts +++ b/yarn-project/sequencer-client/src/publisher/config.ts @@ -1,32 +1,45 @@ import { type BlobClientConfig, blobClientConfigMapping } from '@aztec/blob-client/client/config'; import { type L1ReaderConfig, l1ReaderConfigMappings } from '@aztec/ethereum/l1-reader'; import { type L1TxUtilsConfig, l1TxUtilsConfigMappings } from '@aztec/ethereum/l1-tx-utils/config'; -import { - type ConfigMappingsType, - SecretValue, - booleanConfigHelper, - getConfigFromMappings, -} from '@aztec/foundation/config'; +import { type ConfigMappingsType, SecretValue, booleanConfigHelper } from '@aztec/foundation/config'; import { EthAddress } from '@aztec/foundation/eth-address'; -/** - * The configuration of the rollup transaction publisher. - */ +/** Configuration of the transaction publisher. */ export type TxSenderConfig = L1ReaderConfig & { - /** - * The private key to be used by the publisher. - */ + /** The private key to be used by the publisher. */ publisherPrivateKeys?: SecretValue<`0x${string}`>[]; - /** - * Publisher addresses to be used with a remote signer - */ + /** Publisher addresses to be used with a remote signer */ publisherAddresses?: EthAddress[]; }; -/** - * Configuration of the L1Publisher. - */ +export type ProverTxSenderConfig = L1ReaderConfig & { + proverPublisherPrivateKeys?: SecretValue<`0x${string}`>[]; + proverPublisherAddresses?: EthAddress[]; +}; + +export type SequencerTxSenderConfig = L1ReaderConfig & { + sequencerPublisherPrivateKeys?: SecretValue<`0x${string}`>[]; + sequencerPublisherAddresses?: EthAddress[]; +}; + +export function getTxSenderConfigFromProverConfig(config: ProverTxSenderConfig): TxSenderConfig { + return { + ...config, + publisherPrivateKeys: config.proverPublisherPrivateKeys, + publisherAddresses: config.proverPublisherAddresses, + }; +} + +export function getTxSenderConfigFromSequencerConfig(config: SequencerTxSenderConfig): TxSenderConfig { + return { + ...config, + publisherPrivateKeys: config.sequencerPublisherPrivateKeys, + publisherAddresses: config.sequencerPublisherAddresses, + }; +} + +/** Configuration of the L1Publisher. */ export type PublisherConfig = L1TxUtilsConfig & BlobClientConfig & { /** True to use publishers in invalid states (timed out, cancelled, etc) if no other is available */ @@ -37,35 +50,76 @@ export type PublisherConfig = L1TxUtilsConfig & publisherForwarderAddress?: EthAddress; }; -export const getTxSenderConfigMappings: ( - scope: 'PROVER' | 'SEQ', -) => ConfigMappingsType> = (scope: 'PROVER' | 'SEQ') => ({ +export type ProverPublisherConfig = L1TxUtilsConfig & + BlobClientConfig & { + fishermanMode?: boolean; + proverPublisherAllowInvalidStates?: boolean; + proverPublisherForwarderAddress?: EthAddress; + }; + +export type SequencerPublisherConfig = L1TxUtilsConfig & + BlobClientConfig & { + fishermanMode?: boolean; + sequencerPublisherAllowInvalidStates?: boolean; + sequencerPublisherForwarderAddress?: EthAddress; + }; + +export function getPublisherConfigFromProverConfig(config: ProverPublisherConfig): PublisherConfig { + return { + ...config, + publisherAllowInvalidStates: config.proverPublisherAllowInvalidStates, + publisherForwarderAddress: config.proverPublisherForwarderAddress, + }; +} + +export function getPublisherConfigFromSequencerConfig(config: SequencerPublisherConfig): PublisherConfig { + return { + ...config, + publisherAllowInvalidStates: config.sequencerPublisherAllowInvalidStates, + publisherForwarderAddress: config.sequencerPublisherForwarderAddress, + }; +} + +export const proverTxSenderConfigMappings: ConfigMappingsType> = { ...l1ReaderConfigMappings, - publisherPrivateKeys: { - env: scope === 'PROVER' ? `PROVER_PUBLISHER_PRIVATE_KEYS` : `SEQ_PUBLISHER_PRIVATE_KEYS`, - description: 'The private keys to be used by the publisher.', + proverPublisherPrivateKeys: { + env: `PROVER_PUBLISHER_PRIVATE_KEYS`, + description: 'The private keys to be used by the prover publisher.', parseEnv: (val: string) => val.split(',').map(key => new SecretValue(`0x${key.replace('0x', '')}`)), defaultValue: [], - fallback: [scope === 'PROVER' ? `PROVER_PUBLISHER_PRIVATE_KEY` : `SEQ_PUBLISHER_PRIVATE_KEY`], + fallback: [`PROVER_PUBLISHER_PRIVATE_KEY`], }, - publisherAddresses: { - env: scope === 'PROVER' ? `PROVER_PUBLISHER_ADDRESSES` : `SEQ_PUBLISHER_ADDRESSES`, + proverPublisherAddresses: { + env: `PROVER_PUBLISHER_ADDRESSES`, description: 'The addresses of the publishers to use with remote signers', parseEnv: (val: string) => val.split(',').map(address => EthAddress.fromString(address)), defaultValue: [], }, -}); +}; -export function getTxSenderConfigFromEnv(scope: 'PROVER' | 'SEQ'): Omit { - return getConfigFromMappings(getTxSenderConfigMappings(scope)); -} +export const sequencerTxSenderConfigMappings: ConfigMappingsType> = { + ...l1ReaderConfigMappings, + sequencerPublisherPrivateKeys: { + env: `SEQ_PUBLISHER_PRIVATE_KEYS`, + description: 'The private keys to be used by the sequencer publisher.', + parseEnv: (val: string) => val.split(',').map(key => new SecretValue(`0x${key.replace('0x', '')}`)), + defaultValue: [], + fallback: [`SEQ_PUBLISHER_PRIVATE_KEY`], + }, + sequencerPublisherAddresses: { + env: `SEQ_PUBLISHER_ADDRESSES`, + description: 'The addresses of the publishers to use with remote signers', + parseEnv: (val: string) => val.split(',').map(address => EthAddress.fromString(address)), + defaultValue: [], + }, +}; -export const getPublisherConfigMappings: ( - scope: 'PROVER' | 'SEQ', -) => ConfigMappingsType = scope => ({ - publisherAllowInvalidStates: { +export const sequencerPublisherConfigMappings: ConfigMappingsType = { + ...l1TxUtilsConfigMappings, + ...blobClientConfigMapping, + sequencerPublisherAllowInvalidStates: { + env: `SEQ_PUBLISHER_ALLOW_INVALID_STATES`, description: 'True to use publishers in invalid states (timed out, cancelled, etc) if no other is available', - env: scope === `PROVER` ? `PROVER_PUBLISHER_ALLOW_INVALID_STATES` : `SEQ_PUBLISHER_ALLOW_INVALID_STATES`, ...booleanConfigHelper(true), }, fishermanMode: { @@ -74,15 +128,30 @@ export const getPublisherConfigMappings: ( 'Whether to run in fisherman mode: builds blocks on every slot for validation without publishing to L1', ...booleanConfigHelper(false), }, - publisherForwarderAddress: { - env: scope === `PROVER` ? `PROVER_PUBLISHER_FORWARDER_ADDRESS` : `SEQ_PUBLISHER_FORWARDER_ADDRESS`, + sequencerPublisherForwarderAddress: { + env: `SEQ_PUBLISHER_FORWARDER_ADDRESS`, description: 'Address of the forwarder contract to wrap all L1 transactions through (for testing purposes only)', parseEnv: (val: string) => (val ? EthAddress.fromString(val) : undefined), }, +}; + +export const proverPublisherConfigMappings: ConfigMappingsType = { ...l1TxUtilsConfigMappings, ...blobClientConfigMapping, -}); - -export function getPublisherConfigFromEnv(scope: 'PROVER' | 'SEQ'): PublisherConfig { - return getConfigFromMappings(getPublisherConfigMappings(scope)); -} + proverPublisherAllowInvalidStates: { + env: `PROVER_PUBLISHER_ALLOW_INVALID_STATES`, + description: 'True to use publishers in invalid states (timed out, cancelled, etc) if no other is available', + ...booleanConfigHelper(true), + }, + fishermanMode: { + env: 'FISHERMAN_MODE', + description: + 'Whether to run in fisherman mode: builds blocks on every slot for validation without publishing to L1', + ...booleanConfigHelper(false), + }, + proverPublisherForwarderAddress: { + env: `PROVER_PUBLISHER_FORWARDER_ADDRESS`, + description: 'Address of the forwarder contract to wrap all L1 transactions through (for testing purposes only)', + parseEnv: (val: string) => (val ? EthAddress.fromString(val) : undefined), + }, +}; diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.test.ts index 0f960c656d32..a06a0b62e2e7 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.test.ts @@ -1,7 +1,7 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; import type { EpochCache } from '@aztec/epoch-cache'; import type { GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { PublisherManager } from '@aztec/ethereum/publisher-manager'; import { EthAddress } from '@aztec/foundation/eth-address'; import type { DateProvider } from '@aztec/foundation/timer'; @@ -17,7 +17,7 @@ import { SequencerPublisherFactory } from './sequencer-publisher-factory.js'; describe('SequencerPublisherFactory', () => { let factory: SequencerPublisherFactory; let mockConfig: SequencerClientConfig; - let mockPublisherManager: MockProxy>; + let mockPublisherManager: MockProxy>; let mockBlobClient: MockProxy; let mockDateProvider: MockProxy; let mockEpochCache: MockProxy; @@ -25,7 +25,7 @@ describe('SequencerPublisherFactory', () => { let mockGovernanceProposerContract: MockProxy; let mockSlashFactoryContract: MockProxy; let mockNodeKeyStore: MockProxy; - let mockL1TxUtils: MockProxy; + let mockL1TxUtils: MockProxy; const validatorAddress = EthAddress.random(); const publisherAddress = EthAddress.random(); @@ -35,12 +35,12 @@ describe('SequencerPublisherFactory', () => { mockConfig = { ethereumSlotDuration: 12, } as SequencerClientConfig; - mockPublisherManager = mock>(); + mockPublisherManager = mock>(); mockBlobClient = mock(); mockDateProvider = mock(); mockEpochCache = mock(); mockNodeKeyStore = mock(); - mockL1TxUtils = mock(); + mockL1TxUtils = mock(); mockRollupContract = mock(); mockGovernanceProposerContract = mock(); mockSlashFactoryContract = mock(); @@ -140,6 +140,61 @@ describe('SequencerPublisherFactory', () => { expect(result.attestorAddress).toBe(validatorAddress); }); + it('should reject validator added via updateNodeKeyStore with a different publisher key', async () => { + // Initial keystore knows validator → publisherAddress + mockNodeKeyStore.getPublisherAddresses.mockReturnValue([publisherAddress]); + + // After updateNodeKeyStore, a new validator maps to a DIFFERENT publisher key + const newValidatorAddress = EthAddress.random(); + const differentPublisherAddress = EthAddress.random(); + const updatedKeyStore = mock(); + updatedKeyStore.getPublisherAddresses.mockImplementation((addr: EthAddress) => { + if (addr.equals(newValidatorAddress)) { + return [differentPublisherAddress]; // not in L1TxUtils pool + } + return [publisherAddress]; + }); + + factory.updateNodeKeyStore(updatedKeyStore); + + // The L1TxUtils pool only has publisherAddress, not differentPublisherAddress + mockL1TxUtils.getSenderAddress.mockReturnValue(publisherAddress); + mockPublisherManager.getAvailablePublisher.mockRejectedValueOnce( + new Error('Failed to find an available publisher.'), + ); + + await expect(factory.create(newValidatorAddress)).rejects.toThrow('Failed to find an available publisher.'); + + // Verify the filter rejects the available publisher (wrong key) + const filterFn = mockPublisherManager.getAvailablePublisher.mock.calls[0][0]!; + expect(filterFn(mockL1TxUtils)).toBe(false); + }); + + it('should allow validator added via updateNodeKeyStore with an existing publisher key', async () => { + // A new validator maps to the SAME publisher key that's already in the L1TxUtils pool + const newValidatorAddress = EthAddress.random(); + const updatedKeyStore = mock(); + updatedKeyStore.getPublisherAddresses.mockImplementation((addr: EthAddress) => { + if (addr.equals(newValidatorAddress)) { + return [publisherAddress]; // same key as L1TxUtils + } + return []; + }); + + factory.updateNodeKeyStore(updatedKeyStore); + + mockL1TxUtils.getSenderAddress.mockReturnValue(publisherAddress); + + const result = await factory.create(newValidatorAddress); + + // Verify the filter accepts the publisher (same key) + const filterFn = mockPublisherManager.getAvailablePublisher.mock.calls[0][0]!; + expect(filterFn(mockL1TxUtils)).toBe(true); + + expect(result.attestorAddress).toBe(newValidatorAddress); + expect(result.publisher).toBeDefined(); + }); + it('should create SequencerPublisher with correct configuration', async () => { mockNodeKeyStore.getAttestorForPublisher.mockReturnValue(attestorAddress); const mockSlashingProposer = { address: EthAddress.random() }; diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts index 2942b7ec3be1..c58bc4d40afd 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts @@ -3,7 +3,7 @@ import { type Logger, createLogger } from '@aztec/aztec.js/log'; import type { BlobClientInterface } from '@aztec/blob-client/client'; import type { EpochCache } from '@aztec/epoch-cache'; import type { GovernanceProposerContract, RollupContract } from '@aztec/ethereum/contracts'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { PublisherFilter, PublisherManager } from '@aztec/ethereum/publisher-manager'; import { SlotNumber } from '@aztec/foundation/branded-types'; import type { DateProvider } from '@aztec/foundation/timer'; @@ -26,13 +26,15 @@ export class SequencerPublisherFactory { /** Stores the last slot in which every action was carried out by a publisher */ private lastActions: Partial> = {}; + private nodeKeyStore: NodeKeystoreAdapter; + private logger: Logger; constructor( private sequencerConfig: SequencerClientConfig, private deps: { telemetry: TelemetryClient; - publisherManager: PublisherManager; + publisherManager: PublisherManager; blobClient: BlobClientInterface; dateProvider: DateProvider; epochCache: EpochCache; @@ -45,7 +47,17 @@ export class SequencerPublisherFactory { ) { this.publisherMetrics = new SequencerPublisherMetrics(deps.telemetry, 'SequencerPublisher'); this.logger = deps.logger ?? createLogger('sequencer'); + this.nodeKeyStore = this.deps.nodeKeyStore; + } + + /** + * Updates the node keystore adapter used for publisher lookups. + * Called when the keystore is reloaded at runtime to reflect new validator-publisher mappings. + */ + public updateNodeKeyStore(adapter: NodeKeystoreAdapter): void { + this.nodeKeyStore = adapter; } + /** * Creates a new SequencerPublisher instance. * @param _validatorAddress - The address of the validator that will be using the publisher. @@ -54,17 +66,17 @@ export class SequencerPublisherFactory { public async create(validatorAddress?: EthAddress): Promise { // If we have been given an attestor address we must only allow publishers permitted for that attestor - const allowedPublishers = !validatorAddress ? [] : this.deps.nodeKeyStore.getPublisherAddresses(validatorAddress); - const filter: PublisherFilter = !validatorAddress + const allowedPublishers = !validatorAddress ? [] : this.nodeKeyStore.getPublisherAddresses(validatorAddress); + const filter: PublisherFilter = !validatorAddress ? () => true - : (utils: L1TxUtilsWithBlobs) => { + : (utils: L1TxUtils) => { const publisherAddress = utils.getSenderAddress(); return allowedPublishers.some(allowedPublisher => allowedPublisher.equals(publisherAddress)); }; const l1Publisher = await this.deps.publisherManager.getAvailablePublisher(filter); const attestorAddress = - validatorAddress ?? this.deps.nodeKeyStore.getAttestorForPublisher(l1Publisher.getSenderAddress()); + validatorAddress ?? this.nodeKeyStore.getAttestorForPublisher(l1Publisher.getSenderAddress()); const rollup = this.deps.rollupContract; const slashingProposerContract = await rollup.getSlashingProposer(); @@ -89,4 +101,9 @@ export class SequencerPublisherFactory { publisher, }; } + + /** Interrupts all publishers managed by this factory. Used during sequencer shutdown. */ + public interruptAll(): void { + this.deps.publisherManager.interrupt(); + } } diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts index 75796e050874..d944412263d5 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts @@ -1,15 +1,19 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; import { getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib'; import type { EpochCache } from '@aztec/epoch-cache'; -import { DefaultL1ContractsConfig, type L1ContractsConfig } from '@aztec/ethereum/config'; +import type { L1ContractsConfig } from '@aztec/ethereum/config'; import { type EmpireSlashingProposerContract, type GovernanceProposerContract, Multicall3, type RollupContract, } from '@aztec/ethereum/contracts'; -import { type GasPrice, type L1TxUtilsConfig, defaultL1TxUtilsConfig } from '@aztec/ethereum/l1-tx-utils'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import { + type GasPrice, + type L1TxUtils, + type L1TxUtilsConfig, + defaultL1TxUtilsConfig, +} from '@aztec/ethereum/l1-tx-utils'; import { FormattedViemError } from '@aztec/ethereum/utils'; import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -58,7 +62,7 @@ describe('SequencerPublisher', () => { let slashingProposerContract: MockProxy; let governanceProposerContract: MockProxy; let slashFactoryContract: MockProxy; - let l1TxUtils: MockProxy; + let l1TxUtils: MockProxy; let l1Metrics: MockProxy; let forwardSpy: jest.SpiedFunction; @@ -101,7 +105,7 @@ describe('SequencerPublisher', () => { testHarnessAttesterAccount = privateKeyToAccount( '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', ); - l1TxUtils = mock(); + l1TxUtils = mock(); l1TxUtils.getBlock.mockResolvedValue({ timestamp: 12n } as any); l1TxUtils.getBlockNumber.mockResolvedValue(1n); l1TxUtils.getSenderAddress.mockReturnValue(EthAddress.fromString(testHarnessAttesterAccount.address)); @@ -113,7 +117,6 @@ describe('SequencerPublisher', () => { rollupAddress: EthAddress.ZERO.toString(), governanceProposerAddress: mockGovernanceProposerAddress, }, - ethereumSlotDuration: DefaultL1ContractsConfig.ethereumSlotDuration, ...defaultL1TxUtilsConfig, } as unknown as TxSenderConfig & @@ -123,6 +126,7 @@ describe('SequencerPublisher', () => { rollup = mock(); rollup.validateHeader.mockReturnValue(Promise.resolve()); + rollup.getL1StartBlock.mockResolvedValue(1n); (rollup as any).address = mockRollupAddress; forwardSpy = jest.spyOn(Multicall3, 'forward'); @@ -203,7 +207,7 @@ describe('SequencerPublisher', () => { it('bundles propose and vote tx to l1', async () => { const checkpoint = new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber); - const expectedBlobs = getBlobsPerL1Block(checkpoint.toBlobFields()); + const expectedBlobs = await getBlobsPerL1Block(checkpoint.toBlobFields()); await publisher.enqueueProposeCheckpoint(checkpoint, CommitteeAttestationsAndSigners.empty(), Signature.empty()); const { govPayload, voteSig } = mockGovernancePayload(); @@ -415,4 +419,119 @@ describe('SequencerPublisher', () => { ), ).toEqual(false); }); + + it('stops signalling when payload was previously proposed', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed.mockResolvedValue(true); + + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(false); + }); + + it('continues signalling when payload was NOT proposed', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed.mockResolvedValue(false); + + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(true); + }); + + it('caches proposed result and prevents repeated L1 calls', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed.mockResolvedValue(true); + + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ); + + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(3), + 2n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ); + + expect(governanceProposerContract.hasPayloadBeenProposed).toHaveBeenCalledTimes(1); + }); + + it('retries on transient RPC failure and succeeds', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed + .mockRejectedValueOnce(new Error('RPC error')) + .mockRejectedValueOnce(new Error('RPC error')) + .mockResolvedValueOnce(false); + + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(true); + }); + + it('fails closed on persistent RPC failure', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed.mockRejectedValue(new Error('RPC error')); + + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(false); + }); + + it('does not cache false result and re-checks on subsequent calls', async () => { + const { govPayload } = mockGovernancePayload(); + governanceProposerContract.hasPayloadBeenProposed.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + + // First call: not proposed, signalling proceeds + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(2), + 1n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(true); + + // Second call: now proposed, signalling stops + expect( + await publisher.enqueueGovernanceCastSignal( + govPayload, + SlotNumber(3), + 2n, + EthAddress.fromString(testHarnessAttesterAccount.address), + msg => testHarnessAttesterAccount.signTypedData(msg), + ), + ).toEqual(false); + + expect(governanceProposerContract.hasPayloadBeenProposed).toHaveBeenCalledTimes(2); + }); }); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index cdf317576111..bbc0335d29c8 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -19,11 +19,11 @@ import { type L1BlobInputs, type L1TxConfig, type L1TxRequest, + type L1TxUtils, MAX_L1_TX_LIMIT, type TransactionStats, WEI_CONST, } from '@aztec/ethereum/l1-tx-utils'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils'; import { sumBigint } from '@aztec/foundation/bigint'; import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer'; @@ -33,6 +33,7 @@ import type { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Signature, type ViemSignature } from '@aztec/foundation/eth-signature'; import { type Logger, createLogger } from '@aztec/foundation/log'; +import { makeBackoff, retry } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; import { DateProvider, Timer } from '@aztec/foundation/timer'; import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts'; @@ -46,7 +47,7 @@ import { type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from import { type StateOverride, type TransactionReceipt, type TypedDataDefinition, encodeFunctionData, toHex } from 'viem'; -import type { PublisherConfig, TxSenderConfig } from './config.js'; +import type { SequencerPublisherConfig } from './config.js'; import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js'; /** Arguments to the process method of the rollup contract */ @@ -115,6 +116,7 @@ export class SequencerPublisher { protected lastActions: Partial> = {}; private isPayloadEmptyCache: Map = new Map(); + private payloadProposedCache: Set = new Set(); protected log: Logger; protected ethereumSlotDuration: bigint; @@ -136,7 +138,7 @@ export class SequencerPublisher { // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet public static VOTE_GAS_GUESS: bigint = 800_000n; - public l1TxUtils: L1TxUtilsWithBlobs; + public l1TxUtils: L1TxUtils; public rollupContract: RollupContract; public govProposerContract: GovernanceProposerContract; public slashingProposerContract: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined; @@ -147,11 +149,12 @@ export class SequencerPublisher { protected requests: RequestWithExpiry[] = []; constructor( - private config: TxSenderConfig & PublisherConfig & Pick, + private config: Pick & + Pick & { l1ChainId: number }, deps: { telemetry?: TelemetryClient; blobClient: BlobClientInterface; - l1TxUtils: L1TxUtilsWithBlobs; + l1TxUtils: L1TxUtils; rollupContract: RollupContract; slashingProposerContract: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined; governanceProposerContract: GovernanceProposerContract; @@ -639,24 +642,8 @@ export class SequencerPublisher { options: { forcePendingCheckpointNumber?: CheckpointNumber }, ): Promise { const ts = BigInt((await this.l1TxUtils.getBlock()).timestamp + this.ethereumSlotDuration); - - // TODO(palla/mbps): This should not be needed, there's no flow where we propose with zero attestations. Or is there? - // If we have no attestations, we still need to provide the empty attestations - // so that the committee is recalculated correctly - // const ignoreSignatures = attestationsAndSigners.attestations.length === 0; - // if (ignoreSignatures) { - // const { committee } = await this.epochCache.getCommittee(block.header.globalVariables.slotNumber); - // if (!committee) { - // this.log.warn(`No committee found for slot ${block.header.globalVariables.slotNumber}`); - // throw new Error(`No committee found for slot ${block.header.globalVariables.slotNumber}`); - // } - // attestationsAndSigners.attestations = committee.map(committeeMember => - // CommitteeAttestation.fromAddress(committeeMember), - // ); - // } - const blobFields = checkpoint.toBlobFields(); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); const blobInput = getPrefixedEthBlobCommitments(blobs); const args = [ @@ -713,6 +700,32 @@ export class SequencerPublisher { return false; } + // Check if payload was already submitted to governance + const cacheKey = payload.toString(); + if (!this.payloadProposedCache.has(cacheKey)) { + try { + const l1StartBlock = await this.rollupContract.getL1StartBlock(); + const proposed = await retry( + () => base.hasPayloadBeenProposed(payload.toString(), l1StartBlock), + 'Check if payload was proposed', + makeBackoff([0, 1, 2]), + this.log, + true, + ); + if (proposed) { + this.payloadProposedCache.add(cacheKey); + } + } catch (err) { + this.log.warn(`Failed to check if payload ${payload} was proposed after retries, skipping signal`, err); + return false; + } + } + + if (this.payloadProposedCache.has(cacheKey)) { + this.log.info(`Payload ${payload} was already proposed to governance, stopping signals`); + return false; + } + const cachedLastVote = this.lastActions[signalType]; this.lastActions[signalType] = slotNumber; const action = signalType; @@ -940,7 +953,7 @@ export class SequencerPublisher { const checkpointHeader = checkpoint.header; const blobFields = checkpoint.toBlobFields(); - const blobs = getBlobsPerL1Block(blobFields); + const blobs = await getBlobsPerL1Block(blobFields); const proposeTxArgs: L1ProcessArgs = { header: checkpointHeader, diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index f71618d90b37..8ef5a19129ba 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -24,7 +24,7 @@ import { type P2P, P2PClientState } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { CommitteeAttestation, L2Block, type L2BlockSink, type L2BlockSource } from '@aztec/stdlib/block'; -import { Checkpoint } from '@aztec/stdlib/checkpoint'; +import { Checkpoint, type CheckpointData, L1PublishedData } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { GasFees } from '@aztec/stdlib/gas'; import { @@ -196,7 +196,9 @@ describe('CheckpointProposalJob', () => { p2p.broadcastProposal.mockResolvedValue(undefined); worldState = mockDeep(); - const mockFork = mock({ [Symbol.dispose]: jest.fn() }); + const mockFork = mock({ + [Symbol.asyncDispose]: jest.fn().mockReturnValue(Promise.resolve()) as () => Promise, + }); worldState.fork.mockResolvedValue(mockFork); // Create fake CheckpointsBuilder and CheckpointBuilder @@ -216,7 +218,7 @@ describe('CheckpointProposalJob', () => { l1ToL2MessageSource.getL1ToL2Messages.mockResolvedValue(Array(4).fill(Fr.ZERO)); l2BlockSource = mock(); - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([]); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); @@ -369,6 +371,7 @@ describe('CheckpointProposalJob', () => { it('passes previous checkpoint out hashes when there are earlier checkpoints in the epoch', async () => { // Create two previous checkpoints in the same epoch const previousCheckpoints = await timesAsync(2, i => Checkpoint.random(CheckpointNumber(i + 1))); + const previousCheckpointsData: CheckpointData[] = previousCheckpoints.map(c => toCheckpointData(c)); // Update job to be for checkpoint 3 checkpointNumber = CheckpointNumber(3); @@ -383,7 +386,7 @@ describe('CheckpointProposalJob', () => { ); // Mock l2BlockSource to return the previous checkpoints - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue(previousCheckpoints); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue(previousCheckpointsData); // Build block successfully const { txs, block } = await setupTxsAndBlock(p2p, globalVariables, 1, chainId); @@ -419,8 +422,12 @@ describe('CheckpointProposalJob', () => { }), ); - // Mock l2BlockSource to return all three checkpoints - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([previousCheckpoint, currentCheckpoint, futureCheckpoint]); + // Mock l2BlockSource to return all three checkpoints as data + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([ + toCheckpointData(previousCheckpoint), + toCheckpointData(currentCheckpoint), + toCheckpointData(futureCheckpoint), + ]); // Build block successfully const { txs, block } = await setupTxsAndBlock(p2p, globalVariables, 1, chainId); @@ -1114,3 +1121,17 @@ class TestCheckpointProposalJob extends CheckpointProposalJob { return super.buildSingleBlock(checkpointBuilder, opts); } } + +/** Creates a CheckpointData from a Checkpoint for testing. */ +function toCheckpointData(checkpoint: Checkpoint): CheckpointData { + return { + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: checkpoint.getCheckpointOutHash(), + startBlock: BlockNumber(checkpoint.blocks[0]?.number ?? 1), + blockCount: checkpoint.blocks.length, + attestations: [], + l1: L1PublishedData.random(), + }; +} diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index b25291dfc60d..2e9ebb18219e 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -62,7 +62,7 @@ class TimingAwareMockCheckpointBuilder extends MockCheckpointBuilder { public recordedBuildTimes: Array<{ blockNumber: number; startTime: number; endTime: number }> = []; constructor( - constants: CheckpointGlobalVariables & { timestamp: bigint }, + constants: CheckpointGlobalVariables, checkpointNumber: CheckpointNumber, private readonly dateProvider: ManualDateProvider, private readonly getSecondsIntoSlot: () => number, @@ -368,7 +368,7 @@ describe('CheckpointProposalJob Timing Tests', () => { ); // Create timing-aware checkpoint builder - const checkpointConstants: CheckpointGlobalVariables & { timestamp: bigint } = { ...globalVariables }; + const checkpointConstants: CheckpointGlobalVariables = { ...globalVariables }; checkpointBuilder = new TimingAwareMockCheckpointBuilder( checkpointConstants, checkpointNumber, @@ -415,14 +415,16 @@ describe('CheckpointProposalJob Timing Tests', () => { p2p.getPendingTxCount.mockResolvedValue(100); // Always have enough txs worldState = mockDeep(); - const mockFork = mock({ [Symbol.dispose]: jest.fn() }); + const mockFork = mock({ + [Symbol.asyncDispose]: jest.fn().mockReturnValue(Promise.resolve()) as () => Promise, + }); worldState.fork.mockResolvedValue(mockFork); l1ToL2MessageSource = mock(); l1ToL2MessageSource.getL1ToL2Messages.mockResolvedValue(Array(4).fill(Fr.ZERO)); l2BlockSource = mock(); - l2BlockSource.getCheckpointsForEpoch.mockResolvedValue([]); + l2BlockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index e7b84bb9cddc..44abe045ba91 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -129,7 +129,7 @@ export class CheckpointProposalJob implements Traceable { await Promise.all(votesPromises); if (checkpoint) { - this.metrics.recordBlockProposalSuccess(); + this.metrics.recordCheckpointProposalSuccess(); } // Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis @@ -186,16 +186,15 @@ export class CheckpointProposalJob implements Traceable { const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages); // Collect the out hashes of all the checkpoints before this one in the same epoch - const previousCheckpoints = (await this.l2BlockSource.getCheckpointsForEpoch(this.epoch)).filter( - c => c.number < this.checkpointNumber, - ); - const previousCheckpointOutHashes = previousCheckpoints.map(c => c.getCheckpointOutHash()); + const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.epoch)) + .filter(c => c.checkpointNumber < this.checkpointNumber) + .map(c => c.checkpointOutHash); // Get the fee asset price modifier from the oracle const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(); // Create a long-lived forked world state for the checkpoint builder - using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); + await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); // Create checkpoint builder for the entire slot const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint( @@ -221,6 +220,7 @@ export class CheckpointProposalJob implements Traceable { let blocksInCheckpoint: L2Block[] = []; let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined; + const checkpointBuildTimer = new Timer(); try { // Main loop: build blocks for the checkpoint @@ -248,11 +248,28 @@ export class CheckpointProposalJob implements Traceable { return undefined; } + const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint; + if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) { + this.log.warn( + `Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`, + { slot: this.slot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint }, + ); + return undefined; + } + // Assemble and broadcast the checkpoint proposal, including the last block that was not // broadcasted yet, and wait to collect the committee attestations. this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.slot); const checkpoint = await checkpointBuilder.completeCheckpoint(); + // Record checkpoint-level build metrics + this.metrics.recordCheckpointBuild( + checkpointBuildTimer.ms(), + blocksInCheckpoint.length, + checkpoint.getStats().txCount, + Number(checkpoint.header.totalManaUsed.toBigInt()), + ); + // Do not collect attestations nor publish to L1 in fisherman mode if (this.config.fishermanMode) { this.log.info( @@ -318,6 +335,21 @@ export class CheckpointProposalJob implements Traceable { const aztecSlotDuration = this.l1Constants.slotDuration; const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp(); const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000); + + // If we have been configured to potentially skip publishing checkpoint then roll the dice here + if ( + this.config.skipPublishingCheckpointsPercent !== undefined && + this.config.skipPublishingCheckpointsPercent > 0 + ) { + const result = Math.max(0, randomInt(100)); + if (result < this.config.skipPublishingCheckpointsPercent) { + this.log.warn( + `Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`, + ); + return checkpoint; + } + } + await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, { txTimeoutAt, forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber, @@ -826,7 +858,7 @@ export class CheckpointProposalJob implements Traceable { slot: this.slot, feeAnalysisId: feeAnalysis?.id, }); - this.metrics.recordBlockProposalFailed('block_build_failed'); + this.metrics.recordCheckpointProposalFailed('block_build_failed'); } this.publisher.clearPendingRequests(); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts index 91e778c05105..65ed41a5ae48 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts @@ -14,7 +14,7 @@ import type { RollupContract, } from '@aztec/ethereum/contracts'; import { Multicall3 } from '@aztec/ethereum/contracts'; -import type { L1TxUtilsWithBlobs } from '@aztec/ethereum/l1-tx-utils-with-blobs'; +import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { SecretValue } from '@aztec/foundation/config'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -53,7 +53,7 @@ describe('CheckpointVoter HA Integration', () => { let rollupContract: MockProxy; let governanceProposerContract: MockProxy; let slashingProposerContract: MockProxy; - let l1TxUtils: MockProxy; + let l1TxUtils: MockProxy; let dateProvider: TestDateProvider; let sequencerMetrics: MockProxy; let publisherMetrics: MockProxy; @@ -147,8 +147,8 @@ describe('CheckpointVoter HA Integration', () => { /** * Helper to create mock L1 tx utils */ - function createMockL1TxUtils(validatorAccount: PrivateKeyAccount): MockProxy { - const txUtils = mock(); + function createMockL1TxUtils(validatorAccount: PrivateKeyAccount): MockProxy { + const txUtils = mock(); txUtils.client = { account: validatorAccount, getCode: () => Promise.resolve('0x1234' as `0x${string}`), diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 788f764e70bd..58655d3fbd74 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -18,7 +18,6 @@ import { type Hex, formatUnits } from 'viem'; import type { SequencerState } from './utils.js'; -// TODO(palla/mbps): Review all metrics and add any missing ones per checkpoint export class SequencerMetrics { public readonly tracer: Tracer; private meter: Meter; @@ -40,11 +39,16 @@ export class SequencerMetrics { private filledSlots: UpDownCounter; private blockProposalFailed: UpDownCounter; - private blockProposalSuccess: UpDownCounter; - private blockProposalPrecheckFailed: UpDownCounter; + private checkpointProposalSuccess: UpDownCounter; + private checkpointPrecheckFailed: UpDownCounter; + private checkpointProposalFailed: UpDownCounter; private checkpointSuccess: UpDownCounter; private slashingAttempts: UpDownCounter; private checkpointAttestationDelay: Histogram; + private checkpointBuildDuration: Histogram; + private checkpointBlockCount: Gauge; + private checkpointTxCount: Gauge; + private checkpointTotalMana: Gauge; // Fisherman fee analysis metrics private fishermanWouldBeIncluded: UpDownCounter; @@ -84,7 +88,7 @@ export class SequencerMetrics { this.checkpointAttestationDelay = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_ATTESTATION_DELAY); - this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_BLOCK_REWARDS); + this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS); this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT); @@ -107,16 +111,16 @@ export class SequencerMetrics { Metrics.SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT, ); - this.blockProposalSuccess = createUpDownCounterWithDefault( + this.checkpointProposalSuccess = createUpDownCounterWithDefault( this.meter, - Metrics.SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT, + Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT, ); this.checkpointSuccess = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_CHECKPOINT_SUCCESS_COUNT); - this.blockProposalPrecheckFailed = createUpDownCounterWithDefault( + this.checkpointPrecheckFailed = createUpDownCounterWithDefault( this.meter, - Metrics.SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT, + Metrics.SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT, { [Attributes.ERROR_TYPE]: [ 'slot_already_taken', @@ -127,6 +131,16 @@ export class SequencerMetrics { }, ); + this.checkpointProposalFailed = createUpDownCounterWithDefault( + this.meter, + Metrics.SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT, + ); + + this.checkpointBuildDuration = this.meter.createHistogram(Metrics.SEQUENCER_CHECKPOINT_BUILD_DURATION); + this.checkpointBlockCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_BLOCK_COUNT); + this.checkpointTxCount = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TX_COUNT); + this.checkpointTotalMana = this.meter.createGauge(Metrics.SEQUENCER_CHECKPOINT_TOTAL_MANA); + this.slashingAttempts = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLASHING_ATTEMPTS_COUNT); // Fisherman fee analysis metrics @@ -262,18 +276,30 @@ export class SequencerMetrics { }); } - recordBlockProposalSuccess() { - this.blockProposalSuccess.add(1); + recordCheckpointProposalSuccess() { + this.checkpointProposalSuccess.add(1); } - recordBlockProposalPrecheckFailed( + recordCheckpointPrecheckFailed( checkType: 'slot_already_taken' | 'rollup_contract_check_failed' | 'slot_mismatch' | 'block_number_mismatch', ) { - this.blockProposalPrecheckFailed.add(1, { - [Attributes.ERROR_TYPE]: checkType, + this.checkpointPrecheckFailed.add(1, { [Attributes.ERROR_TYPE]: checkType }); + } + + recordCheckpointProposalFailed(reason?: string) { + this.checkpointProposalFailed.add(1, { + ...(reason && { [Attributes.ERROR_TYPE]: reason }), }); } + /** Records aggregate metrics for a completed checkpoint build. */ + recordCheckpointBuild(durationMs: number, blockCount: number, txCount: number, totalMana: number) { + this.checkpointBuildDuration.record(Math.ceil(durationMs)); + this.checkpointBlockCount.record(blockCount); + this.checkpointTxCount.record(txCount); + this.checkpointTotalMana.record(totalMana); + } + recordSlashingAttempt(actionCount: number) { this.slashingAttempts.add(actionCount); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 0aa7c921015a..cb625f07002d 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -1,7 +1,13 @@ import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants'; import type { EpochCache, EpochCommitteeInfo } from '@aztec/epoch-cache'; import type { RollupContract } from '@aztec/ethereum/contracts'; -import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { + BlockNumber, + CheckpointNumber, + EpochNumber, + IndexWithinCheckpoint, + SlotNumber, +} from '@aztec/foundation/branded-types'; import { omit, times, timesParallel } from '@aztec/foundation/collection'; import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; import { Fr } from '@aztec/foundation/curves/bn254'; @@ -12,6 +18,7 @@ import type { P2P } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { + type BlockData, CommitteeAttestation, CommitteeAttestationsAndSigners, GENESIS_CHECKPOINT_HEADER_HASH, @@ -31,7 +38,8 @@ import { type WorldStateSynchronizerStatus, } from '@aztec/stdlib/interfaces/server'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; -import { GlobalVariables, type Tx } from '@aztec/stdlib/tx'; +import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; +import { BlockHeader, GlobalVariables, type Tx } from '@aztec/stdlib/tx'; import type { FullNodeCheckpointsBuilder, ValidatorClient } from '@aztec/validator-client'; import { expect } from '@jest/globals'; @@ -235,7 +243,13 @@ describe('sequencer', () => { checkpointBuilder.setBlockProvider(() => block); l2BlockSource = mock({ - getL2Block: mockFn().mockResolvedValue(L2Block.empty()), + getBlockData: mockFn().mockResolvedValue({ + header: BlockHeader.empty(), + archive: AppendOnlyTreeSnapshot.empty(), + blockHash: Fr.ZERO, + checkpointNumber: CheckpointNumber(0), + indexWithinCheckpoint: IndexWithinCheckpoint(0), + } satisfies BlockData), getBlockNumber: mockFn().mockResolvedValue(lastBlockNumber), getL2Tips: mockFn().mockResolvedValue({ proposed: { number: lastBlockNumber, hash }, @@ -257,6 +271,7 @@ describe('sequencer', () => { getPendingChainValidationStatus: mockFn().mockResolvedValue({ valid: true }), getCheckpointedBlocksForEpoch: mockFn().mockResolvedValue([]), getCheckpointsForEpoch: mockFn().mockResolvedValue([]), + getCheckpointsDataForEpoch: mockFn().mockResolvedValue([]), }); l1ToL2MessageSource = mock({ diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index fd14e9b12e4a..071613e1c491 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -12,7 +12,7 @@ import type { DateProvider } from '@aztec/foundation/timer'; import type { TypedEventEmitter } from '@aztec/foundation/types'; import type { P2P } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; -import type { L2Block, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block'; +import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; import { getSlotAtTimestamp, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers'; import { @@ -25,7 +25,7 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import { pickFromSchema } from '@aztec/stdlib/schemas'; import { MerkleTreeId } from '@aztec/stdlib/trees'; import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client'; -import { FullNodeCheckpointsBuilder, type ValidatorClient } from '@aztec/validator-client'; +import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client'; import EventEmitter from 'node:events'; @@ -75,14 +75,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter { this.log.info(`Stopping sequencer`); this.setState(SequencerState.STOPPING, undefined, { force: true }); - this.publisher?.interrupt(); + this.publisherFactory.interruptAll(); await this.runningPromise?.stop(); this.setState(SequencerState.STOPPED, undefined, { force: true }); this.log.info('Stopped sequencer'); @@ -169,7 +160,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter= slot) { + if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= slot) { this.log.warn( `Cannot propose block at next L2 slot ${slot} since that slot was taken by block ${syncedTo.blockNumber}`, - { ...logCtx, block: syncedTo.block.header.toInspect() }, + { ...logCtx, block: syncedTo.blockData.header.toInspect() }, ); - this.metrics.recordBlockProposalPrecheckFailed('slot_already_taken'); + this.metrics.recordCheckpointPrecheckFailed('slot_already_taken'); return undefined; } @@ -326,7 +316,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter; + declare public publisherManager: PublisherManager; } export type TestSequencerClient = TestSequencerClient_; diff --git a/yarn-project/simulator/src/public/hinting_db_sources.ts b/yarn-project/simulator/src/public/hinting_db_sources.ts index 538fee0ed8df..79044c631e64 100644 --- a/yarn-project/simulator/src/public/hinting_db_sources.ts +++ b/yarn-project/simulator/src/public/hinting_db_sources.ts @@ -572,7 +572,7 @@ export class HintingMerkleWriteOperations implements MerkleTreeWriteOperations { return await this.db.close(); } - async [Symbol.dispose](): Promise { + async [Symbol.asyncDispose](): Promise { await this.close(); } diff --git a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts index 1a7f644e08f5..bcbd818a03f0 100644 --- a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts +++ b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts @@ -82,7 +82,7 @@ export class GuardedMerkleTreeOperations implements MerkleTreeWriteOperations { return this.guardAndPush(() => this.target.close()); } - async [Symbol.dispose](): Promise { + async [Symbol.asyncDispose](): Promise { await this.close(); } getTreeInfo(treeId: MerkleTreeId): Promise { diff --git a/yarn-project/slasher/src/slash_offenses_collector.ts b/yarn-project/slasher/src/slash_offenses_collector.ts index 274357a97c6a..551f868ccec3 100644 --- a/yarn-project/slasher/src/slash_offenses_collector.ts +++ b/yarn-project/slasher/src/slash_offenses_collector.ts @@ -85,11 +85,7 @@ export class SlashOffensesCollector { } } - this.log.info(`Adding pending offense for validator ${arg.validator}`, { - ...pendingOffense, - epochOrSlot: pendingOffense.epochOrSlot.toString(), - amount: pendingOffense.amount.toString(), - }); + this.log.info(`Adding pending offense for validator ${arg.validator}`, pendingOffense); await this.offensesStore.addPendingOffense(pendingOffense); } } diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts index 24355f8545d8..dd52040bdef7 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts @@ -253,6 +253,7 @@ describe('EpochPruneWatcher', () => { class MockL2BlockSource { public readonly events = new EventEmitter(); public getCheckpointsForEpoch = () => []; + public getCheckpointsDataForEpoch = () => []; constructor() {} } diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts index 77da4ac6d957..0de0f6b27f65 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts @@ -132,9 +132,9 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter const blocksByCheckpoint = chunkBy(sortedBlocks, b => b.checkpointNumber); // Get prior checkpoints in the epoch (in case this was a partial prune) to extract the out hashes - const priorCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsForEpoch(epochNumber)) - .filter(c => c.number < sortedBlocks[0].checkpointNumber) - .map(c => c.getCheckpointOutHash()); + const priorCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(epochNumber)) + .filter(c => c.checkpointNumber < sortedBlocks[0].checkpointNumber) + .map(c => c.checkpointOutHash); let previousCheckpointOutHashes: Fr[] = [...priorCheckpointOutHashes]; const fork = await this.checkpointsBuilder.getFork( @@ -172,6 +172,7 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter chainId: gv.chainId, version: gv.version, slotNumber: gv.slotNumber, + timestamp: gv.timestamp, coinbase: gv.coinbase, feeRecipient: gv.feeRecipient, gasFees: gv.gasFees, diff --git a/yarn-project/stdlib/src/block/block_data.ts b/yarn-project/stdlib/src/block/block_data.ts new file mode 100644 index 000000000000..a0305f88f9df --- /dev/null +++ b/yarn-project/stdlib/src/block/block_data.ts @@ -0,0 +1,26 @@ +import { CheckpointNumberSchema, IndexWithinCheckpointSchema } from '@aztec/foundation/branded-types'; +import type { CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types'; +import type { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; + +import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; +import { BlockHeader } from '../tx/block_header.js'; + +/** L2Block metadata. Equivalent to L2Block but without block body containing tx data. */ +export type BlockData = { + header: BlockHeader; + archive: AppendOnlyTreeSnapshot; + blockHash: Fr; + checkpointNumber: CheckpointNumber; + indexWithinCheckpoint: IndexWithinCheckpoint; +}; + +export const BlockDataSchema = z.object({ + header: BlockHeader.schema, + archive: AppendOnlyTreeSnapshot.schema, + blockHash: schemas.Fr, + checkpointNumber: CheckpointNumberSchema, + indexWithinCheckpoint: IndexWithinCheckpointSchema, +}); diff --git a/yarn-project/stdlib/src/block/index.ts b/yarn-project/stdlib/src/block/index.ts index 80ce846cf277..ded8ffecdf57 100644 --- a/yarn-project/stdlib/src/block/index.ts +++ b/yarn-project/stdlib/src/block/index.ts @@ -1,3 +1,4 @@ +export * from './block_data.js'; export * from './l2_block.js'; export * from './l2_block_stream/index.js'; export * from './in_block.js'; diff --git a/yarn-project/stdlib/src/block/l2_block_source.ts b/yarn-project/stdlib/src/block/l2_block_source.ts index 76013631eccb..f1daf550d7eb 100644 --- a/yarn-project/stdlib/src/block/l2_block_source.ts +++ b/yarn-project/stdlib/src/block/l2_block_source.ts @@ -13,6 +13,7 @@ import type { TypedEventEmitter } from '@aztec/foundation/types'; import { z } from 'zod'; import type { Checkpoint } from '../checkpoint/checkpoint.js'; +import type { CheckpointData } from '../checkpoint/checkpoint_data.js'; import type { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import type { L1RollupConstants } from '../epoch-helpers/index.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; @@ -20,6 +21,7 @@ import type { BlockHeader } from '../tx/block_header.js'; import type { IndexedTxEffect } from '../tx/indexed_tx_effect.js'; import type { TxHash } from '../tx/tx_hash.js'; import type { TxReceipt } from '../tx/tx_receipt.js'; +import type { BlockData } from './block_data.js'; import type { BlockHash } from './block_hash.js'; import type { CheckpointedL2Block } from './checkpointed_l2_block.js'; import type { L2Block } from './l2_block.js'; @@ -98,6 +100,12 @@ export interface L2BlockSource { */ getCheckpointsForEpoch(epochNumber: EpochNumber): Promise; + /** + * Gets lightweight checkpoint metadata for a given epoch, without fetching full block data. + * @param epochNumber - Epoch for which we want checkpoint data + */ + getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise; + /** * Gets a block header by its hash. * @param blockHash - The block hash to retrieve. @@ -112,6 +120,20 @@ export interface L2BlockSource { */ getBlockHeaderByArchive(archive: Fr): Promise; + /** + * Gets block metadata (without tx data) by block number. + * @param number - The block number to retrieve. + * @returns The requested block data (or undefined if not found). + */ + getBlockData(number: BlockNumber): Promise; + + /** + * Gets block metadata (without tx data) by archive root. + * @param archive - The archive root to retrieve. + * @returns The requested block data (or undefined if not found). + */ + getBlockDataByArchive(archive: Fr): Promise; + /** * Gets an L2 block by block number. * @param number - The block number to return. diff --git a/yarn-project/stdlib/src/checkpoint/checkpoint.ts b/yarn-project/stdlib/src/checkpoint/checkpoint.ts index 9e633345f89c..2c95d3c0be4a 100644 --- a/yarn-project/stdlib/src/checkpoint/checkpoint.ts +++ b/yarn-project/stdlib/src/checkpoint/checkpoint.ts @@ -94,9 +94,11 @@ export class Checkpoint { return this.header.hash(); } - // Returns the out hash computed from all l2-to-l1 messages in this checkpoint. - // Note: This value is different from the out hash in the header, which is the **accumulated** out hash over all - // checkpoints up to and including this one in the epoch. + /** + * Returns the out hash computed from all l2-to-l1 messages in this checkpoint. + * Note: This value is different from the out hash in the header, which is the **accumulated** out hash over all + * checkpoints up to and including this one in the epoch. + */ public getCheckpointOutHash(): Fr { const msgs = this.blocks.map(block => block.body.txEffects.map(txEffect => txEffect.l2ToL1Msgs)); return computeCheckpointOutHash(msgs); diff --git a/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts b/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts new file mode 100644 index 000000000000..32dd799181ee --- /dev/null +++ b/yarn-project/stdlib/src/checkpoint/checkpoint_data.ts @@ -0,0 +1,51 @@ +import { + BlockNumber, + BlockNumberSchema, + CheckpointNumber, + CheckpointNumberSchema, +} from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas } from '@aztec/foundation/schemas'; + +import { z } from 'zod'; + +import { CommitteeAttestation } from '../block/proposal/committee_attestation.js'; +import { CheckpointHeader } from '../rollup/checkpoint_header.js'; +import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; +import { L1PublishedData } from './published_checkpoint.js'; + +/** Lightweight checkpoint metadata without full block data. */ +export type CheckpointData = { + checkpointNumber: CheckpointNumber; + header: CheckpointHeader; + archive: AppendOnlyTreeSnapshot; + checkpointOutHash: Fr; + startBlock: BlockNumber; + blockCount: number; + attestations: CommitteeAttestation[]; + l1: L1PublishedData; +}; + +export const CheckpointDataSchema = z + .object({ + checkpointNumber: CheckpointNumberSchema, + header: CheckpointHeader.schema, + archive: AppendOnlyTreeSnapshot.schema, + checkpointOutHash: schemas.Fr, + startBlock: BlockNumberSchema, + blockCount: schemas.Integer, + attestations: z.array(CommitteeAttestation.schema), + l1: L1PublishedData.schema, + }) + .transform( + (obj): CheckpointData => ({ + checkpointNumber: obj.checkpointNumber, + header: obj.header, + archive: obj.archive, + checkpointOutHash: obj.checkpointOutHash, + startBlock: obj.startBlock, + blockCount: obj.blockCount, + attestations: obj.attestations, + l1: obj.l1, + }), + ); diff --git a/yarn-project/stdlib/src/checkpoint/index.ts b/yarn-project/stdlib/src/checkpoint/index.ts index 6c189e5a5ddc..d86f88c87bbb 100644 --- a/yarn-project/stdlib/src/checkpoint/index.ts +++ b/yarn-project/stdlib/src/checkpoint/index.ts @@ -1,3 +1,4 @@ export * from './checkpoint.js'; +export * from './checkpoint_data.js'; export * from './checkpoint_info.js'; export * from './published_checkpoint.js'; diff --git a/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts b/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts index d5afc5c2e3e0..67c2104fe05b 100644 --- a/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts +++ b/yarn-project/stdlib/src/checkpoint/published_checkpoint.ts @@ -55,9 +55,11 @@ export class L1PublishedData { export class PublishedCheckpoint { constructor( + /** The checkpoint itself. */ public checkpoint: Checkpoint, + /** Info on when this checkpoint was published on L1. */ public l1: L1PublishedData, - // The attestations for the last block in the checkpoint. + /** The attestations for the last block in the checkpoint. */ public attestations: CommitteeAttestation[], ) {} diff --git a/yarn-project/stdlib/src/file-store/local.ts b/yarn-project/stdlib/src/file-store/local.ts index e2c3409666ba..a01ade236dff 100644 --- a/yarn-project/stdlib/src/file-store/local.ts +++ b/yarn-project/stdlib/src/file-store/local.ts @@ -1,15 +1,21 @@ import { access, mkdir, readFile, writeFile } from 'fs/promises'; import { dirname, resolve } from 'path'; +import { promisify } from 'util'; +import { gunzip as gunzipCb, gzip as gzipCb } from 'zlib'; -import type { FileStore } from './interface.js'; +import type { FileStore, FileStoreSaveOptions } from './interface.js'; + +const gzip = promisify(gzipCb); +const gunzip = promisify(gunzipCb); export class LocalFileStore implements FileStore { constructor(private readonly basePath: string) {} - public async save(path: string, data: Buffer): Promise { + public async save(path: string, data: Buffer, opts?: FileStoreSaveOptions): Promise { const fullPath = this.getFullPath(path); await mkdir(dirname(fullPath), { recursive: true }); - await writeFile(fullPath, data); + const toWrite = opts?.compress ? await gzip(data) : data; + await writeFile(fullPath, toWrite); return `file://${fullPath}`; } @@ -18,9 +24,13 @@ export class LocalFileStore implements FileStore { return this.save(destPath, data); } - public read(pathOrUrlStr: string): Promise { + public async read(pathOrUrlStr: string): Promise { const fullPath = this.getFullPath(pathOrUrlStr); - return readFile(fullPath); + const data = await readFile(fullPath); + if (data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b) { + return await gunzip(data); + } + return data; } public async download(pathOrUrlStr: string, destPath: string): Promise { diff --git a/yarn-project/stdlib/src/file-store/s3.ts b/yarn-project/stdlib/src/file-store/s3.ts index 2033a1e86d9d..22d41347b4bc 100644 --- a/yarn-project/stdlib/src/file-store/s3.ts +++ b/yarn-project/stdlib/src/file-store/s3.ts @@ -13,10 +13,14 @@ import { tmpdir } from 'os'; import { basename, dirname, join } from 'path'; import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; -import { createGzip } from 'zlib'; +import { promisify } from 'util'; +import { createGzip, gunzip as gunzipCb, gzip as gzipCb } from 'zlib'; import type { FileStore, FileStoreSaveOptions } from './interface.js'; +const gzip = promisify(gzipCb); +const gunzip = promisify(gunzipCb); + function normalizeBasePath(path: string): string { return path?.replace(/^\/+|\/+$/g, '') ?? ''; } @@ -52,7 +56,7 @@ export class S3FileStore implements FileStore { const key = this.getFullPath(path); const shouldCompress = !!opts.compress; - const body = shouldCompress ? (await import('zlib')).gzipSync(data) : data; + const body = shouldCompress ? await gzip(data) : data; const contentLength = body.length; const contentType = this.detectContentType(key, shouldCompress); const put = new PutObjectCommand({ @@ -60,6 +64,7 @@ export class S3FileStore implements FileStore { Key: key, Body: body, ContentType: contentType, + ContentEncoding: shouldCompress ? 'gzip' : undefined, CacheControl: opts.metadata?.['Cache-control'], Metadata: this.extractUserMetadata(opts.metadata), ContentLength: contentLength, @@ -134,7 +139,11 @@ export class S3FileStore implements FileStore { for await (const chunk of stream) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } - return Buffer.concat(chunks); + const buffer = Buffer.concat(chunks); + if (out.ContentEncoding === 'gzip') { + return await gunzip(buffer); + } + return buffer; } public async download(pathOrUrlStr: string, destPath: string): Promise { diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index 2d605b35b0b4..2b5cb983325b 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -10,10 +10,11 @@ import type { ContractArtifact } from '../abi/abi.js'; import { FunctionSelector } from '../abi/function_selector.js'; import { AztecAddress } from '../aztec-address/index.js'; import { CheckpointedL2Block } from '../block/checkpointed_l2_block.js'; -import { BlockHash, CommitteeAttestation, L2Block } from '../block/index.js'; +import { type BlockData, BlockHash, CommitteeAttestation, L2Block } from '../block/index.js'; import type { L2Tips } from '../block/l2_block_source.js'; import type { ValidateCheckpointResult } from '../block/validate_block_result.js'; import { Checkpoint } from '../checkpoint/checkpoint.js'; +import type { CheckpointData } from '../checkpoint/checkpoint_data.js'; import { L1PublishedData, PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { getContractClassFromArtifact } from '../contract/contract_class.js'; import { @@ -110,6 +111,16 @@ describe('ArchiverApiSchema', () => { expect(result).toBeInstanceOf(BlockHeader); }); + it('getBlockData', async () => { + const result = await context.client.getBlockData(BlockNumber(1)); + expect(result).toBeUndefined(); + }); + + it('getBlockDataByArchive', async () => { + const result = await context.client.getBlockDataByArchive(Fr.random()); + expect(result).toBeUndefined(); + }); + it('getBlockHeaderByHash', async () => { const result = await context.client.getBlockHeaderByHash(BlockHash.random()); expect(result).toBeInstanceOf(BlockHeader); @@ -184,6 +195,14 @@ describe('ArchiverApiSchema', () => { expect(result).toEqual([expect.any(Checkpoint)]); }); + it('getCheckpointsDataForEpoch', async () => { + const result = await context.client.getCheckpointsDataForEpoch(EpochNumber(1)); + expect(result).toHaveLength(1); + expect(result[0].checkpointNumber).toBeDefined(); + expect(result[0].checkpointOutHash).toBeDefined(); + expect(result[0].attestations[0]).toBeInstanceOf(CommitteeAttestation); + }); + it('getCheckpointedBlock', async () => { const result = await context.client.getCheckpointedBlock(BlockNumber(1)); expect(result).toBeDefined(); @@ -453,6 +472,12 @@ class MockArchiver implements ArchiverApi { getBlockHeaderByArchive(_archive: Fr): Promise { return Promise.resolve(BlockHeader.empty()); } + getBlockData(_number: BlockNumber): Promise { + return Promise.resolve(undefined); + } + getBlockDataByArchive(_archive: Fr): Promise { + return Promise.resolve(undefined); + } getL2Block(number: BlockNumber): Promise { return L2Block.random(number); } @@ -485,6 +510,22 @@ class MockArchiver implements ArchiverApi { expect(epochNumber).toEqual(EpochNumber(1)); return [await Checkpoint.random(CheckpointNumber(1))]; } + async getCheckpointsDataForEpoch(epochNumber: EpochNumber): Promise { + expect(epochNumber).toEqual(EpochNumber(1)); + const checkpoint = await Checkpoint.random(CheckpointNumber(1)); + return [ + { + checkpointNumber: checkpoint.number, + header: checkpoint.header, + archive: checkpoint.archive, + checkpointOutHash: checkpoint.getCheckpointOutHash(), + startBlock: BlockNumber(1), + blockCount: checkpoint.blocks.length, + attestations: [CommitteeAttestation.random()], + l1: L1PublishedData.random(), + }, + ]; + } async getCheckpointedBlocksForEpoch(epochNumber: EpochNumber): Promise { expect(epochNumber).toEqual(EpochNumber(1)); const block = await L2Block.random(BlockNumber(Number(epochNumber))); diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 51d62fd5cb6b..9af2b49e6fbc 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -4,12 +4,14 @@ import type { ApiSchemaFor } from '@aztec/foundation/schemas'; import { z } from 'zod'; +import { BlockDataSchema } from '../block/block_data.js'; import { BlockHash } from '../block/block_hash.js'; import { CheckpointedL2Block } from '../block/checkpointed_l2_block.js'; import { L2Block } from '../block/l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../block/l2_block_source.js'; import { ValidateCheckpointResultSchema } from '../block/validate_block_result.js'; import { Checkpoint } from '../checkpoint/checkpoint.js'; +import { CheckpointDataSchema } from '../checkpoint/checkpoint_data.js'; import { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { ContractClassPublicSchema, @@ -104,6 +106,8 @@ export const ArchiverApiSchema: ApiSchemaFor = { getCheckpointedBlockByArchive: z.function().args(schemas.Fr).returns(CheckpointedL2Block.schema.optional()), getBlockHeaderByHash: z.function().args(BlockHash.schema).returns(BlockHeader.schema.optional()), getBlockHeaderByArchive: z.function().args(schemas.Fr).returns(BlockHeader.schema.optional()), + getBlockData: z.function().args(BlockNumberSchema).returns(BlockDataSchema.optional()), + getBlockDataByArchive: z.function().args(schemas.Fr).returns(BlockDataSchema.optional()), getL2Block: z.function().args(BlockNumberSchema).returns(L2Block.schema.optional()), getL2BlockByHash: z.function().args(BlockHash.schema).returns(L2Block.schema.optional()), getL2BlockByArchive: z.function().args(schemas.Fr).returns(L2Block.schema.optional()), @@ -112,6 +116,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { getL2SlotNumber: z.function().args().returns(schemas.SlotNumber.optional()), getL2EpochNumber: z.function().args().returns(EpochNumberSchema.optional()), getCheckpointsForEpoch: z.function().args(EpochNumberSchema).returns(z.array(Checkpoint.schema)), + getCheckpointsDataForEpoch: z.function().args(EpochNumberSchema).returns(z.array(CheckpointDataSchema)), getCheckpointedBlocksForEpoch: z.function().args(EpochNumberSchema).returns(z.array(CheckpointedL2Block.schema)), getBlocksForSlot: z.function().args(schemas.SlotNumber).returns(z.array(L2Block.schema)), getCheckpointedBlockHeadersForEpoch: z.function().args(EpochNumberSchema).returns(z.array(BlockHeader.schema)), diff --git a/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts index 01ad4f83e43a..61014496fe1e 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts @@ -85,6 +85,10 @@ describe('AztecNodeAdminApiSchema', () => { epochOrSlot: expect.any(BigInt), }); }); + + it('reloadKeystore', async () => { + await context.client.reloadKeystore(); + }); }); class MockAztecNodeAdmin implements AztecNodeAdmin { @@ -189,4 +193,7 @@ class MockAztecNodeAdmin implements AztecNodeAdmin { resumeSync(): Promise { return Promise.resolve(); } + reloadKeystore(): Promise { + return Promise.resolve(); + } } diff --git a/yarn-project/stdlib/src/interfaces/aztec-node-admin.ts b/yarn-project/stdlib/src/interfaces/aztec-node-admin.ts index 1003734261f8..8c3f41786dce 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node-admin.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node-admin.ts @@ -50,6 +50,26 @@ export interface AztecNodeAdmin { /** Returns all offenses applicable for the given round. */ getSlashOffenses(round: bigint | 'all' | 'current'): Promise; + + /** + * Reloads keystore configuration from disk. + * + * What is updated: + * - Validator attester keys + * - Coinbase address per validator + * - Fee recipient address per validator + * + * What is NOT updated (requires node restart): + * - L1 publisher signers (the funded accounts that send L1 transactions) + * - Prover keys + * - HA signer PostgreSQL connections + * + * Notes: + * - New validators must use a publisher key that was already configured at node + * startup (or omit the publisher field to fall back to the attester key). + * A validator with an unknown publisher key will cause the reload to be rejected. + */ + reloadKeystore(): Promise; } // L1 contracts are not mutable via admin updates. @@ -88,16 +108,19 @@ export const AztecNodeAdminApiSchema: ApiSchemaFor = { .function() .args(z.union([z.bigint(), z.literal('all'), z.literal('current')])) .returns(z.array(OffenseSchema)), + reloadKeystore: z.function().returns(z.void()), }; export function createAztecNodeAdminClient( url: string, versions: Partial = {}, fetch = defaultFetch, + apiKey?: string, ): AztecNodeAdmin { return createSafeJsonRpcClient(url, AztecNodeAdminApiSchema, { namespaceMethods: 'nodeAdmin', fetch, onResponse: getVersioningResponseHandler(versions), + ...(apiKey ? { extraHeaders: { 'x-api-key': apiKey } } : {}), }); } diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts index 2243efd963dc..436945dd773e 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts @@ -16,7 +16,7 @@ import times from 'lodash.times'; import type { ContractArtifact } from '../abi/abi.js'; import { AztecAddress } from '../aztec-address/index.js'; import type { DataInBlock } from '../block/in_block.js'; -import { BlockHash, type BlockParameter, CommitteeAttestation, L2Block } from '../block/index.js'; +import { type BlockData, BlockHash, type BlockParameter, CommitteeAttestation, L2Block } from '../block/index.js'; import type { L2Tips } from '../block/l2_block_source.js'; import { Checkpoint } from '../checkpoint/checkpoint.js'; import { L1PublishedData, PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; @@ -637,6 +637,12 @@ class MockAztecNode implements AztecNode { getBlockHeaderByArchive(_archive: Fr): Promise { return Promise.resolve(BlockHeader.empty()); } + getBlockData(_number: BlockNumber): Promise { + return Promise.resolve(undefined); + } + getBlockDataByArchive(_archive: Fr): Promise { + return Promise.resolve(undefined); + } getCurrentMinFees(): Promise { return Promise.resolve(GasFees.empty()); } diff --git a/yarn-project/stdlib/src/interfaces/configs.ts b/yarn-project/stdlib/src/interfaces/configs.ts index 2b94ba64f1db..5149006d2f65 100644 --- a/yarn-project/stdlib/src/interfaces/configs.ts +++ b/yarn-project/stdlib/src/interfaces/configs.ts @@ -69,6 +69,10 @@ export interface SequencerConfig { buildCheckpointIfEmpty?: boolean; /** Skip pushing proposed blocks to archiver (default: false) */ skipPushProposedBlocksToArchiver?: boolean; + /** Minimum number of blocks required for a checkpoint proposal (test only, defaults to undefined = no minimum) */ + minBlocksForCheckpoint?: number; + /** Skip publishing checkpoint proposals probability (for testing checkpoint prunes only) */ + skipPublishingCheckpointsPercent?: number; } export const SequencerConfigSchema = zodFor()( @@ -103,6 +107,8 @@ export const SequencerConfigSchema = zodFor()( blockDurationMs: z.number().positive().optional(), buildCheckpointIfEmpty: z.boolean().optional(), skipPushProposedBlocksToArchiver: z.boolean().optional(), + minBlocksForCheckpoint: z.number().positive().optional(), + skipPublishingCheckpointsPercent: z.number().gte(0).lte(100).optional(), }), ); @@ -117,7 +123,8 @@ type SequencerConfigOptionalKeys = | 'fakeThrowAfterProcessingTxCount' | 'l1PublishingTime' | 'txPublicSetupAllowList' - | 'minValidTxsPerBlock'; + | 'minValidTxsPerBlock' + | 'minBlocksForCheckpoint'; export type ResolvedSequencerConfig = Prettify< Required> & Pick diff --git a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts index 0da6e8b70080..63ee8e82f9b1 100644 --- a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts @@ -254,7 +254,7 @@ export interface MerkleTreeCheckpointOperations { export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations, MerkleTreeCheckpointOperations, - Disposable { + AsyncDisposable { /** * Appends leaves to a given tree. * @param treeId - The tree to be updated. diff --git a/yarn-project/stdlib/src/p2p/block_proposal.ts b/yarn-project/stdlib/src/p2p/block_proposal.ts index 91f1c96e821b..8371d8adb9b1 100644 --- a/yarn-project/stdlib/src/p2p/block_proposal.ts +++ b/yarn-project/stdlib/src/p2p/block_proposal.ts @@ -56,8 +56,6 @@ export class BlockProposal extends Gossipable { /** The per-block header containing block state and global variables */ public readonly blockHeader: BlockHeader, - // TODO(palla/mbps): Is this really needed? Can we just derive it from the indexWithinCheckpoint of the parent block and the slot number? - // See the block-proposal-handler, we have a lot of extra validations to check this is correct, so maybe we can avoid storing it here. /** Index of this block within the checkpoint (0-indexed) */ public readonly indexWithinCheckpoint: IndexWithinCheckpoint, diff --git a/yarn-project/stdlib/src/rollup/checkpoint_header.ts b/yarn-project/stdlib/src/rollup/checkpoint_header.ts index 81ce41fd6976..2f42b8993a71 100644 --- a/yarn-project/stdlib/src/rollup/checkpoint_header.ts +++ b/yarn-project/stdlib/src/rollup/checkpoint_header.ts @@ -19,8 +19,8 @@ import type { UInt64 } from '../types/shared.js'; /** * Header of a checkpoint. A checkpoint is a collection of blocks submitted to L1 all within the same slot. - * TODO(palla/mbps): Should this include chainId and version as well? Is this used just in circuits? - * TODO(palla/mbps): What about CheckpointNumber? + * This header is verified as-is in the rollup circuits, posted to the L1 rollup contract, stored in the archiver, + * and exposed via the Aztec Node API. See `CheckpointData` for a struct that includes the header plus extra metadata. */ export class CheckpointHeader { constructor( diff --git a/yarn-project/stdlib/src/tx/global_variables.ts b/yarn-project/stdlib/src/tx/global_variables.ts index f475f73ff42a..a5b5ff521ab8 100644 --- a/yarn-project/stdlib/src/tx/global_variables.ts +++ b/yarn-project/stdlib/src/tx/global_variables.ts @@ -22,10 +22,10 @@ import { schemas } from '../schemas/index.js'; import type { UInt64 } from '../types/index.js'; /** - * Global variables that are constant across the entire slot. - * TODO(palla/mbps): Should timestamp be included here as well? + * Global variables that are constant across the entire checkpoint (slot). + * Excludes blockNumber since that varies per block within a checkpoint. */ -export type CheckpointGlobalVariables = Omit, 'blockNumber' | 'timestamp'>; +export type CheckpointGlobalVariables = Omit, 'blockNumber'>; /** * Global variables of the L2 block. diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 77b1d019f22c..297746ae2a61 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -128,6 +128,9 @@ export const NODEJS_EVENT_LOOP_STATE = 'nodejs.eventloop.state'; export const TOPIC_NAME = 'aztec.gossip.topic_name'; +/** The reason a transaction was evicted from the tx pool */ +export const TX_POOL_EVICTION_REASON = 'aztec.mempool.eviction_reason'; + export const TX_COLLECTION_METHOD = 'aztec.tx_collection.method'; /** Scope of L1 transaction (sequencer, prover, or other) */ diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index c76fc0926ed5..0187114e0d5d 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -167,6 +167,55 @@ export const MEMPOOL_TX_MINED_DELAY: MetricDefinition = { valueType: ValueType.INT, }; +export const MEMPOOL_TX_POOL_V2_EVICTED_COUNT: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.evicted_count', + description: 'The number of transactions evicted from the tx pool', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_IGNORED_COUNT: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.ignored_count', + description: 'The number of transactions ignored in addPendingTxs', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_REJECTED_COUNT: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.rejected_count', + description: 'The number of transactions rejected in addPendingTxs', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_SOFT_DELETED_HITS: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.soft_deleted_hits', + description: 'The number of transactions found in the soft-deleted pool', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_MISSING_ON_PROTECT: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.missing_on_protect', + description: 'The number of truly missing transactions in protectTxs', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_MISSING_PREVIOUSLY_EVICTED: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.missing_previously_evicted', + description: 'The number of truly missing transactions in protectTxs that were previously evicted', + valueType: ValueType.INT, +}; +export const MEMPOOL_TX_POOL_V2_METADATA_MEMORY: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.metadata_memory', + description: 'Estimated total memory consumed by in-memory transaction metadata', + unit: 'By', + valueType: ValueType.INT, +}; + +export const MEMPOOL_TX_POOL_V2_DUPLICATE_ADD: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.duplicate_add', + description: 'Transactions received via addPendingTxs that were already in the pool', + valueType: ValueType.INT, +}; + +export const MEMPOOL_TX_POOL_V2_ALREADY_PROTECTED_ADD: MetricDefinition = { + name: 'aztec.mempool.tx_pool_v2.already_protected_add', + description: 'Transactions received via addPendingTxs that were already pre-protected', + valueType: ValueType.INT, +}; + export const DB_NUM_ITEMS: MetricDefinition = { name: 'aztec.db.num_items', description: 'LMDB Num Items', @@ -224,6 +273,11 @@ export const ARCHIVER_BLOCK_HEIGHT: MetricDefinition = { description: 'The height of the latest block processed by the archiver', valueType: ValueType.INT, }; +export const ARCHIVER_CHECKPOINT_HEIGHT: MetricDefinition = { + name: 'aztec.archiver.checkpoint_height', + description: 'The height of the latest checkpoint processed by the archiver', + valueType: ValueType.INT, +}; export const ARCHIVER_ROLLUP_PROOF_DELAY: MetricDefinition = { name: 'aztec.archiver.rollup_proof_delay', description: 'Time after a block is submitted until its proof is published', @@ -344,9 +398,9 @@ export const SEQUENCER_BLOCK_COUNT: MetricDefinition = { description: 'Number of blocks built by this sequencer', valueType: ValueType.INT, }; -export const SEQUENCER_CURRENT_BLOCK_REWARDS: MetricDefinition = { - name: 'aztec.sequencer.current_block_rewards', - description: 'The rewards earned', +export const SEQUENCER_CURRENT_SLOT_REWARDS: MetricDefinition = { + name: 'aztec.sequencer.current_slot_rewards', + description: 'The rewards earned per filled slot', valueType: ValueType.DOUBLE, }; export const SEQUENCER_SLOT_COUNT: MetricDefinition = { @@ -369,12 +423,12 @@ export const SEQUENCER_CHECKPOINT_ATTESTATION_DELAY: MetricDefinition = { export const SEQUENCER_COLLECTED_ATTESTATIONS_COUNT: MetricDefinition = { name: 'aztec.sequencer.attestations.collected_count', - description: 'The number of attestations collected for a block proposal', + description: 'The number of attestations collected for a checkpoint proposal', valueType: ValueType.INT, }; export const SEQUENCER_REQUIRED_ATTESTATIONS_COUNT: MetricDefinition = { name: 'aztec.sequencer.attestations.required_count', - description: 'The minimum number of attestations required to publish a block', + description: 'The minimum number of attestations required to publish a checkpoint', valueType: ValueType.INT, }; export const SEQUENCER_COLLECT_ATTESTATIONS_DURATION: MetricDefinition = { @@ -395,14 +449,42 @@ export const SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT: MetricDefinition = { description: 'The number of times block proposal failed (including validation builds)', valueType: ValueType.INT, }; -export const SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT: MetricDefinition = { - name: 'aztec.sequencer.block.proposal_success_count', - description: 'The number of times block proposal succeeded (including validation builds)', +export const SEQUENCER_CHECKPOINT_PROPOSAL_SUCCESS_COUNT: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.proposal_success_count', + description: 'The number of times checkpoint proposal succeeded', + valueType: ValueType.INT, +}; +export const SEQUENCER_CHECKPOINT_PRECHECK_FAILED_COUNT: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.precheck_failed_count', + description: 'The number of times checkpoint pre-build checks failed', + valueType: ValueType.INT, +}; +export const SEQUENCER_CHECKPOINT_PROPOSAL_FAILED_COUNT: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.proposal_failed_count', + description: 'The number of times checkpoint proposal failed', + valueType: ValueType.INT, +}; +export const SEQUENCER_CHECKPOINT_BUILD_DURATION: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.build_duration', + description: 'Total duration to build all blocks in a checkpoint', + unit: 'ms', + valueType: ValueType.INT, +}; +export const SEQUENCER_CHECKPOINT_BLOCK_COUNT: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.block_count', + description: 'Number of blocks built in a checkpoint', + valueType: ValueType.INT, +}; +export const SEQUENCER_CHECKPOINT_TX_COUNT: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.tx_count', + description: 'Total number of transactions across all blocks in a checkpoint', + unit: 'tx', valueType: ValueType.INT, }; -export const SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT: MetricDefinition = { - name: 'aztec.sequencer.block.proposal_precheck_failed_count', - description: 'The number of times block proposal pre-build checks failed', +export const SEQUENCER_CHECKPOINT_TOTAL_MANA: MetricDefinition = { + name: 'aztec.sequencer.checkpoint.total_mana', + description: 'Total L2 mana used across all blocks in a checkpoint', + unit: 'mana', valueType: ValueType.INT, }; export const SEQUENCER_SLASHING_ATTEMPTS_COUNT: MetricDefinition = { @@ -1320,6 +1402,28 @@ export const TX_FILE_STORE_QUEUE_SIZE: MetricDefinition = { description: 'Number of txs pending upload', valueType: ValueType.INT, }; +export const TX_FILE_STORE_DOWNLOADS_SUCCESS: MetricDefinition = { + name: 'aztec.p2p.tx_file_store.downloads_success', + description: 'Number of successful tx downloads from file storage', + valueType: ValueType.INT, +}; +export const TX_FILE_STORE_DOWNLOADS_FAILED: MetricDefinition = { + name: 'aztec.p2p.tx_file_store.downloads_failed', + description: 'Number of failed tx downloads from file storage', + valueType: ValueType.INT, +}; +export const TX_FILE_STORE_DOWNLOAD_DURATION: MetricDefinition = { + name: 'aztec.p2p.tx_file_store.download_duration', + description: 'Duration to download a tx from file storage', + unit: 'ms', + valueType: ValueType.INT, +}; +export const TX_FILE_STORE_DOWNLOAD_SIZE: MetricDefinition = { + name: 'aztec.p2p.tx_file_store.download_size', + description: 'Size of a downloaded tx from file storage', + unit: 'By', + valueType: ValueType.INT, +}; export const IVC_VERIFIER_TIME: MetricDefinition = { name: 'aztec.ivc_verifier.time', diff --git a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts index fa6615cc8b13..db9c94b94942 100644 --- a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts +++ b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts @@ -16,7 +16,7 @@ import type { StatusMessage, } from '@aztec/p2p'; import type { EthAddress, L2BlockStreamEvent, L2Tips } from '@aztec/stdlib/block'; -import type { PeerInfo } from '@aztec/stdlib/interfaces/server'; +import type { ITxProvider, PeerInfo } from '@aztec/stdlib/interfaces/server'; import type { BlockProposal, CheckpointAttestation, CheckpointProposal } from '@aztec/stdlib/p2p'; import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx'; @@ -131,6 +131,10 @@ export class DummyP2P implements P2P { throw new Error('DummyP2P does not implement "isP2PClient"'); } + public getTxProvider(): ITxProvider { + throw new Error('DummyP2P does not implement "getTxProvider"'); + } + public getTxsByHash(_txHashes: TxHash[]): Promise { throw new Error('DummyP2P does not implement "getTxsByHash"'); } @@ -171,10 +175,6 @@ export class DummyP2P implements P2P { throw new Error('DummyP2P does not implement "hasTxsInPool"'); } - public addTxsToPool(_txs: Tx[]): Promise { - throw new Error('DummyP2P does not implement "addTxs"'); - } - public getSyncedLatestBlockNum(): Promise { throw new Error('DummyP2P does not implement "getSyncedLatestBlockNum"'); } diff --git a/yarn-project/txe/src/state_machine/index.ts b/yarn-project/txe/src/state_machine/index.ts index 194166720e15..cf7884c24ff4 100644 --- a/yarn-project/txe/src/state_machine/index.ts +++ b/yarn-project/txe/src/state_machine/index.ts @@ -50,6 +50,7 @@ export class TXEStateMachine { undefined, undefined, undefined, + undefined, VERSION, CHAIN_ID, new TXEGlobalVariablesBuilder(), diff --git a/yarn-project/validator-client/src/block_proposal_handler.ts b/yarn-project/validator-client/src/block_proposal_handler.ts index fba4fcb25f3f..0c8812aee9a8 100644 --- a/yarn-project/validator-client/src/block_proposal_handler.ts +++ b/yarn-project/validator-client/src/block_proposal_handler.ts @@ -1,7 +1,6 @@ import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; import type { EpochCache } from '@aztec/epoch-cache'; import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { chunkBy } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { TimeoutError } from '@aztec/foundation/error'; import { createLogger } from '@aztec/foundation/log'; @@ -9,16 +8,12 @@ import { retryUntil } from '@aztec/foundation/retry'; import { DateProvider, Timer } from '@aztec/foundation/timer'; import type { P2P, PeerId } from '@aztec/p2p'; import { BlockProposalValidator } from '@aztec/p2p/msg_validators'; -import type { L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; import { getEpochAtSlot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import type { ITxProvider, ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; -import { - type L1ToL2MessageSource, - computeCheckpointOutHash, - computeInHashFromL1ToL2Messages, -} from '@aztec/stdlib/messaging'; +import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; import type { BlockProposal } from '@aztec/stdlib/p2p'; -import { BlockHeader, type CheckpointGlobalVariables, type FailedTx, type Tx } from '@aztec/stdlib/tx'; +import type { CheckpointGlobalVariables, FailedTx, Tx } from '@aztec/stdlib/tx'; import { ReExFailedTxsError, ReExStateMismatchError, @@ -153,16 +148,16 @@ export class BlockProposalHandler { } // Check that the parent proposal is a block we know, otherwise reexecution would fail - const parentBlockHeader = await this.getParentBlock(proposal); - if (parentBlockHeader === undefined) { + const parentBlock = await this.getParentBlock(proposal); + if (parentBlock === undefined) { this.log.warn(`Parent block for proposal not found, skipping processing`, proposalInfo); return { isValid: false, reason: 'parent_block_not_found' }; } // Check that the parent block's slot is not greater than the proposal's slot. - if (parentBlockHeader !== 'genesis' && parentBlockHeader.getSlot() > slotNumber) { + if (parentBlock !== 'genesis' && parentBlock.header.getSlot() > slotNumber) { this.log.warn(`Parent block slot is greater than proposal slot, skipping processing`, { - parentBlockSlot: parentBlockHeader.getSlot().toString(), + parentBlockSlot: parentBlock.header.getSlot().toString(), proposalSlot: slotNumber.toString(), ...proposalInfo, }); @@ -171,9 +166,9 @@ export class BlockProposalHandler { // Compute the block number based on the parent block const blockNumber = - parentBlockHeader === 'genesis' + parentBlock === 'genesis' ? BlockNumber(INITIAL_L2_BLOCK_NUM) - : BlockNumber(parentBlockHeader.getBlockNumber() + 1); + : BlockNumber(parentBlock.header.getBlockNumber() + 1); // Check that this block number does not exist already const existingBlock = await this.blockSource.getBlockHeader(blockNumber); @@ -190,7 +185,7 @@ export class BlockProposalHandler { }); // Compute the checkpoint number for this block and validate checkpoint consistency - const checkpointResult = await this.computeCheckpointNumber(proposal, parentBlockHeader, proposalInfo); + const checkpointResult = this.computeCheckpointNumber(proposal, parentBlock, proposalInfo); if (checkpointResult.reason) { return { isValid: false, blockNumber, reason: checkpointResult.reason }; } @@ -218,17 +213,11 @@ export class BlockProposalHandler { // Try re-executing the transactions in the proposal if needed let reexecutionResult; if (shouldReexecute) { - // Compute the previous checkpoint out hashes for the epoch. - // TODO(leila/mbps): There can be a more efficient way to get the previous checkpoint out - // hashes without having to fetch all the blocks. + // Collect the out hashes of all the checkpoints before this one in the same epoch const epoch = getEpochAtSlot(slotNumber, this.epochCache.getL1Constants()); - const checkpointedBlocks = (await this.blockSource.getCheckpointedBlocksForEpoch(epoch)) - .filter(b => b.block.number < blockNumber) - .sort((a, b) => a.block.number - b.block.number); - const blocksByCheckpoint = chunkBy(checkpointedBlocks, b => b.checkpointNumber); - const previousCheckpointOutHashes = blocksByCheckpoint.map(checkpointBlocks => - computeCheckpointOutHash(checkpointBlocks.map(b => b.block.body.txEffects.map(tx => tx.l2ToL1Msgs))), - ); + const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) + .filter(c => c.checkpointNumber < checkpointNumber) + .map(c => c.checkpointOutHash); try { this.log.verbose(`Re-executing transactions in the proposal`, proposalInfo); @@ -260,7 +249,7 @@ export class BlockProposalHandler { return { isValid: true, blockNumber, reexecutionResult }; } - private async getParentBlock(proposal: BlockProposal): Promise<'genesis' | BlockHeader | undefined> { + private async getParentBlock(proposal: BlockProposal): Promise<'genesis' | BlockData | undefined> { const parentArchive = proposal.blockHeader.lastArchive.root; const slot = proposal.slotNumber; const config = this.checkpointsBuilder.getConfig(); @@ -276,12 +265,11 @@ export class BlockProposalHandler { try { return ( - (await this.blockSource.getBlockHeaderByArchive(parentArchive)) ?? + (await this.blockSource.getBlockDataByArchive(parentArchive)) ?? (timeoutDurationMs <= 0 ? undefined : await retryUntil( - () => - this.blockSource.syncImmediate().then(() => this.blockSource.getBlockHeaderByArchive(parentArchive)), + () => this.blockSource.syncImmediate().then(() => this.blockSource.getBlockDataByArchive(parentArchive)), 'force archiver sync', timeoutDurationMs / 1000, 0.5, @@ -297,12 +285,12 @@ export class BlockProposalHandler { } } - private async computeCheckpointNumber( + private computeCheckpointNumber( proposal: BlockProposal, - parentBlockHeader: 'genesis' | BlockHeader, + parentBlock: 'genesis' | BlockData, proposalInfo: object, - ): Promise { - if (parentBlockHeader === 'genesis') { + ): CheckpointComputationResult { + if (parentBlock === 'genesis') { // First block is in checkpoint 1 if (proposal.indexWithinCheckpoint !== 0) { this.log.warn(`First block proposal has non-zero indexWithinCheckpoint`, proposalInfo); @@ -311,19 +299,9 @@ export class BlockProposalHandler { return { checkpointNumber: CheckpointNumber.INITIAL }; } - // Get the parent block to find its checkpoint number - // TODO(palla/mbps): The block header should include the checkpoint number to avoid this lookup, - // or at least the L2BlockSource should return a different struct that includes it. - const parentBlockNumber = parentBlockHeader.getBlockNumber(); - const parentBlock = await this.blockSource.getL2Block(parentBlockNumber); - if (!parentBlock) { - this.log.warn(`Parent block ${parentBlockNumber} not found in archiver`, proposalInfo); - return { reason: 'invalid_proposal' }; - } - if (proposal.indexWithinCheckpoint === 0) { // If this is the first block in a new checkpoint, increment the checkpoint number - if (!(proposal.blockHeader.getSlot() > parentBlockHeader.getSlot())) { + if (!(proposal.blockHeader.getSlot() > parentBlock.header.getSlot())) { this.log.warn(`Slot should be greater than parent block slot for first block in checkpoint`, proposalInfo); return { reason: 'invalid_proposal' }; } @@ -335,7 +313,7 @@ export class BlockProposalHandler { this.log.warn(`Non-sequential indexWithinCheckpoint`, proposalInfo); return { reason: 'invalid_proposal' }; } - if (proposal.blockHeader.getSlot() !== parentBlockHeader.getSlot()) { + if (proposal.blockHeader.getSlot() !== parentBlock.header.getSlot()) { this.log.warn(`Slot should be equal to parent block slot for non-first block in checkpoint`, proposalInfo); return { reason: 'invalid_proposal' }; } @@ -356,7 +334,7 @@ export class BlockProposalHandler { */ private validateNonFirstBlockInCheckpoint( proposal: BlockProposal, - parentBlock: L2Block, + parentBlock: BlockData, proposalInfo: object, ): CheckpointComputationResult | undefined { const proposalGlobals = proposal.blockHeader.globalVariables; @@ -475,13 +453,14 @@ export class BlockProposalHandler { // Fork before the block to be built const parentBlockNumber = BlockNumber(blockNumber - 1); await this.worldState.syncImmediate(parentBlockNumber); - using fork = await this.worldState.fork(parentBlockNumber); + await using fork = await this.worldState.fork(parentBlockNumber); - // Build checkpoint constants from proposal (excludes blockNumber and timestamp which are per-block) + // Build checkpoint constants from proposal (excludes blockNumber which is per-block) const constants: CheckpointGlobalVariables = { chainId: new Fr(config.l1ChainId), version: new Fr(config.rollupVersion), slotNumber: slot, + timestamp: blockHeader.globalVariables.timestamp, coinbase: blockHeader.globalVariables.coinbase, feeRecipient: blockHeader.globalVariables.feeRecipient, gasFees: blockHeader.globalVariables.gasFees, diff --git a/yarn-project/validator-client/src/checkpoint_builder.test.ts b/yarn-project/validator-client/src/checkpoint_builder.test.ts index 9b1323316a0e..76899d131bdd 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.test.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.test.ts @@ -41,6 +41,7 @@ describe('CheckpointBuilder', () => { chainId: new Fr(1), version: new Fr(1), slotNumber, + timestamp: BigInt(Date.now()), coinbase: EthAddress.random(), feeRecipient: AztecAddress.fromField(Fr.random()), gasFees: GasFees.empty(), diff --git a/yarn-project/validator-client/src/validator.integration.test.ts b/yarn-project/validator-client/src/validator.integration.test.ts index 109996b6ca23..dd30f91bd4be 100644 --- a/yarn-project/validator-client/src/validator.integration.test.ts +++ b/yarn-project/validator-client/src/validator.integration.test.ts @@ -114,7 +114,7 @@ describe('ValidatorClient Integration', () => { worldStateBlockCheckIntervalMS: 20, worldStateBlockRequestBatchSize: 10, worldStateDbMapSizeKb: 1024 * 1024, - worldStateBlockHistory: 0, + worldStateCheckpointHistory: 0, }; const worldStateDb = await NativeWorldStateService.tmp(rollupAddress, true, prefilledPublicData); const synchronizer = new ServerWorldStateSynchronizer(worldStateDb, archiver, wsConfig); @@ -279,9 +279,10 @@ describe('ValidatorClient Integration', () => { feeRecipient: await AztecAddress.random(), gasFees: GasFees.empty(), slotNumber: slot, + timestamp: BigInt(Date.now()), }; - using fork = await proposer.worldStateDb.fork(); + await using fork = await proposer.worldStateDb.fork(); const builder = await proposer.checkpointsBuilder.startCheckpoint( checkpointNumber, globalVariables, diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index 620e3b2b0b79..7485939da4fc 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -23,7 +23,7 @@ import { } from '@aztec/p2p'; import { OffenseType, WANT_TO_SLASH_EVENT } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import type { L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; import type { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; import { Gas } from '@aztec/stdlib/gas'; import type { SlasherConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; @@ -51,9 +51,32 @@ import type { FullNodeCheckpointsBuilder, } from './checkpoint_builder.js'; import { type ValidatorClientConfig, validatorClientConfigMappings } from './config.js'; -import type { HAKeyStore } from './key_store/ha_key_store.js'; +import { HAKeyStore } from './key_store/ha_key_store.js'; import { ValidatorClient } from './validator.js'; +function makeKeyStore(validator: { + attester: Hex<32>[] | Hex<32>; + coinbase?: EthAddress; + feeRecipient?: AztecAddress; + publisher?: Hex<32>[]; +}): KeyStore { + return { + schemaVersion: 1, + slasher: undefined, + prover: undefined, + remoteSigner: undefined, + validators: [ + { + attester: Array.isArray(validator.attester) ? validator.attester : [validator.attester], + feeRecipient: validator.feeRecipient ?? AztecAddress.ZERO, + coinbase: validator.coinbase, + remoteSigner: undefined, + publisher: validator.publisher ?? [], + }, + ], + }; +} + describe('ValidatorClient', () => { let config: ValidatorClientConfig & Pick< @@ -62,7 +85,7 @@ describe('ValidatorClient', () => { > & { disableTransactions: boolean; }; - let validatorClient: ValidatorClient; + let validatorClient: TestValidatorClient; let p2pClient: MockProxy; let blockSource: MockProxy; let l1ToL2MessageSource: MockProxy; @@ -96,6 +119,7 @@ describe('ValidatorClient', () => { >[1] as any); blockSource = mock(); blockSource.getCheckpointedBlocksForEpoch.mockResolvedValue([]); + blockSource.getCheckpointsDataForEpoch.mockResolvedValue([]); blockSource.getBlocksForSlot.mockResolvedValue([]); epochCache.isEscapeHatchOpenAtSlot.mockResolvedValue(false); l1ToL2MessageSource = mock(); @@ -132,24 +156,9 @@ describe('ValidatorClient', () => { maxStuckDutiesAgeMs: 72000, }; - const keyStore: KeyStore = { - schemaVersion: 1, - slasher: undefined, - prover: undefined, - remoteSigner: undefined, - validators: [ - { - attester: validatorPrivateKeys.map(key => key as Hex<32>), - feeRecipient: AztecAddress.ZERO, - coinbase: undefined, - remoteSigner: undefined, - publisher: [], - }, - ], - }; - keyStoreManager = new KeystoreManager(keyStore); + keyStoreManager = new KeystoreManager(makeKeyStore({ attester: validatorPrivateKeys.map(key => key as Hex<32>) })); - validatorClient = await ValidatorClient.new( + validatorClient = (await ValidatorClient.new( config, checkpointsBuilder, worldState, @@ -161,7 +170,7 @@ describe('ValidatorClient', () => { keyStoreManager, blobClient, dateProvider, - ); + )) as TestValidatorClient; }); describe('createBlockProposal', () => { @@ -301,7 +310,7 @@ describe('ValidatorClient', () => { checkpointsBuilder.openCheckpoint.mockResolvedValue(mockCheckpointBuilder); worldState.fork.mockResolvedValue({ close: () => Promise.resolve(), - [Symbol.dispose]: () => {}, + [Symbol.asyncDispose]: () => Promise.resolve(), } as never); }; @@ -336,23 +345,19 @@ describe('ValidatorClient', () => { epochCache.filterInCommittee.mockResolvedValue([EthAddress.fromString(validatorAccounts[0].address)]); epochCache.isEscapeHatchOpenAtSlot.mockResolvedValue(false); - // Return parent block header when requested - blockSource.getBlockHeaderByArchive.mockResolvedValue({ - getBlockNumber: () => blockNumber - 1, - getSlot: () => SlotNumber(Number(blockHeader.globalVariables.slotNumber) - 1), - } as BlockHeader); - - // Return parent block when requested (needed for checkpoint number computation) - // The parent block has slot - 1, which is different from the proposal's slot + // Return parent block data when requested (includes checkpoint info, avoids loading full L2Block) const parentSlot = SlotNumber(Number(blockHeader.globalVariables.slotNumber) - 1); - blockSource.getL2Block.mockResolvedValue({ - checkpointNumber: CheckpointNumber(1), - indexWithinCheckpoint: IndexWithinCheckpoint(0), + blockSource.getBlockDataByArchive.mockResolvedValue({ header: { - globalVariables: blockHeader.globalVariables, + getBlockNumber: () => blockNumber - 1, getSlot: () => parentSlot, + globalVariables: blockHeader.globalVariables, }, - } as unknown as L2Block); + archive: new AppendOnlyTreeSnapshot(Fr.random(), blockNumber - 1), + blockHash: Fr.random(), + checkpointNumber: CheckpointNumber(1), + indexWithinCheckpoint: IndexWithinCheckpoint(0), + } as unknown as BlockData); blockSource.getGenesisValues.mockResolvedValue({ genesisArchiveRoot: new Fr(GENESIS_ARCHIVE_ROOT) }); blockSource.syncImmediate.mockImplementation(() => Promise.resolve()); @@ -453,6 +458,7 @@ describe('ValidatorClient', () => { it('should attest to a checkpoint proposal after validating a block for that slot', async () => { const addCheckpointAttestationsSpy = jest.spyOn(p2pClient, 'addOwnCheckpointAttestations'); + const uploadBlobsSpy = jest.spyOn(validatorClient, 'uploadBlobsForCheckpoint'); const didValidate = await validatorClient.validateBlockProposal(proposal, sender); expect(didValidate).toBe(true); @@ -467,21 +473,65 @@ describe('ValidatorClient', () => { }, }); + // Enable blob upload for this attestation + blobClient.canUpload.mockReturnValue(true); + validatorClient.updateConfig({ skipCheckpointProposalValidation: true }); const attestations = await validatorClient.attestToCheckpointProposal(checkpointProposal, sender); expect(attestations).toBeDefined(); expect(attestations).toHaveLength(1); expect(addCheckpointAttestationsSpy).toHaveBeenCalledTimes(1); + expect(uploadBlobsSpy).toHaveBeenCalled(); + + uploadBlobsSpy.mockRestore(); + }); + + it('should not attest to a checkpoint proposal that references a middle block instead of the last', async () => { + const addCheckpointAttestationsSpy = jest.spyOn(p2pClient, 'addOwnCheckpointAttestations'); + + // First validate a block proposal so the validator has seen a block for this slot + const didValidate = await validatorClient.validateBlockProposal(proposal, sender); + expect(didValidate).toBe(true); + + // Create 3 blocks for the slot, each with a distinct archive root + const block1Archive = new AppendOnlyTreeSnapshot(Fr.random(), 1); + const block2Archive = new AppendOnlyTreeSnapshot(Fr.random(), 2); + const block3Archive = new AppendOnlyTreeSnapshot(Fr.random(), 3); + const blocks = [ + { archive: block1Archive, number: 1 }, + { archive: block2Archive, number: 2 }, + { archive: block3Archive, number: 3 }, + ] as unknown as L2Block[]; + + // Proposal references the middle block's archive (block 2), not the last (block 3) + const checkpointProposal = await makeCheckpointProposal({ + archiveRoot: block2Archive.root, + checkpointHeader: makeCheckpointHeader(0, { slotNumber: proposal.slotNumber }), + lastBlock: { + blockHeader: makeBlockHeader(1, { blockNumber: BlockNumber(2), slotNumber: proposal.slotNumber }), + indexWithinCheckpoint: IndexWithinCheckpoint(1), + txHashes: proposal.txHashes, + }, + }); + + // Mock getBlockHeaderByArchive to return a header so retryUntil succeeds + blockSource.getBlockHeaderByArchive.mockResolvedValue(makeBlockHeader()); + blockSource.getBlocksForSlot.mockResolvedValue(blocks); + + // Checkpoint validation should fail: proposal points to block 2 but last block in slot is block 3 + const attestations = await validatorClient.attestToCheckpointProposal(checkpointProposal, sender); + expect(attestations).toBeUndefined(); + expect(addCheckpointAttestationsSpy).not.toHaveBeenCalled(); }); it('should wait for previous block to sync', async () => { epochCache.filterInCommittee.mockResolvedValue([EthAddress.fromString(validatorAccounts[0].address)]); - blockSource.getBlockHeaderByArchive.mockResolvedValueOnce(undefined); - blockSource.getBlockHeaderByArchive.mockResolvedValueOnce(undefined); - blockSource.getBlockHeaderByArchive.mockResolvedValueOnce(undefined); + blockSource.getBlockDataByArchive.mockResolvedValueOnce(undefined); + blockSource.getBlockDataByArchive.mockResolvedValueOnce(undefined); + blockSource.getBlockDataByArchive.mockResolvedValueOnce(undefined); const isValid = await validatorClient.validateBlockProposal(proposal, sender); - expect(blockSource.getBlockHeaderByArchive).toHaveBeenCalledTimes(4); + expect(blockSource.getBlockDataByArchive).toHaveBeenCalledTimes(4); expect(isValid).toBe(true); }); @@ -686,23 +736,18 @@ describe('ValidatorClient', () => { nextSlot: SlotNumber(nonFirstBlockProposal.slotNumber + 1), }); - // Mock parent block header returned by getBlockHeaderByArchive - const parentBlockHeader = { - getBlockNumber: () => BlockNumber(parentBlockNumber), - getSlot: () => SlotNumber(parentSlotNumber), - globalVariables: parentGlobalVariables, - } as BlockHeader; - blockSource.getBlockHeaderByArchive.mockResolvedValue(parentBlockHeader); - - // Mock parent block returned by getL2Block - const parentBlock = { - checkpointNumber: parentCheckpointNumber, - indexWithinCheckpoint: IndexWithinCheckpoint(0), // Parent is first block in checkpoint + // Mock parent block data returned by getBlockDataByArchive + blockSource.getBlockDataByArchive.mockResolvedValue({ header: { + getBlockNumber: () => BlockNumber(parentBlockNumber), + getSlot: () => SlotNumber(parentSlotNumber), globalVariables: parentGlobalVariables, }, - } as unknown as L2Block; - blockSource.getL2Block.mockResolvedValue(parentBlock); + archive: new AppendOnlyTreeSnapshot(Fr.random(), parentBlockNumber), + blockHash: Fr.random(), + checkpointNumber: parentCheckpointNumber, + indexWithinCheckpoint: IndexWithinCheckpoint(0), // Parent is first block in checkpoint + } as unknown as BlockData); // Set time for the slot const genesisTime = 1n; @@ -731,16 +776,6 @@ describe('ValidatorClient', () => { // block_proposal_handler.ts. }); - // TODO(palla/mbps): Blob upload functionality has been moved to checkpoint proposal handling (Phase 6) - // These tests are skipped until the blob upload is implemented in the new location. - describe.skip('filestore blob upload', () => { - it.todo('should upload blobs to filestore after successful checkpoint proposal'); - it.todo('should not attempt upload when fileStoreBlobUploadClient is undefined'); - it.todo('should not fail when blob upload fails'); - it.todo('should trigger re-execution when filestore is configured even if validatorReexecute is false'); - it.todo('should not upload blobs when validation fails'); - }); - it('should validate proposals in fisherman mode but not create or broadcast attestations', async () => { // Enable fisherman mode (which also triggers re-execution) validatorClient.updateConfig({ fishermanMode: true }); @@ -833,6 +868,42 @@ describe('ValidatorClient', () => { }); }); + describe('uploadBlobsForCheckpoint', () => { + const proposalInfo = { slotNumber: 1, archive: '0x00', proposer: '0x00', txCount: 0 }; + + it('should send blobs from blocks in the slot to filestore', async () => { + const blobFields = [Fr.random(), Fr.random()]; + const mockBlock = { toBlobFields: () => blobFields } as unknown as L2Block; + blockSource.getBlockHeaderByArchive.mockResolvedValue(makeBlockHeader()); + blockSource.getBlocksForSlot.mockResolvedValue([mockBlock]); + + const proposal = await makeCheckpointProposal({ lastBlock: {} }); + await validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo); + + expect(blockSource.getBlocksForSlot).toHaveBeenCalledWith(proposal.slotNumber); + expect(blobClient.sendBlobsToFilestore).toHaveBeenCalled(); + }); + + it('should not upload if last block header is not found', async () => { + blockSource.getBlockHeaderByArchive.mockResolvedValue(undefined); + + const proposal = await makeCheckpointProposal({ lastBlock: {} }); + await validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo); + + expect(blobClient.sendBlobsToFilestore).not.toHaveBeenCalled(); + }); + + it('should not throw when blob upload fails', async () => { + const mockBlock = { toBlobFields: () => [Fr.random()] } as unknown as L2Block; + blockSource.getBlockHeaderByArchive.mockResolvedValue(makeBlockHeader()); + blockSource.getBlocksForSlot.mockResolvedValue([mockBlock]); + blobClient.sendBlobsToFilestore.mockRejectedValue(new Error('upload failed')); + + const proposal = await makeCheckpointProposal({ lastBlock: {} }); + await expect(validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo)).resolves.toBeUndefined(); + }); + }); + describe('configuration', () => { it('should use VALIDATOR_PRIVATE_KEY for validatorPrivateKeys when VALIDATOR_PRIVATE_KEYS is not set', () => { const originalEnv = process.env; @@ -871,4 +942,131 @@ describe('ValidatorClient', () => { expect(haKeyStore.stop).toHaveBeenCalledTimes(1); }); }); + + describe('reloadKeystore', () => { + // build a KeystoreManager from a single-validator KeyStore and reload. + const reloadWith = (overrides: Parameters[0]) => { + const manager = new KeystoreManager(makeKeyStore(overrides)); + validatorClient.reloadKeystore(manager); + return manager; + }; + + const allKeys = () => config.validatorPrivateKeys!.getValue().map(k => k as Hex<32>); + + it('should update coinbase after reload', () => { + const newCoinbase = EthAddress.random(); + reloadWith({ attester: allKeys(), coinbase: newCoinbase }); + + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(newCoinbase); + }); + + it('should update fee recipient after reload', async () => { + const newFeeRecipient = await AztecAddress.random(); + reloadWith({ attester: allKeys(), feeRecipient: newFeeRecipient }); + + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + expect(validatorClient.getFeeRecipientForAttestor(attestorAddress)).toEqual(newFeeRecipient); + }); + + it('should add new validator after reload', () => { + const newPrivateKey = generatePrivateKey(); + const newAccount = privateKeyToAccount(newPrivateKey); + reloadWith({ attester: [...allKeys(), newPrivateKey as Hex<32>] }); + + const addresses = validatorClient.getValidatorAddresses(); + expect(addresses).toHaveLength(3); + expect(addresses.some(a => a.equals(EthAddress.fromString(newAccount.address)))).toBe(true); + }); + + it('should update attester key after reload', () => { + const newPrivateKey = generatePrivateKey(); + const newAccount = privateKeyToAccount(newPrivateKey); + reloadWith({ attester: newPrivateKey as Hex<32> }); + + const addresses = validatorClient.getValidatorAddresses(); + expect(addresses).toHaveLength(1); + expect(addresses[0]).toEqual(EthAddress.fromString(newAccount.address)); + }); + + it('should remove a validator after reload', () => { + const remainingKey = config.validatorPrivateKeys!.getValue()[0] as Hex<32>; + const removedAccount = validatorAccounts[1]; + reloadWith({ attester: remainingKey }); + + const addresses = validatorClient.getValidatorAddresses(); + expect(addresses).toHaveLength(1); + expect(addresses.some(a => a.equals(EthAddress.fromString(removedAccount.address)))).toBe(false); + + // Accessing the removed validator's coinbase should throw + expect(() => validatorClient.getCoinbaseForAttestor(EthAddress.fromString(removedAccount.address))).toThrow( + /not found in any validator configuration/, + ); + }); + + it('should change coinbase and no longer return the old one', () => { + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + + const oldCoinbase = EthAddress.random(); + reloadWith({ attester: allKeys(), coinbase: oldCoinbase }); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(oldCoinbase); + + const newCoinbase = EthAddress.random(); + reloadWith({ attester: allKeys(), coinbase: newCoinbase }); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(newCoinbase); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).not.toEqual(oldCoinbase); + }); + + it('should reset coinbase to attester fallback when removed', () => { + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + + const explicitCoinbase = EthAddress.random(); + reloadWith({ attester: allKeys(), coinbase: explicitCoinbase }); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(explicitCoinbase); + + // Reload without coinbase — falls back to the attester address itself + reloadWith({ attester: allKeys() }); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(attestorAddress); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).not.toEqual(explicitCoinbase); + }); + + it('should change fee recipient and no longer return the old one', async () => { + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + + const oldFeeRecipient = await AztecAddress.random(); + reloadWith({ attester: allKeys(), feeRecipient: oldFeeRecipient }); + expect(validatorClient.getFeeRecipientForAttestor(attestorAddress)).toEqual(oldFeeRecipient); + + const newFeeRecipient = await AztecAddress.random(); + reloadWith({ attester: allKeys(), feeRecipient: newFeeRecipient }); + expect(validatorClient.getFeeRecipientForAttestor(attestorAddress)).toEqual(newFeeRecipient); + expect(validatorClient.getFeeRecipientForAttestor(attestorAddress)).not.toEqual(oldFeeRecipient); + }); + + it('should preserve HA signer and wrap new adapter in HAKeyStore after reload', () => { + // Simulate HA mode by setting the haSigner and wrapping in HAKeyStore + const mockHASigner = { nodeId: 'test-ha-node' }; + (validatorClient as any).haSigner = mockHASigner; + (validatorClient as any).keyStore = haKeyStore; + + const newCoinbase = EthAddress.random(); + reloadWith({ attester: allKeys(), coinbase: newCoinbase }); + + // Verify the keyStore is an HAKeyStore wrapping the same haSigner + const keyStoreAfterReload = (validatorClient as any).keyStore; + expect(keyStoreAfterReload).toBeInstanceOf(HAKeyStore); + expect((keyStoreAfterReload as any).haSigner).toBe(mockHASigner); + + // Verify the new coinbase is accessible through the HAKeyStore + const attestorAddress = EthAddress.fromString(validatorAccounts[0].address); + expect(validatorClient.getCoinbaseForAttestor(attestorAddress)).toEqual(newCoinbase); + }); + }); }); + +/** Exposes protected methods for direct testing */ +class TestValidatorClient extends ValidatorClient { + declare public uploadBlobsForCheckpoint: ( + ...args: Parameters + ) => Promise; +} diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index a383f2dfe029..a1a8e112b264 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -47,6 +47,7 @@ import { AttestationTimeoutError } from '@aztec/stdlib/validators'; import { type TelemetryClient, type Tracer, getTelemetryClient } from '@aztec/telemetry-client'; import { createHASigner } from '@aztec/validator-ha-signer/factory'; import { DutyType, type SigningContext } from '@aztec/validator-ha-signer/types'; +import type { ValidatorHASigner } from '@aztec/validator-ha-signer/validator-ha-signer'; import { EventEmitter } from 'events'; import type { TypedDataDefinition } from 'viem'; @@ -77,7 +78,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private validationService: ValidationService; private metrics: ValidatorMetrics; private log: Logger; - // Whether it has already registered handlers on the p2p client private hasRegisteredHandlers = false; @@ -106,6 +106,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private l1ToL2MessageSource: L1ToL2MessageSource, private config: ValidatorClientFullConfig, private blobClient: BlobClientInterface, + private haSigner: ValidatorHASigner | undefined, private dateProvider: DateProvider = new DateProvider(), telemetry: TelemetryClient = getTelemetryClient(), log = createLogger('validator'), @@ -211,7 +212,9 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) telemetry, ); - let validatorKeyStore: ExtendedValidatorKeyStore = NodeKeystoreAdapter.fromKeyStoreManager(keyStoreManager); + const nodeKeystoreAdapter = NodeKeystoreAdapter.fromKeyStoreManager(keyStoreManager); + let validatorKeyStore: ExtendedValidatorKeyStore = nodeKeystoreAdapter; + let haSigner: ValidatorHASigner | undefined; if (config.haSigningEnabled) { // If maxStuckDutiesAgeMs is not explicitly set, compute it from Aztec slot duration const haConfig = { @@ -219,7 +222,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) maxStuckDutiesAgeMs: config.maxStuckDutiesAgeMs ?? epochCache.getL1Constants().slotDuration * 2 * 1000, }; const { signer } = await createHASigner(haConfig); - validatorKeyStore = new HAKeyStore(validatorKeyStore, signer); + haSigner = signer; + validatorKeyStore = new HAKeyStore(nodeKeystoreAdapter, signer); } const validator = new ValidatorClient( @@ -233,6 +237,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) l1ToL2MessageSource, config, blobClient, + haSigner, dateProvider, telemetry, ); @@ -270,6 +275,28 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) this.config = { ...this.config, ...config }; } + public reloadKeystore(newManager: KeystoreManager): void { + if (this.config.haSigningEnabled && !this.haSigner) { + this.log.warn( + 'HA signing is enabled in config but was not initialized at startup. ' + + 'Restart the node to enable HA signing.', + ); + } else if (!this.config.haSigningEnabled && this.haSigner) { + this.log.warn( + 'HA signing was disabled via config update but the HA signer is still active. ' + + 'Restart the node to fully disable HA signing.', + ); + } + + const newAdapter = NodeKeystoreAdapter.fromKeyStoreManager(newManager); + if (this.haSigner) { + this.keyStore = new HAKeyStore(newAdapter, this.haSigner); + } else { + this.keyStore = newAdapter; + } + this.validationService = new ValidationService(this.keyStore, this.log.createChild('validation-service')); + } + public async start() { if (this.epochCacheUpdateLoop.isRunning()) { this.log.warn(`Validator client already started`); @@ -643,6 +670,12 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return { isValid: false, reason: 'no_blocks_for_slot' }; } + // Ensure the last block for this slot matches the archive in the checkpoint proposal + if (!blocks.at(-1)?.archive.root.equals(proposal.archive)) { + this.log.warn(`Last block archive mismatch for checkpoint proposal`, proposalInfo); + return { isValid: false, reason: 'last_block_archive_mismatch' }; + } + this.log.debug(`Found ${blocks.length} blocks for slot ${slot}`, { ...proposalInfo, blockNumbers: blocks.map(b => b.number), @@ -656,14 +689,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) // Get L1-to-L2 messages for this checkpoint const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpointNumber); - // Compute the previous checkpoint out hashes for the epoch. - // TODO: There can be a more efficient way to get the previous checkpoint out hashes without having to fetch the - // actual checkpoints and the blocks/txs in them. + // Collect the out hashes of all the checkpoints before this one in the same epoch const epoch = getEpochAtSlot(slot, this.epochCache.getL1Constants()); - const previousCheckpoints = (await this.blockSource.getCheckpointsForEpoch(epoch)) - .filter(b => b.number < checkpointNumber) - .sort((a, b) => a.number - b.number); - const previousCheckpointOutHashes = previousCheckpoints.map(c => c.getCheckpointOutHash()); + const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) + .filter(c => c.checkpointNumber < checkpointNumber) + .map(c => c.checkpointOutHash); // Fork world state at the block before the first block const parentBlockNumber = BlockNumber(firstBlock.number - 1); @@ -737,6 +767,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) chainId: gv.chainId, version: gv.version, slotNumber: gv.slotNumber, + timestamp: gv.timestamp, coinbase: gv.coinbase, feeRecipient: gv.feeRecipient, gasFees: gv.gasFees, @@ -746,7 +777,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) /** * Uploads blobs for a checkpoint to the filestore (fire and forget). */ - private async uploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): Promise { + protected async uploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): Promise { try { const lastBlockHeader = await this.blockSource.getBlockHeaderByArchive(proposal.archive); if (!lastBlockHeader) { @@ -761,7 +792,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) } const blobFields = blocks.flatMap(b => b.toBlobFields()); - const blobs: Blob[] = getBlobsPerL1Block(blobFields); + const blobs: Blob[] = await getBlobsPerL1Block(blobFields); await this.blobClient.sendBlobsToFilestore(blobs); this.log.debug(`Uploaded ${blobs.length} blobs to filestore for checkpoint at slot ${proposal.slotNumber}`, { ...proposalInfo, diff --git a/yarn-project/validator-ha-signer/README.md b/yarn-project/validator-ha-signer/README.md index fa69ded55bc5..3969f709e34f 100644 --- a/yarn-project/validator-ha-signer/README.md +++ b/yarn-project/validator-ha-signer/README.md @@ -178,6 +178,16 @@ All signing operations require a `SigningContext` that includes: Note: `AUTH_REQUEST` duties bypass HA protection since signing multiple times is safe for authentication requests. +## Important Limitations + +### Database Isolation Per Rollup Version + +**You cannot use the same database to provide slashing protection for validator nodes running on different rollup versions** (e.g., current rollup and old rollup simultaneously). + +When the HA signer performs background cleanup via `cleanupOutdatedRollupDuties()`, it removes all duties where the rollup address doesn't match the current rollup address. If two validators running on different rollup versions share the same database, they will delete each other's duties during cleanup. + +**Solution**: Use separate databases for validators running on different rollup versions. Each rollup version requires its own isolated slashing protection database. + ## Development ```bash diff --git a/yarn-project/validator-ha-signer/src/db/postgres.test.ts b/yarn-project/validator-ha-signer/src/db/postgres.test.ts index 125f15bb593c..92758dde54ef 100644 --- a/yarn-project/validator-ha-signer/src/db/postgres.test.ts +++ b/yarn-project/validator-ha-signer/src/db/postgres.test.ts @@ -1476,15 +1476,16 @@ describe('PostgresSlashingProtectionDatabase', () => { it('should only clean up old signed duties, not signing or recent duties', async () => { const spDb = new PostgresSlashingProtectionDatabase(pool); - const oldTimestamp = new Date(Date.now() - 2 * 60 * 60 * 1000); // 2 hours ago - // Insert old signed duties (should be cleaned up) + // Insert old signed duties (should be cleaned up) - 2 hours old for (let i = 0; i < 2; i++) { await pglite.query( `INSERT INTO validator_duties ( rollup_address, validator_address, slot, block_number, block_index_within_checkpoint, duty_type, status, message_hash, signature, node_id, lock_token, started_at, completed_at - ) VALUES ($1, $2, $3, $4, $5, $6, 'signed', $7, '0xsignature', $8, 'token', $9, $9)`, + ) VALUES ($1, $2, $3, $4, $5, $6, 'signed', $7, '0xsignature', $8, 'token', + CURRENT_TIMESTAMP - INTERVAL '2 hours', + CURRENT_TIMESTAMP - INTERVAL '2 hours')`, [ ROLLUP_ADDRESS.toString(), VALIDATOR_ADDRESS.toString(), @@ -1494,18 +1495,18 @@ describe('PostgresSlashingProtectionDatabase', () => { DutyType.BLOCK_PROPOSAL, Buffer32.random().toString(), NODE_ID, - oldTimestamp, ], ); } - // Insert old signing duties (should NOT be cleaned up) + // Insert old signing duties (should NOT be cleaned up) - 2 hours old but still signing for (let i = 0; i < 2; i++) { await pglite.query( `INSERT INTO validator_duties ( rollup_address, validator_address, slot, block_number, block_index_within_checkpoint, duty_type, status, message_hash, node_id, lock_token, started_at - ) VALUES ($1, $2, $3, $4, $5, $6, 'signing', $7, $8, 'token', $9)`, + ) VALUES ($1, $2, $3, $4, $5, $6, 'signing', $7, $8, 'token', + CURRENT_TIMESTAMP - INTERVAL '2 hours')`, [ ROLLUP_ADDRESS.toString(), VALIDATOR_ADDRESS.toString(), @@ -1515,7 +1516,6 @@ describe('PostgresSlashingProtectionDatabase', () => { DutyType.BLOCK_PROPOSAL, Buffer32.random().toString(), NODE_ID, - oldTimestamp, ], ); } diff --git a/yarn-project/validator-ha-signer/src/db/postgres.ts b/yarn-project/validator-ha-signer/src/db/postgres.ts index 73aad3799b65..8c80f22ddd7b 100644 --- a/yarn-project/validator-ha-signer/src/db/postgres.ts +++ b/yarn-project/validator-ha-signer/src/db/postgres.ts @@ -254,8 +254,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat * @returns the number of duties cleaned up */ async cleanupOwnStuckDuties(nodeId: string, maxAgeMs: number): Promise { - const cutoff = new Date(Date.now() - maxAgeMs); - const result = await this.pool.query(CLEANUP_OWN_STUCK_DUTIES, [nodeId, cutoff]); + const result = await this.pool.query(CLEANUP_OWN_STUCK_DUTIES, [nodeId, maxAgeMs]); return result.rowCount ?? 0; } @@ -277,8 +276,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat * @returns the number of duties cleaned up */ async cleanupOldDuties(maxAgeMs: number): Promise { - const cutoff = new Date(Date.now() - maxAgeMs); - const result = await this.pool.query(CLEANUP_OLD_DUTIES, [cutoff]); + const result = await this.pool.query(CLEANUP_OLD_DUTIES, [maxAgeMs]); return result.rowCount ?? 0; } } diff --git a/yarn-project/validator-ha-signer/src/db/schema.ts b/yarn-project/validator-ha-signer/src/db/schema.ts index 92cd57c5e618..8b06a2c812a4 100644 --- a/yarn-project/validator-ha-signer/src/db/schema.ts +++ b/yarn-project/validator-ha-signer/src/db/schema.ts @@ -203,23 +203,24 @@ WHERE status = 'signed' /** * Query to clean up old duties (for maintenance) - * Removes SIGNED duties older than a specified timestamp + * Removes SIGNED duties older than a specified age (in milliseconds) */ export const CLEANUP_OLD_DUTIES = ` DELETE FROM validator_duties WHERE status = 'signed' - AND started_at < $1; + AND started_at < CURRENT_TIMESTAMP - ($1 || ' milliseconds')::INTERVAL; `; /** * Query to cleanup own stuck duties * Removes duties in 'signing' status for a specific node that are older than maxAgeMs + * Uses DB's CURRENT_TIMESTAMP to avoid clock skew issues between nodes */ export const CLEANUP_OWN_STUCK_DUTIES = ` DELETE FROM validator_duties WHERE node_id = $1 AND status = 'signing' - AND started_at < $2; + AND started_at < CURRENT_TIMESTAMP - ($2 || ' milliseconds')::INTERVAL; `; /** diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index d0eed23ccbdb..b7a107a8eb80 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -304,7 +304,7 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr } } - async [Symbol.dispose](): Promise { + async [Symbol.asyncDispose](): Promise { if (this.opts.closeDelayMs) { void sleep(this.opts.closeDelayMs) .then(() => this.close()) diff --git a/yarn-project/world-state/src/synchronizer/config.ts b/yarn-project/world-state/src/synchronizer/config.ts index 2ba5dd6b8a9e..b1f4a7321781 100644 --- a/yarn-project/world-state/src/synchronizer/config.ts +++ b/yarn-project/world-state/src/synchronizer/config.ts @@ -29,8 +29,8 @@ export interface WorldStateConfig { /** Optional directory for the world state DB, if unspecified will default to the general data directory */ worldStateDataDirectory?: string; - /** The number of historic blocks to maintain */ - worldStateBlockHistory: number; + /** The number of historic checkpoints worth of blocks to maintain */ + worldStateCheckpointHistory: number; } export const worldStateConfigMappings: ConfigMappingsType = { @@ -84,9 +84,11 @@ export const worldStateConfigMappings: ConfigMappingsType = { env: 'WS_DATA_DIRECTORY', description: 'Optional directory for the world state database', }, - worldStateBlockHistory: { - env: 'WS_NUM_HISTORIC_BLOCKS', - description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained', + worldStateCheckpointHistory: { + env: 'WS_NUM_HISTORIC_CHECKPOINTS', + description: + 'The number of historic checkpoints worth of blocks to maintain. Values less than 1 mean all history is maintained', + fallback: ['WS_NUM_HISTORIC_BLOCKS'], ...numberConfigHelper(64), }, }; diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index bc5576d98d95..c9eebe16914a 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -80,7 +80,7 @@ describe('ServerWorldStateSynchronizer', () => { const config: WorldStateConfig = { worldStateBlockCheckIntervalMS: 100, worldStateDbMapSizeKb: 1024 * 1024, - worldStateBlockHistory: 0, + worldStateCheckpointHistory: 0, }; server = new TestWorldStateSynchronizer(merkleTreeDb, blockAndMessagesSource, config, l2BlockStream); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index daa67b1882d1..38a4e9ce81e5 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -1,5 +1,5 @@ import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM, INITIAL_L2_CHECKPOINT_NUM } from '@aztec/constants'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import type { Fr } from '@aztec/foundation/curves/bn254'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; @@ -64,7 +64,7 @@ export class ServerWorldStateSynchronizer private readonly log: Logger = createLogger('world_state'), ) { this.merkleTreeCommitted = this.merkleTreeDb.getCommitted(); - this.historyToKeep = config.worldStateBlockHistory < 1 ? undefined : config.worldStateBlockHistory; + this.historyToKeep = config.worldStateCheckpointHistory < 1 ? undefined : config.worldStateCheckpointHistory; this.log.info( `Created world state synchroniser with block history of ${ this.historyToKeep === undefined ? 'infinity' : this.historyToKeep @@ -364,12 +364,37 @@ export class ServerWorldStateSynchronizer if (this.historyToKeep === undefined) { return; } - const newHistoricBlock = summary.finalizedBlockNumber - this.historyToKeep + 1; - if (newHistoricBlock <= 1) { + // Get the checkpointed block for the finalized block number + const finalisedCheckpoint = await this.l2BlockSource.getCheckpointedBlock(summary.finalizedBlockNumber); + if (finalisedCheckpoint === undefined) { + this.log.warn( + `Failed to retrieve checkpointed block for finalized block number: ${summary.finalizedBlockNumber}`, + ); + return; + } + // Compute the required historic checkpoint number + const newHistoricCheckpointNumber = finalisedCheckpoint.checkpointNumber - this.historyToKeep + 1; + if (newHistoricCheckpointNumber <= 1) { + return; + } + // Retrieve the historic checkpoint + const historicCheckpoints = await this.l2BlockSource.getCheckpoints( + CheckpointNumber(newHistoricCheckpointNumber), + 1, + ); + if (historicCheckpoints.length === 0 || historicCheckpoints[0] === undefined) { + this.log.warn(`Failed to retrieve checkpoint number ${newHistoricCheckpointNumber} from Archiver`); + return; + } + const historicCheckpoint = historicCheckpoints[0]; + if (historicCheckpoint.checkpoint.blocks.length === 0 || historicCheckpoint.checkpoint.blocks[0] === undefined) { + this.log.warn(`Retrieved checkpoint number ${newHistoricCheckpointNumber} has no blocks!`); return; } - this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`); - const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock)); + // Find the block at the start of the checkpoint and remove blocks up to this one + const newHistoricBlock = historicCheckpoint.checkpoint.blocks[0]; + this.log.verbose(`Pruning historic blocks to ${newHistoricBlock.number}`); + const status = await this.merkleTreeDb.removeHistoricalBlocks(BlockNumber(newHistoricBlock.number)); this.log.debug(`World state summary `, status.summary); } diff --git a/yarn-project/world-state/src/test/integration.test.ts b/yarn-project/world-state/src/test/integration.test.ts index 9a784bcfec27..4f75871da279 100644 --- a/yarn-project/world-state/src/test/integration.test.ts +++ b/yarn-project/world-state/src/test/integration.test.ts @@ -52,7 +52,7 @@ describe('world-state integration', () => { worldStateBlockCheckIntervalMS: 20, worldStateBlockRequestBatchSize: 5, worldStateDbMapSizeKb: 1024 * 1024, - worldStateBlockHistory: 0, + worldStateCheckpointHistory: 0, }; archiver = new MockPrefilledArchiver(checkpoints); diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 4e971365bfb2..aeb6b8c124d7 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -802,6 +802,7 @@ __metadata: "@aztec/archiver": "workspace:^" "@aztec/bb-prover": "workspace:^" "@aztec/blob-client": "workspace:^" + "@aztec/blob-lib": "workspace:^" "@aztec/constants": "workspace:^" "@aztec/epoch-cache": "workspace:^" "@aztec/ethereum": "workspace:^" @@ -815,6 +816,7 @@ __metadata: "@aztec/p2p": "workspace:^" "@aztec/protocol-contracts": "workspace:^" "@aztec/prover-client": "workspace:^" + "@aztec/prover-node": "workspace:^" "@aztec/sequencer-client": "workspace:^" "@aztec/simulator": "workspace:^" "@aztec/slasher": "workspace:^"