chore: use CloudFlare R2 storage for snapshot storage

ci3/bootstrap_ec2

@@ -267,6 +267,8 @@ function run {
         -e DOCKERHUB_PASSWORD=${DOCKERHUB_PASSWORD:-} \
         -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} \
         -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} \
+        -e R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID:-} \
+        -e R2_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY:-} \
         -e BUILD_SYSTEM_DEBUG=${BUILD_SYSTEM_DEBUG:-} \
         -e GITHUB_TOKEN=${GITHUB_TOKEN:-} \
         -e NETLIFY_SITE_ID=${NETLIFY_SITE_ID:-} \

spartan/aztec-network/templates/full-node.yaml

@@ -165,6 +165,14 @@ spec:
               value: "{{ .Values.aztec.sponsoredFPC }}"
             - name: SYNC_SNAPSHOTS_URL
               value: "{{ .Values.snapshots.syncUrl }}"
+            {{- if .Values.snapshots.s3AccessKeyId }}
+            - name: AWS_ACCESS_KEY_ID
+              value: "{{ .Values.snapshots.s3AccessKeyId }}"
+            {{- end }}
+            {{- if .Values.snapshots.s3SecretAccessKey }}
+            - name: AWS_SECRET_ACCESS_KEY
+              value: "{{ .Values.snapshots.s3SecretAccessKey }}"
+            {{- end }}
             - name: SENTINEL_ENABLED
               value: "{{ .Values.fullNode.sentinelEnabled }}"
             {{- if .Values.blobSink.enabled }}

spartan/aztec-network/values.yaml

@@ -38,6 +38,8 @@ snapshots:
   uploadLocation:
   syncUrl:
   frequency:
+  s3AccessKeyId:
+  s3SecretAccessKey:
 
 images:
   aztec:

spartan/aztec-network/values/archival-node.yaml

@@ -2,8 +2,8 @@ telemetry:
   enabled: true
 
 snapshots:
-  uploadLocation: "gs://aztec-testnet/snapshots/"
-  syncUrl: "https://storage.googleapis.com/aztec-testnet/snapshots/"
+  uploadLocation: ""
+  syncUrl: "https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev/snapshots/staging-public/"
   frequency: "0 0 * * *" # daily uploads at midnight
 
 network:

spartan/aztec-snapshots/values.yaml

@@ -7,6 +7,9 @@ snapshot:
   uploadLocation: null
   frequency: "0 0 * * *" # daily uploads at midnight
   aztecNodeAdminUrl: null
+  s3AccessKeyId:
+  s3SecretAccessKey:
+  s3SessionToken:
 
 image:
   repository: curlimages/curl

spartan/environments/staging-ignition.env

@@ -21,7 +21,9 @@ ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET
 OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
 VERIFY_CONTRACTS=true
 ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET
-STORE_SNAPSHOT_URL=REPLACE_WITH_GCP_SECRET/staging-ignition/
+STORE_SNAPSHOT_URL=s3://testnet-bucket/staging-ignition/?endpoint=https://REPLACE_WITH_GCP_SECRET.r2.cloudflarestorage.com&publicBaseUrl=https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev
+R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET
+R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET
 BOT_TRANSFERS_REPLICAS=0
 BOT_SWAPS_REPLICAS=0
 DEPLOY_INTERNAL_BOOTNODE=false

spartan/environments/staging-public.env

@@ -16,7 +16,9 @@ OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
 VERIFY_CONTRACTS=true
 ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET
 DEPLOY_INTERNAL_BOOTNODE=false
-STORE_SNAPSHOT_URL=REPLACE_WITH_GCP_SECRET/staging-public/
+STORE_SNAPSHOT_URL=s3://testnet-bucket/staging-public/?endpoint=https://REPLACE_WITH_GCP_SECRET.r2.cloudflarestorage.com&publicBaseUrl=https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev
+R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET
+R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET
 PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging-public/failed-proofs
 TEST_ACCOUNTS=false
 SPONSORED_FPC=true

spartan/environments/testnet.env

@@ -21,7 +21,9 @@ OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
 VERIFY_CONTRACTS=true
 ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET
 
-STORE_SNAPSHOT_URL=REPLACE_WITH_GCP_SECRET/testnet/
+STORE_SNAPSHOT_URL=s3://testnet-bucket/testnet/?endpoint=https://REPLACE_WITH_GCP_SECRET.r2.cloudflarestorage.com&publicBaseUrl=https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev
+R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET
+R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET
 DEPLOY_INTERNAL_BOOTNODE=false
 BOT_TRANSFERS_REPLICAS=0
 BOT_SWAPS_REPLICAS=0

spartan/scripts/deploy_network.sh

@@ -277,6 +277,8 @@ RELEASE_PREFIX = "${NAMESPACE}"
 NAMESPACE = "${NAMESPACE}"
 GCP_PROJECT_ID = "${GCP_PROJECT_ID}"
 GCP_REGION = "${GCP_REGION}"
+R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}"
+R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}"
 P2P_BOOTSTRAP_RESOURCE_PROFILE = "${RESOURCE_PROFILE}"
 VALIDATOR_RESOURCE_PROFILE = "${RESOURCE_PROFILE}"
 PROVER_RESOURCE_PROFILE = "${RESOURCE_PROFILE}"

spartan/scripts/setup_gcp_secrets.sh

@@ -41,8 +41,10 @@ declare -A SECRET_MAPPINGS=(
     ["ROLLUP_DEPLOYMENT_PRIVATE_KEY"]="sepolia-labs-rollup-private-key"
     ["OTEL_COLLECTOR_ENDPOINT"]="otel-collector-url"
     ["ETHERSCAN_API_KEY"]="etherscan-api-key"
-    ["STORE_SNAPSHOT_URL"]="gcs-testnet-snapshot-url"
     ["LABS_INFRA_MNEMONIC"]="sepolia-labs-${NETWORK}-mnemonic"
+    ["STORE_SNAPSHOT_URL"]="r2-account-id"
+    ["R2_ACCESS_KEY_ID"]="r2-access-key-id"
+    ["R2_SECRET_ACCESS_KEY"]="r2-secret-access-key"
 )
 
 # Replace placeholders with actual secrets

spartan/terraform/cloudflare/main.tf

@@ -0,0 +1,30 @@
+terraform {
+  backend "gcs" {
+    bucket = "aztec-terraform"
+    prefix = "terraform/state/cloudflare"
+  }
+  required_providers {
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "cloudflare" {
+  api_token = var.R2_API_TOKEN
+}
+
+# Create the R2 bucket
+resource "cloudflare_r2_bucket" "bucket" {
+  account_id = var.R2_ACCOUNT_ID
+  name       = var.BUCKET_NAME
+}
+
+# Enable the r2.dev public URL for this bucket
+resource "cloudflare_r2_managed_domain" "r2dev" {
+  account_id  = var.R2_ACCOUNT_ID
+  bucket_name = cloudflare_r2_bucket.bucket.name
+  enabled     = true
+}
+

spartan/terraform/cloudflare/outputs.tf

@@ -0,0 +1,15 @@
+output "bucket_name" {
+  value = cloudflare_r2_bucket.bucket.name
+}
+
+output "account_id" {
+  value = var.R2_ACCOUNT_ID
+}
+
+output "s3_endpoint" {
+  value = "https://${var.R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
+}
+
+output "upload_location" {
+  value = "s3://${cloudflare_r2_bucket.bucket.name}/snapshots/testnet/?endpoint=https://${var.R2_ACCOUNT_ID}.r2.cloudflarestorage.com&publicBaseUrl=https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev"
+}

spartan/terraform/cloudflare/variables.tf

@@ -0,0 +1,22 @@
+variable "R2_API_TOKEN" {
+  type = string
+}
+
+variable "R2_ACCOUNT_ID" {
+  type = string
+}
+
+variable "DOMAIN" {
+  type    = string
+  default = "aztec-labs.com"
+}
+
+variable "SUBDOMAIN" {
+  type    = string
+  default = "aztec-testnet"
+}
+
+variable "BUCKET_NAME" {
+  type    = string
+  default = "testnet-bucket"
+}

spartan/terraform/deploy-aztec-infra/main.tf

@@ -219,14 +219,14 @@ locals {
         }
       })] : []
       custom_settings = {
-        "nodeType"              = "rpc"
-        "node.env.NETWORK"      = var.NETWORK
-        "node.proverRealProofs" = var.PROVER_REAL_PROOFS
-
-        "ingress.rpc.enabled" = var.RPC_INGRESS_ENABLED
-        "ingress.rpc.host"    = var.RPC_INGRESS_HOST
+        "nodeType"                       = "rpc"
+        "node.env.NETWORK"               = var.NETWORK
+        "node.proverRealProofs"          = var.PROVER_REAL_PROOFS
+        "ingress.rpc.enabled"            = var.RPC_INGRESS_ENABLED
+        "ingress.rpc.host"               = var.RPC_INGRESS_HOST
+        "node.env.AWS_ACCESS_KEY_ID"     = var.R2_ACCESS_KEY_ID
+        "node.env.AWS_SECRET_ACCESS_KEY" = var.R2_SECRET_ACCESS_KEY
       }
-      boot_node_host_path  = "node.env.BOOT_NODE_HOST"
       bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES"
       wait                 = true
     }

spartan/terraform/deploy-aztec-infra/values/rpc.yaml

@@ -1,6 +1,8 @@
 node:
   env:
     OTEL_SERVICE_NAME: "node"
+    AWS_ACCESS_KEY_ID: ""
+    AWS_SECRET_ACCESS_KEY: ""
 
   preStartScript: |
     if [ -n "${BOOT_NODE_HOST:-}" ] && [ -z "${NETWORK:-}" ]; then

spartan/terraform/deploy-aztec-infra/variables.tf

@@ -1,3 +1,15 @@
+variable "R2_ACCESS_KEY_ID" {
+  description = "Cloudflare R2 access key id for RPC node snapshot uploads"
+  type        = string
+  default     = null
+}
+
+variable "R2_SECRET_ACCESS_KEY" {
+  description = "Cloudflare R2 secret access key for RPC node snapshot uploads"
+  type        = string
+  default     = null
+}
+
 variable "GCP_PROJECT_ID" {
   description = "GCP project id"
   type        = string

yarn-project/cli/src/config/chain_l2_config.ts

@@ -9,6 +9,8 @@ import path, { dirname, join } from 'path';
 
 import publicIncludeMetrics from '../../public_include_metric_prefixes.json' with { type: 'json' };
 
+const SNAPSHOT_URL = 'https://pub-f4a8c34d4bb7441ebf8f48d904512180.r2.dev/snapshots';
+
 export type L2ChainConfig = L1ContractsConfig &
   Omit<SlasherConfig, 'slashValidatorsNever' | 'slashValidatorsAlways'> & {
     l1ChainId: number;
@@ -83,7 +85,7 @@ export const stagingIgnitionL2ChainConfig: L2ChainConfig = {
   seqMinTxsPerBlock: 0,
   seqMaxTxsPerBlock: 0,
   realProofs: true,
-  snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/staging-ignition/',
+  snapshotsUrl: `${SNAPSHOT_URL}/staging-ignition/`,
   autoUpdate: 'config-and-version',
   autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-ignition.json',
   maxTxPoolSize: 100_000_000, // 100MB
@@ -164,7 +166,7 @@ export const stagingPublicL2ChainConfig: L2ChainConfig = {
   seqMinTxsPerBlock: 0,
   seqMaxTxsPerBlock: 20,
   realProofs: true,
-  snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/staging-public/',
+  snapshotsUrl: `${SNAPSHOT_URL}/staging-public/`,
   autoUpdate: 'config-and-version',
   autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/staging-public.json',
   publicIncludeMetrics,
@@ -217,7 +219,7 @@ export const testnetL2ChainConfig: L2ChainConfig = {
   seqMinTxsPerBlock: 0,
   seqMaxTxsPerBlock: 20,
   realProofs: true,
-  snapshotsUrl: 'https://storage.googleapis.com/aztec-testnet/snapshots/testnet/',
+  snapshotsUrl: `${SNAPSHOT_URL}/testnet/`,
   autoUpdate: 'config-and-version',
   autoUpdateUrl: 'https://storage.googleapis.com/aztec-testnet/auto-update/testnet.json',
   maxTxPoolSize: 100_000_000, // 100MB

yarn-project/stdlib/package.json

@@ -69,6 +69,7 @@
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.892.0",
     "@aztec/bb.js": "portal:../../barretenberg/ts",
     "@aztec/blob-lib": "workspace:^",
     "@aztec/constants": "workspace:^",

yarn-project/stdlib/src/file-store/factory.ts

@@ -4,9 +4,11 @@ import { GoogleCloudFileStore } from './gcs.js';
 import { HttpFileStore } from './http.js';
 import type { FileStore, ReadOnlyFileStore } from './interface.js';
 import { LocalFileStore } from './local.js';
+import { S3FileStore } from './s3.js';
 
 const supportedExamples = [
   `gs://bucket-name/path/to/store`,
+  `s3://bucket-name/path/to/store`,
   `file:///absolute/local/path/to/store`,
   `https://host/path`,
 ];
@@ -39,6 +41,19 @@ export async function createFileStore(
     } catch {
       throw new Error(`Invalid google cloud store definition: '${config}'.`);
     }
+  } else if (config.startsWith('s3://')) {
+    try {
+      const url = new URL(config);
+      const bucket = url.host;
+      const path = url.pathname.replace(/^\/+/, '');
+      const endpoint = url.searchParams.get('endpoint');
+      const publicBaseUrl = url.searchParams.get('publicBaseUrl') ?? undefined;
+      logger.info(`Creating S3 file store at ${bucket} ${path}`);
+      const store = new S3FileStore(bucket, path, { endpoint: endpoint ?? undefined, publicBaseUrl });
+      return store;
+    } catch {
+      throw new Error(`Invalid S3 store definition: '${config}'.`);
+    }
   } else {
     throw new Error(`Unknown file store config: '${config}'. Supported values are ${supportedExamples.join(', ')}.`);
   }