From 4cdd0d7317994967641c301c79568e0ecbd3a790 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 11:54:57 -0300
Subject: [PATCH 01/10] chore: Terraform template for prover-node

Adds a new tf template for prover-node. Updates the node so it no longer
submits proofs, and updates the prover agent so it connects to the
prover-node instead of the node.
---
 yarn-project/aztec/terraform/node/main.tf     |   4 +
 .../aztec/terraform/node/variables.tf         |   5 +
 .../aztec/terraform/prover-node/main.tf       | 378 ++++++++++++++++++
 .../aztec/terraform/prover-node/variables.tf  |  74 ++++
 yarn-project/aztec/terraform/prover/main.tf   |   8 +-
 5 files changed, 465 insertions(+), 4 deletions(-)
 create mode 100644 yarn-project/aztec/terraform/prover-node/main.tf
 create mode 100644 yarn-project/aztec/terraform/prover-node/variables.tf

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
index e64f28a961dd..edfa18cdfa41 100644
--- a/yarn-project/aztec/terraform/node/main.tf
+++ b/yarn-project/aztec/terraform/node/main.tf
@@ -236,6 +236,10 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "SEQ_PUBLISHER_PRIVATE_KEY"
           value = local.sequencer_private_keys[count.index]
         },
+        {
+          name  = "SEQ_SKIP_SUBMIT_PROOFS"
+          value = tostring(var.SEQ_SKIP_SUBMIT_PROOFS)
+        },
         {
           name  = "ROLLUP_CONTRACT_ADDRESS"
           value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
index fe6b5c88ec84..a017135f0b68 100644
--- a/yarn-project/aztec/terraform/node/variables.tf
+++ b/yarn-project/aztec/terraform/node/variables.tf
@@ -58,6 +58,11 @@ variable "SEQ_MIN_SECONDS_BETWEEN_BLOCKS" {
   default = 30
 }
 
+variable "SEQ_SKIP_SUBMIT_PROOFS" {
+  type    = bool
+  default = true
+}
+
 variable "P2P_MIN_PEERS" {
   type    = string
   default = 5
diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
new file mode 100644
index 000000000000..da0779133049
--- /dev/null
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -0,0 +1,378 @@
+terraform {
+  backend "s3" {
+    bucket = "aztec-terraform"
+    region = "eu-west-2"
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "3.74.2"
+    }
+  }
+}
+
+# Define provider and region
+provider "aws" {
+  region = "eu-west-2"
+}
+
+data "terraform_remote_state" "setup_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "setup/setup-iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "aztec2_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "aztec2/iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "aztec-network_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "aztec-network/iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "l1_contracts" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "${var.DEPLOY_TAG}/l1-contracts"
+    region = "eu-west-2"
+  }
+}
+
+# Compute local variables
+locals {
+  prover_private_keys   = var.PROVER_PRIVATE_KEYS
+  node_p2p_private_keys = var.NODE_P2P_PRIVATE_KEYS
+  node_count            = length(local.prover_private_keys)
+  data_dir              = "/usr/src/yarn-project/aztec"
+}
+
+output "node_count" {
+  value = local.node_count
+}
+
+resource "aws_cloudwatch_log_group" "aztec-prover-node-log-group" {
+  count             = local.node_count
+  name              = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}"
+  retention_in_days = 14
+}
+
+resource "aws_service_discovery_service" "aztec-prover-node" {
+  count = local.node_count
+  name  = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+
+  health_check_custom_config {
+    failure_threshold = 1
+  }
+
+  dns_config {
+    namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id
+
+    dns_records {
+      ttl  = 60
+      type = "A"
+    }
+
+    dns_records {
+      ttl  = 60
+      type = "SRV"
+    }
+
+    routing_policy = "MULTIVALUE"
+  }
+
+  # Terraform just fails if this resource changes and you have registered instances.
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/../servicediscovery-drain.sh ${self.id}"
+  }
+}
+
+# Configure an EFS filesystem.
+resource "aws_efs_file_system" "prover_node_data_store" {
+  creation_token                  = "${var.DEPLOY_TAG}-prover-node-data"
+  throughput_mode                 = "provisioned"
+  provisioned_throughput_in_mibps = 20
+
+  tags = {
+    Name = "${var.DEPLOY_TAG}-prover-node-data"
+  }
+
+  lifecycle_policy {
+    transition_to_ia = "AFTER_14_DAYS"
+  }
+}
+
+resource "aws_efs_mount_target" "public_az1" {
+  file_system_id  = aws_efs_file_system.prover_node_data_store.id
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
+}
+
+resource "aws_efs_mount_target" "public_az2" {
+  file_system_id  = aws_efs_file_system.prover_node_data_store.id
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
+}
+
+# Define task definitions for each node.
+resource "aws_ecs_task_definition" "aztec-prover-node" {
+  count                    = local.node_count
+  family                   = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  requires_compatibilities = ["FARGATE"]
+  network_mode             = "awsvpc"
+  cpu                      = "2048"
+  memory                   = "4096"
+  execution_role_arn       = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn
+  task_role_arn            = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn
+
+  volume {
+    name = "efs-data-store"
+    efs_volume_configuration {
+      root_directory = "/"
+      file_system_id = aws_efs_file_system.prover_node_data_store.id
+    }
+  }
+
+  container_definitions = jsonencode([
+    {
+      name      = "init-container"
+      image     = "amazonlinux:latest"
+      essential = false
+      command   = ["sh", "-c", "mkdir -p ${local.data_dir}/prover_node_${count.index + 1}/data ${local.data_dir}/prover_node_${count.index + 1}/temp"]
+      mountPoints = [
+        {
+          containerPath = local.data_dir
+          sourceVolume  = "efs-data-store"
+        }
+      ]
+    },
+    {
+      name              = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+      image             = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}"
+      command           = ["start", "--prover-node", "--archiver"]
+      essential         = true
+      memoryReservation = 3776
+      portMappings = [
+        {
+          containerPort = 80
+        },
+        {
+          containerPort = var.NODE_P2P_TCP_PORT + count.index
+          protocol      = "tcp"
+        },
+        {
+          containerPort = var.NODE_P2P_UDP_PORT + count.index
+          protocol      = "udp"
+        }
+      ]
+      environment = [
+        // General
+        { name = "NODE_ENV", value = "production" },
+        { name = "LOG_LEVEL", value = "info" },
+        { name = "DEBUG", value = "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" },
+        { name = "DEPLOY_TAG", value = var.DEPLOY_TAG },
+        { name = "NETWORK_NAME", value = "${var.DEPLOY_TAG}" },
+        { name = "ETHEREUM_HOST", value = "https://${var.DEPLOY_TAG}-mainnet-fork.aztec.network:8545/${var.API_KEY}" },
+        { name = "L1_CHAIN_ID", value = var.L1_CHAIN_ID },
+        { name = "DATA_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/data" },
+        { name = "DEPLOY_AZTEC_CONTRACTS", value = "false" },
+
+        // API
+        { name = "AZTEC_PORT", value = "80" },
+        { name = "API_KEY", value = var.API_KEY },
+        { name = "API_PREFIX", value = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}" },
+
+        // Archiver
+        { name = "ARCHIVER_POLLING_INTERVAL", value = "10000" },
+
+        // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection)
+        { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
+
+        // Prover
+        { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] },
+        { name = "PROVER_AGENT_ENABLED", value = "false" },
+        { name = "PROVER_AGENT_CONCURRENCY", value = "0" },
+        { name = "PROVER_REAL_PROOFS", value = tostring(var.PROVING_ENABLED) },
+        { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" },
+        { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" },
+
+        // Metrics
+        { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" },
+        { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" },
+
+        // L1 addresses
+        { name = "ROLLUP_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address },
+        { name = "INBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.inbox_contract_address },
+        { name = "OUTBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.outbox_contract_address },
+        { name = "REGISTRY_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.registry_contract_address },
+        { name = "AVAILABILITY_ORACLE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.availability_oracle_contract_address },
+        { name = "FEE_JUICE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.fee_juice_contract_address },
+        { name = "FEE_JUICE_PORTAL_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.FEE_JUICE_PORTAL_CONTRACT_ADDRESS },
+
+        // P2P (disabled)
+        { name = "P2P_ENABLED", value = tostring(var.P2P_ENABLED) },
+        { name = "BOOTSTRAP_NODES", value = var.BOOTSTRAP_NODES },
+        { name = "PEER_ID_PRIVATE_KEY", value = local.node_p2p_private_keys[count.index] },
+        { name = "P2P_TCP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_TCP_PORT + count.index}" },
+        { name = "P2P_UDP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_UDP_PORT + count.index}" },
+        { name = "P2P_TCP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_TCP_PORT + count.index}" },
+        { name = "P2P_UDP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_UDP_PORT + count.index}" },
+        { name = "P2P_QUERY_FOR_IP", value = "true" },
+        { name = "P2P_MIN_PEERS", value = var.P2P_MIN_PEERS },
+        { name = "P2P_MAX_PEERS", value = var.P2P_MAX_PEERS },
+        { name = "P2P_BLOCK_CHECK_INTERVAL_MS", value = "1000" },
+        { name = "P2P_PEER_CHECK_INTERVAL_MS", value = "2000" },
+        { name = "P2P_TX_POOL_KEEP_PROVEN_FOR", value = tostring(var.P2P_TX_POOL_KEEP_PROVEN_FOR) },
+      ]
+      mountPoints = [
+        {
+          containerPath = "${local.data_dir}/prover_node_${count.index + 1}"
+          sourceVolume  = "efs-data-store"
+        }
+      ]
+      dependsOn = [
+        {
+          containerName = "init-container"
+          condition     = "COMPLETE"
+        }
+      ]
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"         = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}"
+          "awslogs-region"        = "eu-west-2"
+          "awslogs-stream-prefix" = "ecs"
+        }
+      }
+  }])
+}
+
+resource "aws_ecs_service" "aztec-prover-node" {
+  count                              = local.node_count
+  name                               = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  cluster                            = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id
+  launch_type                        = "FARGATE"
+  desired_count                      = 1
+  deployment_maximum_percent         = 100
+  deployment_minimum_healthy_percent = 0
+  platform_version                   = "1.4.0"
+  force_new_deployment               = true
+
+
+  network_configuration {
+    assign_public_ip = true
+    subnets = [
+      data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+    ]
+    security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+  }
+
+  load_balancer {
+    target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn
+    container_name   = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+    container_port   = 80
+  }
+
+  service_registries {
+    registry_arn   = aws_service_discovery_service.aztec-prover-node[count.index].arn
+    container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+    container_port = 80
+  }
+
+  task_definition = aws_ecs_task_definition.aztec-prover-node[count.index].family
+}
+
+# Configure ALB to route /aztec-prover-node to server.
+resource "aws_alb_target_group" "aztec-prover-node-http" {
+  count                = local.node_count
+  name                 = "${var.DEPLOY_TAG}-node-${count.index + 1}-http-target"
+  port                 = 80
+  protocol             = "HTTP"
+  target_type          = "ip"
+  vpc_id               = data.terraform_remote_state.setup_iac.outputs.vpc_id
+  deregistration_delay = 5
+
+  health_check {
+    path                = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}/status"
+    matcher             = "200"
+    interval            = 10
+    healthy_threshold   = 2
+    unhealthy_threshold = 5
+    timeout             = 5
+  }
+
+  tags = {
+    name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  }
+}
+
+resource "aws_lb_listener_rule" "api" {
+  count        = local.node_count
+  listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn
+  priority     = var.PROVER_NODE_LB_RULE_PRIORITY + count.index
+
+  action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn
+  }
+
+  condition {
+    path_pattern {
+      values = ["/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}*"]
+    }
+  }
+}
+
+resource "aws_security_group_rule" "allow-node-tcp-in" {
+  count             = local.node_count
+  type              = "ingress"
+  from_port         = var.NODE_P2P_TCP_PORT + count.index
+  to_port           = var.NODE_P2P_TCP_PORT + count.index
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-tcp-out" {
+  count             = local.node_count
+  type              = "egress"
+  from_port         = var.NODE_P2P_TCP_PORT + count.index
+  to_port           = var.NODE_P2P_TCP_PORT + count.index
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-udp-in" {
+  count             = local.node_count
+  type              = "ingress"
+  from_port         = var.NODE_P2P_UDP_PORT
+  to_port           = var.NODE_P2P_UDP_PORT + count.index
+  protocol          = "udp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-udp-out" {
+  count             = local.node_count
+  type              = "egress"
+  from_port         = var.NODE_P2P_UDP_PORT
+  to_port           = var.NODE_P2P_UDP_PORT + count.index
+  protocol          = "udp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
diff --git a/yarn-project/aztec/terraform/prover-node/variables.tf b/yarn-project/aztec/terraform/prover-node/variables.tf
new file mode 100644
index 000000000000..4f0fbb0ac7f1
--- /dev/null
+++ b/yarn-project/aztec/terraform/prover-node/variables.tf
@@ -0,0 +1,74 @@
+variable "DEPLOY_TAG" {
+  type = string
+}
+
+variable "IMAGE_TAG" {
+  type    = string
+  default = "latest"
+}
+
+variable "API_KEY" {
+  type = string
+}
+
+variable "PROVER_PRIVATE_KEYS" {
+  type = list(string)
+}
+
+variable "NODE_P2P_PRIVATE_KEYS" {
+  type = list(string)
+}
+
+variable "L1_CHAIN_ID" {
+  type    = string
+  default = 677692
+}
+
+variable "NODE_P2P_TCP_PORT" {
+  type    = number
+  default = 40000
+}
+
+variable "NODE_P2P_UDP_PORT" {
+  type    = number
+  default = 45000
+}
+
+variable "DOCKERHUB_ACCOUNT" {
+  type = string
+}
+
+variable "P2P_MIN_PEERS" {
+  type    = string
+  default = 5
+}
+
+variable "P2P_MAX_PEERS" {
+  type    = string
+  default = 100
+}
+
+variable "P2P_ENABLED" {
+  type    = bool
+  default = false
+}
+
+variable "P2P_TX_POOL_KEEP_PROVEN_FOR" {
+  type    = number
+  default = 64
+}
+
+variable "PROVING_ENABLED" {
+  type    = bool
+  default = false
+}
+
+variable "BOOTSTRAP_NODES" {
+  type    = string
+  default = ""
+}
+
+variable "PROVER_NODE_LB_RULE_PRIORITY" {
+  type    = number
+  default = 7000
+}
diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf
index 99d4d770c67b..25056aa95b89 100644
--- a/yarn-project/aztec/terraform/prover/main.tf
+++ b/yarn-project/aztec/terraform/prover/main.tf
@@ -43,17 +43,17 @@ data "terraform_remote_state" "aztec-network_iac" {
   }
 }
 
-data "terraform_remote_state" "aztec-network_node" {
+data "terraform_remote_state" "aztec-network_prover-node" {
   backend = "s3"
   config = {
     bucket = "aztec-terraform"
-    key    = "${var.DEPLOY_TAG}/aztec-node"
+    key    = "${var.DEPLOY_TAG}/aztec-prover-node"
     region = "eu-west-2"
   }
 }
 
 locals {
-  node_count        = data.terraform_remote_state.aztec-network_node.outputs.node_count
+  node_count        = data.terraform_remote_state.aztec-network_prover-node.outputs.node_count
   agents_per_prover = var.AGENTS_PER_PROVER
 }
 
@@ -259,7 +259,7 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
       },
       {
         "name": "AZTEC_NODE_URL",
-        "value": "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}"
+        "value": "http://${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}"
       },
       {
         "name": "PROVER_AGENT_ENABLED",

From b839a9c9a716beaf12f6a9f8b27335036a4d19bb Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 14:45:36 -0300
Subject: [PATCH 02/10] Add prover-node to devnet-deploys

---
 .github/workflows/devnet-deploys.yml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml
index b6d4509538f8..4f82bcdd2c44 100644
--- a/.github/workflows/devnet-deploys.yml
+++ b/.github/workflows/devnet-deploys.yml
@@ -13,8 +13,9 @@ concurrency:
 
 # Anvil Accounts. Anvil provides 10 pre-funded accounts for the mnemonic we have specified in FORK_MNEMONIC. We are using:
 # 1. The first account (index 0) is used in SEQ_1_PUBLISHER_PRIVATE_KEY
-# 2. The 9th account (index 8) is used in this workflow for deploying contracts etc
-# 3. The 10th account (index 9) is used by the deployed faucet
+# 2. The 3rd account (index 2) is used in PROVER_1_PUBLISHER_PRIVATE_KEY
+# 3. The 9th account (index 8) is used in this workflow for deploying contracts etc
+# 4. The 10th account (index 9) is used by the deployed faucet
 # TODO: Convert all this so we take the provided mnemonic and derive the keys from the above indices
 env:
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -47,6 +48,10 @@ env:
   TF_VAR_NODE_P2P_UDP_PORT: 45000
   TF_VAR_NODE_LB_RULE_PRIORITY: 500
 
+  # Prover Node
+  TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: 6000
+  TF_VAR_PROVER_PRIVATE_KEYS: '["${{ secrets.PROVER_1_PUBLISHER_PRIVATE_KEY }}"]'
+
   # Anvil
   TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }}
   TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
@@ -97,6 +102,7 @@ jobs:
       node_udp_range_start: ${{ steps.set_network_vars.outputs.node_udp_range_start }}
       node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.node_lb_priority_range_start }}
       pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }}
+      prover_node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.prover_node_lb_priority_range_start }}
       faucet_lb_priority: ${{ steps.set_network_vars.outputs.faucet_lb_priority }}
       bot_no_wait: ${{ steps.set_network_vars.outputs.bot_no_wait }}
     steps:
@@ -116,6 +122,7 @@ jobs:
               echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4100" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6100" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
           elif [ "$BRANCH_NAME" = "provernet" ]
@@ -129,6 +136,7 @@ jobs:
               echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4200" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6200" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=602" >> $GITHUB_OUTPUT
               echo "bot_no_wait=true" >> $GITHUB_OUTPUT
           elif [ "$BRANCH_NAME" = "alphanet" ]
@@ -142,6 +150,7 @@ jobs:
               echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4000" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6000" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=600" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
           else
@@ -410,6 +419,7 @@ jobs:
       TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
     steps:
       - uses: actions/checkout@v4
@@ -489,6 +499,12 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store"
 
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve
+
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
         run: |
@@ -610,6 +626,7 @@ jobs:
       TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
       TF_VAR_PROVING_ENABLED: true
       TF_VAR_BOT_NO_START: false
@@ -642,6 +659,12 @@ jobs:
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
           terraform apply -input=false -auto-approve
 
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve
+
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
         run: |

From 3c797e5e34e8b7979846e779d5e957decd1f70db Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 15:30:48 -0300
Subject: [PATCH 03/10] See #7858

---
 yarn-project/aztec/terraform/prover-node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
index da0779133049..bc24b2adc72a 100644
--- a/yarn-project/aztec/terraform/prover-node/main.tf
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -200,7 +200,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" {
         { name = "ARCHIVER_POLLING_INTERVAL", value = "10000" },
 
         // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection)
-        { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
+        { name = "TX_PROVIDER_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
 
         // Prover
         { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] },

From 437302fa90b8d7375f9de4f5e0f8abc1232fb4c6 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 11:54:57 -0300
Subject: [PATCH 04/10] chore: Terraform template for prover-node

Adds a new tf template for prover-node. Updates the node so it no longer
submits proofs, and updates the prover agent so it connects to the
prover-node instead of the node.
---
 yarn-project/aztec/terraform/node/main.tf     |   4 +
 .../aztec/terraform/node/variables.tf         |   5 +
 .../aztec/terraform/prover-node/main.tf       | 378 ++++++++++++++++++
 .../aztec/terraform/prover-node/variables.tf  |  74 ++++
 yarn-project/aztec/terraform/prover/main.tf   |   8 +-
 5 files changed, 465 insertions(+), 4 deletions(-)
 create mode 100644 yarn-project/aztec/terraform/prover-node/main.tf
 create mode 100644 yarn-project/aztec/terraform/prover-node/variables.tf

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
index e64f28a961dd..edfa18cdfa41 100644
--- a/yarn-project/aztec/terraform/node/main.tf
+++ b/yarn-project/aztec/terraform/node/main.tf
@@ -236,6 +236,10 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "SEQ_PUBLISHER_PRIVATE_KEY"
           value = local.sequencer_private_keys[count.index]
         },
+        {
+          name  = "SEQ_SKIP_SUBMIT_PROOFS"
+          value = tostring(var.SEQ_SKIP_SUBMIT_PROOFS)
+        },
         {
           name  = "ROLLUP_CONTRACT_ADDRESS"
           value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
index fe6b5c88ec84..a017135f0b68 100644
--- a/yarn-project/aztec/terraform/node/variables.tf
+++ b/yarn-project/aztec/terraform/node/variables.tf
@@ -58,6 +58,11 @@ variable "SEQ_MIN_SECONDS_BETWEEN_BLOCKS" {
   default = 30
 }
 
+variable "SEQ_SKIP_SUBMIT_PROOFS" {
+  type    = bool
+  default = true
+}
+
 variable "P2P_MIN_PEERS" {
   type    = string
   default = 5
diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
new file mode 100644
index 000000000000..da0779133049
--- /dev/null
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -0,0 +1,378 @@
+terraform {
+  backend "s3" {
+    bucket = "aztec-terraform"
+    region = "eu-west-2"
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "3.74.2"
+    }
+  }
+}
+
+# Define provider and region
+provider "aws" {
+  region = "eu-west-2"
+}
+
+data "terraform_remote_state" "setup_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "setup/setup-iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "aztec2_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "aztec2/iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "aztec-network_iac" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "aztec-network/iac"
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "l1_contracts" {
+  backend = "s3"
+  config = {
+    bucket = "aztec-terraform"
+    key    = "${var.DEPLOY_TAG}/l1-contracts"
+    region = "eu-west-2"
+  }
+}
+
+# Compute local variables
+locals {
+  prover_private_keys   = var.PROVER_PRIVATE_KEYS
+  node_p2p_private_keys = var.NODE_P2P_PRIVATE_KEYS
+  node_count            = length(local.prover_private_keys)
+  data_dir              = "/usr/src/yarn-project/aztec"
+}
+
+output "node_count" {
+  value = local.node_count
+}
+
+resource "aws_cloudwatch_log_group" "aztec-prover-node-log-group" {
+  count             = local.node_count
+  name              = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}"
+  retention_in_days = 14
+}
+
+resource "aws_service_discovery_service" "aztec-prover-node" {
+  count = local.node_count
+  name  = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+
+  health_check_custom_config {
+    failure_threshold = 1
+  }
+
+  dns_config {
+    namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id
+
+    dns_records {
+      ttl  = 60
+      type = "A"
+    }
+
+    dns_records {
+      ttl  = 60
+      type = "SRV"
+    }
+
+    routing_policy = "MULTIVALUE"
+  }
+
+  # Terraform just fails if this resource changes and you have registered instances.
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/../servicediscovery-drain.sh ${self.id}"
+  }
+}
+
+# Configure an EFS filesystem.
+resource "aws_efs_file_system" "prover_node_data_store" {
+  creation_token                  = "${var.DEPLOY_TAG}-prover-node-data"
+  throughput_mode                 = "provisioned"
+  provisioned_throughput_in_mibps = 20
+
+  tags = {
+    Name = "${var.DEPLOY_TAG}-prover-node-data"
+  }
+
+  lifecycle_policy {
+    transition_to_ia = "AFTER_14_DAYS"
+  }
+}
+
+resource "aws_efs_mount_target" "public_az1" {
+  file_system_id  = aws_efs_file_system.prover_node_data_store.id
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
+}
+
+resource "aws_efs_mount_target" "public_az2" {
+  file_system_id  = aws_efs_file_system.prover_node_data_store.id
+  subnet_id       = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
+  security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id]
+}
+
+# Define task definitions for each node.
+resource "aws_ecs_task_definition" "aztec-prover-node" {
+  count                    = local.node_count
+  family                   = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  requires_compatibilities = ["FARGATE"]
+  network_mode             = "awsvpc"
+  cpu                      = "2048"
+  memory                   = "4096"
+  execution_role_arn       = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn
+  task_role_arn            = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn
+
+  volume {
+    name = "efs-data-store"
+    efs_volume_configuration {
+      root_directory = "/"
+      file_system_id = aws_efs_file_system.prover_node_data_store.id
+    }
+  }
+
+  container_definitions = jsonencode([
+    {
+      name      = "init-container"
+      image     = "amazonlinux:latest"
+      essential = false
+      command   = ["sh", "-c", "mkdir -p ${local.data_dir}/prover_node_${count.index + 1}/data ${local.data_dir}/prover_node_${count.index + 1}/temp"]
+      mountPoints = [
+        {
+          containerPath = local.data_dir
+          sourceVolume  = "efs-data-store"
+        }
+      ]
+    },
+    {
+      name              = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+      image             = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}"
+      command           = ["start", "--prover-node", "--archiver"]
+      essential         = true
+      memoryReservation = 3776
+      portMappings = [
+        {
+          containerPort = 80
+        },
+        {
+          containerPort = var.NODE_P2P_TCP_PORT + count.index
+          protocol      = "tcp"
+        },
+        {
+          containerPort = var.NODE_P2P_UDP_PORT + count.index
+          protocol      = "udp"
+        }
+      ]
+      environment = [
+        // General
+        { name = "NODE_ENV", value = "production" },
+        { name = "LOG_LEVEL", value = "info" },
+        { name = "DEBUG", value = "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" },
+        { name = "DEPLOY_TAG", value = var.DEPLOY_TAG },
+        { name = "NETWORK_NAME", value = "${var.DEPLOY_TAG}" },
+        { name = "ETHEREUM_HOST", value = "https://${var.DEPLOY_TAG}-mainnet-fork.aztec.network:8545/${var.API_KEY}" },
+        { name = "L1_CHAIN_ID", value = var.L1_CHAIN_ID },
+        { name = "DATA_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/data" },
+        { name = "DEPLOY_AZTEC_CONTRACTS", value = "false" },
+
+        // API
+        { name = "AZTEC_PORT", value = "80" },
+        { name = "API_KEY", value = var.API_KEY },
+        { name = "API_PREFIX", value = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}" },
+
+        // Archiver
+        { name = "ARCHIVER_POLLING_INTERVAL", value = "10000" },
+
+        // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection)
+        { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
+
+        // Prover
+        { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] },
+        { name = "PROVER_AGENT_ENABLED", value = "false" },
+        { name = "PROVER_AGENT_CONCURRENCY", value = "0" },
+        { name = "PROVER_REAL_PROOFS", value = tostring(var.PROVING_ENABLED) },
+        { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" },
+        { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" },
+
+        // Metrics
+        { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" },
+        { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" },
+
+        // L1 addresses
+        { name = "ROLLUP_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address },
+        { name = "INBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.inbox_contract_address },
+        { name = "OUTBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.outbox_contract_address },
+        { name = "REGISTRY_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.registry_contract_address },
+        { name = "AVAILABILITY_ORACLE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.availability_oracle_contract_address },
+        { name = "FEE_JUICE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.fee_juice_contract_address },
+        { name = "FEE_JUICE_PORTAL_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.FEE_JUICE_PORTAL_CONTRACT_ADDRESS },
+
+        // P2P (disabled)
+        { name = "P2P_ENABLED", value = tostring(var.P2P_ENABLED) },
+        { name = "BOOTSTRAP_NODES", value = var.BOOTSTRAP_NODES },
+        { name = "PEER_ID_PRIVATE_KEY", value = local.node_p2p_private_keys[count.index] },
+        { name = "P2P_TCP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_TCP_PORT + count.index}" },
+        { name = "P2P_UDP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_UDP_PORT + count.index}" },
+        { name = "P2P_TCP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_TCP_PORT + count.index}" },
+        { name = "P2P_UDP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_UDP_PORT + count.index}" },
+        { name = "P2P_QUERY_FOR_IP", value = "true" },
+        { name = "P2P_MIN_PEERS", value = var.P2P_MIN_PEERS },
+        { name = "P2P_MAX_PEERS", value = var.P2P_MAX_PEERS },
+        { name = "P2P_BLOCK_CHECK_INTERVAL_MS", value = "1000" },
+        { name = "P2P_PEER_CHECK_INTERVAL_MS", value = "2000" },
+        { name = "P2P_TX_POOL_KEEP_PROVEN_FOR", value = tostring(var.P2P_TX_POOL_KEEP_PROVEN_FOR) },
+      ]
+      mountPoints = [
+        {
+          containerPath = "${local.data_dir}/prover_node_${count.index + 1}"
+          sourceVolume  = "efs-data-store"
+        }
+      ]
+      dependsOn = [
+        {
+          containerName = "init-container"
+          condition     = "COMPLETE"
+        }
+      ]
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"         = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}"
+          "awslogs-region"        = "eu-west-2"
+          "awslogs-stream-prefix" = "ecs"
+        }
+      }
+  }])
+}
+
+resource "aws_ecs_service" "aztec-prover-node" {
+  count                              = local.node_count
+  name                               = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  cluster                            = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id
+  launch_type                        = "FARGATE"
+  desired_count                      = 1
+  deployment_maximum_percent         = 100
+  deployment_minimum_healthy_percent = 0
+  platform_version                   = "1.4.0"
+  force_new_deployment               = true
+
+
+  network_configuration {
+    assign_public_ip = true
+    subnets = [
+      data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+    ]
+    security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
+  }
+
+  load_balancer {
+    target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn
+    container_name   = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+    container_port   = 80
+  }
+
+  service_registries {
+    registry_arn   = aws_service_discovery_service.aztec-prover-node[count.index].arn
+    container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+    container_port = 80
+  }
+
+  task_definition = aws_ecs_task_definition.aztec-prover-node[count.index].family
+}
+
+# Configure ALB to route /aztec-prover-node to server.
+resource "aws_alb_target_group" "aztec-prover-node-http" {
+  count                = local.node_count
+  name                 = "${var.DEPLOY_TAG}-node-${count.index + 1}-http-target"
+  port                 = 80
+  protocol             = "HTTP"
+  target_type          = "ip"
+  vpc_id               = data.terraform_remote_state.setup_iac.outputs.vpc_id
+  deregistration_delay = 5
+
+  health_check {
+    path                = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}/status"
+    matcher             = "200"
+    interval            = 10
+    healthy_threshold   = 2
+    unhealthy_threshold = 5
+    timeout             = 5
+  }
+
+  tags = {
+    name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}"
+  }
+}
+
+resource "aws_lb_listener_rule" "api" {
+  count        = local.node_count
+  listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn
+  priority     = var.PROVER_NODE_LB_RULE_PRIORITY + count.index
+
+  action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn
+  }
+
+  condition {
+    path_pattern {
+      values = ["/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}*"]
+    }
+  }
+}
+
+resource "aws_security_group_rule" "allow-node-tcp-in" {
+  count             = local.node_count
+  type              = "ingress"
+  from_port         = var.NODE_P2P_TCP_PORT + count.index
+  to_port           = var.NODE_P2P_TCP_PORT + count.index
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-tcp-out" {
+  count             = local.node_count
+  type              = "egress"
+  from_port         = var.NODE_P2P_TCP_PORT + count.index
+  to_port           = var.NODE_P2P_TCP_PORT + count.index
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-udp-in" {
+  count             = local.node_count
+  type              = "ingress"
+  from_port         = var.NODE_P2P_UDP_PORT
+  to_port           = var.NODE_P2P_UDP_PORT + count.index
+  protocol          = "udp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
+
+resource "aws_security_group_rule" "allow-node-udp-out" {
+  count             = local.node_count
+  type              = "egress"
+  from_port         = var.NODE_P2P_UDP_PORT
+  to_port           = var.NODE_P2P_UDP_PORT + count.index
+  protocol          = "udp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+}
diff --git a/yarn-project/aztec/terraform/prover-node/variables.tf b/yarn-project/aztec/terraform/prover-node/variables.tf
new file mode 100644
index 000000000000..4f0fbb0ac7f1
--- /dev/null
+++ b/yarn-project/aztec/terraform/prover-node/variables.tf
@@ -0,0 +1,74 @@
+variable "DEPLOY_TAG" {
+  type = string
+}
+
+variable "IMAGE_TAG" {
+  type    = string
+  default = "latest"
+}
+
+variable "API_KEY" {
+  type = string
+}
+
+variable "PROVER_PRIVATE_KEYS" {
+  type = list(string)
+}
+
+variable "NODE_P2P_PRIVATE_KEYS" {
+  type = list(string)
+}
+
+variable "L1_CHAIN_ID" {
+  type    = string
+  default = 677692
+}
+
+variable "NODE_P2P_TCP_PORT" {
+  type    = number
+  default = 40000
+}
+
+variable "NODE_P2P_UDP_PORT" {
+  type    = number
+  default = 45000
+}
+
+variable "DOCKERHUB_ACCOUNT" {
+  type = string
+}
+
+variable "P2P_MIN_PEERS" {
+  type    = string
+  default = 5
+}
+
+variable "P2P_MAX_PEERS" {
+  type    = string
+  default = 100
+}
+
+variable "P2P_ENABLED" {
+  type    = bool
+  default = false
+}
+
+variable "P2P_TX_POOL_KEEP_PROVEN_FOR" {
+  type    = number
+  default = 64
+}
+
+variable "PROVING_ENABLED" {
+  type    = bool
+  default = false
+}
+
+variable "BOOTSTRAP_NODES" {
+  type    = string
+  default = ""
+}
+
+variable "PROVER_NODE_LB_RULE_PRIORITY" {
+  type    = number
+  default = 7000
+}
diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf
index 99d4d770c67b..25056aa95b89 100644
--- a/yarn-project/aztec/terraform/prover/main.tf
+++ b/yarn-project/aztec/terraform/prover/main.tf
@@ -43,17 +43,17 @@ data "terraform_remote_state" "aztec-network_iac" {
   }
 }
 
-data "terraform_remote_state" "aztec-network_node" {
+data "terraform_remote_state" "aztec-network_prover-node" {
   backend = "s3"
   config = {
     bucket = "aztec-terraform"
-    key    = "${var.DEPLOY_TAG}/aztec-node"
+    key    = "${var.DEPLOY_TAG}/aztec-prover-node"
     region = "eu-west-2"
   }
 }
 
 locals {
-  node_count        = data.terraform_remote_state.aztec-network_node.outputs.node_count
+  node_count        = data.terraform_remote_state.aztec-network_prover-node.outputs.node_count
   agents_per_prover = var.AGENTS_PER_PROVER
 }
 
@@ -259,7 +259,7 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" {
       },
       {
         "name": "AZTEC_NODE_URL",
-        "value": "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}"
+        "value": "http://${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}"
       },
       {
         "name": "PROVER_AGENT_ENABLED",

From dacfd0df284d12a4c722177d3ea12b4b8c04d894 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 14:45:36 -0300
Subject: [PATCH 05/10] Add prover-node to devnet-deploys

---
 .github/workflows/devnet-deploys.yml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml
index 11a391294987..244187101eea 100644
--- a/.github/workflows/devnet-deploys.yml
+++ b/.github/workflows/devnet-deploys.yml
@@ -17,8 +17,9 @@ concurrency:
 
 # Anvil Accounts. Anvil provides 10 pre-funded accounts for the mnemonic we have specified in FORK_MNEMONIC. We are using:
 # 1. The first account (index 0) is used in SEQ_1_PUBLISHER_PRIVATE_KEY
-# 2. The 9th account (index 8) is used in this workflow for deploying contracts etc
-# 3. The 10th account (index 9) is used by the deployed faucet
+# 2. The 3rd account (index 2) is used in PROVER_1_PUBLISHER_PRIVATE_KEY
+# 3. The 9th account (index 8) is used in this workflow for deploying contracts etc
+# 4. The 10th account (index 9) is used by the deployed faucet
 # TODO: Convert all this so we take the provided mnemonic and derive the keys from the above indices
 env:
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -51,6 +52,10 @@ env:
   TF_VAR_NODE_P2P_UDP_PORT: 45000
   TF_VAR_NODE_LB_RULE_PRIORITY: 500
 
+  # Prover Node
+  TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: 6000
+  TF_VAR_PROVER_PRIVATE_KEYS: '["${{ secrets.PROVER_1_PUBLISHER_PRIVATE_KEY }}"]'
+
   # Anvil
   TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }}
   TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
@@ -101,6 +106,7 @@ jobs:
       node_udp_range_start: ${{ steps.set_network_vars.outputs.node_udp_range_start }}
       node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.node_lb_priority_range_start }}
       pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }}
+      prover_node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.prover_node_lb_priority_range_start }}
       faucet_lb_priority: ${{ steps.set_network_vars.outputs.faucet_lb_priority }}
       bot_no_wait: ${{ steps.set_network_vars.outputs.bot_no_wait }}
       max_txs_per_block: ${{ steps.set_network_vars.outputs.max_txs_per_block }}
@@ -121,6 +127,7 @@ jobs:
               echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4100" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6100" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
@@ -135,6 +142,7 @@ jobs:
               echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4200" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6200" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=602" >> $GITHUB_OUTPUT
               echo "bot_no_wait=true" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=4" >> $GITHUB_OUTPUT
@@ -149,6 +157,7 @@ jobs:
               echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4000" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6000" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=600" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
@@ -348,6 +357,7 @@ jobs:
       TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
       TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
     steps:
@@ -428,6 +438,12 @@ jobs:
         run: |
           terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store"
 
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve
+
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
         run: |
@@ -549,6 +565,7 @@ jobs:
       TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
       TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
       TF_VAR_PROVING_ENABLED: true
@@ -582,6 +599,12 @@ jobs:
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
           terraform apply -input=false -auto-approve
 
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve
+
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
         run: |

From ff12bdc95f0618cd26ff34d53d59671a0a1da4f5 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Thu, 8 Aug 2024 15:30:48 -0300
Subject: [PATCH 06/10] See #7858

---
 yarn-project/aztec/terraform/prover-node/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
index da0779133049..bc24b2adc72a 100644
--- a/yarn-project/aztec/terraform/prover-node/main.tf
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -200,7 +200,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" {
         { name = "ARCHIVER_POLLING_INTERVAL", value = "10000" },
 
         // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection)
-        { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
+        { name = "TX_PROVIDER_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" },
 
         // Prover
         { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] },

From e13533d1e00b92aff986098e1e3c9f0d47c84d83 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Tue, 13 Aug 2024 19:51:15 -0300
Subject: [PATCH 07/10] Add is_dev_net flag

---
 yarn-project/aztec/terraform/node/main.tf        | 4 ++++
 yarn-project/aztec/terraform/prover-node/main.tf | 1 +
 2 files changed, 5 insertions(+)

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
index edfa18cdfa41..3c204d9558ce 100644
--- a/yarn-project/aztec/terraform/node/main.tf
+++ b/yarn-project/aztec/terraform/node/main.tf
@@ -208,6 +208,10 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "DATA_DIRECTORY"
           value = "${local.data_dir}/node_${count.index + 1}/data"
         },
+        {
+          name  = "IS_DEV_NET"
+          value = "true"
+        },
         {
           name  = "ARCHIVER_POLLING_INTERVAL"
           value = "10000"
diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
index bc24b2adc72a..ef2efbd26d21 100644
--- a/yarn-project/aztec/terraform/prover-node/main.tf
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -190,6 +190,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" {
         { name = "L1_CHAIN_ID", value = var.L1_CHAIN_ID },
         { name = "DATA_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/data" },
         { name = "DEPLOY_AZTEC_CONTRACTS", value = "false" },
+        { name = "IS_DEV_NET", value = "true" },
 
         // API
         { name = "AZTEC_PORT", value = "80" },

From be5f3ec138dac6c9df1edb74fb57bb3048e9d102 Mon Sep 17 00:00:00 2001
From: Santiago Palladino <santiago@aztecprotocol.com>
Date: Tue, 13 Aug 2024 19:53:40 -0300
Subject: [PATCH 08/10] Remove seq_skip_submit_proofs env var and --prover flag
 from node cmd

---
 yarn-project/aztec/terraform/node/main.tf      | 6 +-----
 yarn-project/aztec/terraform/node/variables.tf | 5 -----
 yarn-project/foundation/src/config/env_var.ts  | 1 -
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
index 3c204d9558ce..4bbe91821693 100644
--- a/yarn-project/aztec/terraform/node/main.tf
+++ b/yarn-project/aztec/terraform/node/main.tf
@@ -163,7 +163,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
     {
       name              = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
       image             = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}"
-      command           = ["start", "--node", "--archiver", "--sequencer", "--prover"]
+      command           = ["start", "--node", "--archiver", "--sequencer"]
       essential         = true
       memoryReservation = 3776
       portMappings = [
@@ -240,10 +240,6 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "SEQ_PUBLISHER_PRIVATE_KEY"
           value = local.sequencer_private_keys[count.index]
         },
-        {
-          name  = "SEQ_SKIP_SUBMIT_PROOFS"
-          value = tostring(var.SEQ_SKIP_SUBMIT_PROOFS)
-        },
         {
           name  = "ROLLUP_CONTRACT_ADDRESS"
           value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
index a017135f0b68..fe6b5c88ec84 100644
--- a/yarn-project/aztec/terraform/node/variables.tf
+++ b/yarn-project/aztec/terraform/node/variables.tf
@@ -58,11 +58,6 @@ variable "SEQ_MIN_SECONDS_BETWEEN_BLOCKS" {
   default = 30
 }
 
-variable "SEQ_SKIP_SUBMIT_PROOFS" {
-  type    = bool
-  default = true
-}
-
 variable "P2P_MIN_PEERS" {
   type    = string
   default = 5
diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts
index 233e89328acf..d4383b4b64b1 100644
--- a/yarn-project/foundation/src/config/env_var.ts
+++ b/yarn-project/foundation/src/config/env_var.ts
@@ -57,7 +57,6 @@ export type EnvVar =
   | 'SEQ_ALLOWED_TEARDOWN_FN'
   | 'SEQ_MAX_BLOCK_SIZE_IN_BYTES'
   | 'ENFORCE_FEES'
-  | 'SEQ_SKIP_SUBMIT_PROOFS'
   | 'SEQ_PUBLISHER_PRIVATE_KEY'
   | 'SEQ_REQUIRED_CONFIRMATIONS'
   | 'SEQ_PUBLISH_RETRY_INTERVAL_MS'

From 353eda2effe21fb687f160875884467ea4dfead5 Mon Sep 17 00:00:00 2001
From: PhilWindle <philip.windle@gmail.com>
Date: Wed, 14 Aug 2024 08:52:24 +0000
Subject: [PATCH 09/10] Some more TF and deployment updates

---
 .github/workflows/devnet-deploys.yml          | 28 ++++---
 .../aztec/terraform/node/variables.tf         | 12 +--
 .../aztec/terraform/prover-node/main.tf       | 78 +++++++++----------
 .../aztec/terraform/prover-node/variables.tf  | 12 +--
 yarn-project/aztec/terraform/pxe/variables.tf |  3 +-
 5 files changed, 61 insertions(+), 72 deletions(-)

diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml
index 244187101eea..03383e3f480f 100644
--- a/.github/workflows/devnet-deploys.yml
+++ b/.github/workflows/devnet-deploys.yml
@@ -91,8 +91,6 @@ jobs:
     secrets: inherit
 
   # Set network specific variables as outputs from this job to be referenced in later jobs
-  # The only exception is the network api key which needs to be re-derived in every job as it is a secret
-  # Secrets can't be passed between jobs
   set-network:
     needs: setup
     runs-on: ${{ github.actor }}-x86
@@ -104,6 +102,8 @@ jobs:
       bot_interval: ${{ steps.set_network_vars.outputs.bot_interval }}
       node_tcp_range_start: ${{ steps.set_network_vars.outputs.node_tcp_range_start }}
       node_udp_range_start: ${{ steps.set_network_vars.outputs.node_udp_range_start }}
+      prover_node_tcp_range_start: ${{ steps.set_network_vars.outputs.prover_node_tcp_range_start }}
+      prover_node_udp_range_start: ${{ steps.set_network_vars.outputs.prover_node_udp_range_start }}
       node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.node_lb_priority_range_start }}
       pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }}
       prover_node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.prover_node_lb_priority_range_start }}
@@ -125,6 +125,8 @@ jobs:
               echo "bot_interval=30" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41100" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46100" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4100" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT
               echo "prover_node_lb_priority_range_start=6100" >> $GITHUB_OUTPUT
@@ -140,6 +142,8 @@ jobs:
               echo "bot_interval=300" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40200" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41200" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46200" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4200" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT
               echo "prover_node_lb_priority_range_start=6200" >> $GITHUB_OUTPUT
@@ -155,6 +159,8 @@ jobs:
               echo "bot_interval=30" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46000" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4000" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT
               echo "prover_node_lb_priority_range_start=6000" >> $GITHUB_OUTPUT
@@ -353,8 +359,6 @@ jobs:
       TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }}
       TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }}
       TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }}
-      TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }}
-      TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
       TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
@@ -428,21 +432,17 @@ jobs:
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/bot"
           terraform apply -input=false -auto-approve
 
-      - name: Init Aztec Node Terraform
-        working-directory: ./yarn-project/aztec/terraform/node
-        run: |
-          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
-
       - name: Deploy Aztec Nodes
         working-directory: ./yarn-project/aztec/terraform/node
         run: |
-          terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store"
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
+          terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store" -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}"
 
       - name: Deploy Aztec Prover Nodes
         working-directory: ./yarn-project/aztec/terraform/prover-node
         run: |
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
-          terraform apply -input=false -auto-approve
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}"
 
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
@@ -561,8 +561,6 @@ jobs:
       TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }}
       TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }}
       TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }}
-      TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }}
-      TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
       TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
@@ -597,13 +595,13 @@ jobs:
         run: |
           env
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
-          terraform apply -input=false -auto-approve
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}"
 
       - name: Deploy Aztec Prover Nodes
         working-directory: ./yarn-project/aztec/terraform/prover-node
         run: |
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
-          terraform apply -input=false -auto-approve
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}"
 
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
index a017135f0b68..0dcdfa4b24ff 100644
--- a/yarn-project/aztec/terraform/node/variables.tf
+++ b/yarn-project/aztec/terraform/node/variables.tf
@@ -20,18 +20,15 @@ variable "NODE_P2P_PRIVATE_KEYS" {
 }
 
 variable "L1_CHAIN_ID" {
-  type    = string
-  default = 677692
+  type = string
 }
 
 variable "NODE_P2P_TCP_PORT" {
-  type    = number
-  default = 40000
+  type = number
 }
 
 variable "NODE_P2P_UDP_PORT" {
-  type    = number
-  default = 45000
+  type = number
 }
 
 variable "DOCKERHUB_ACCOUNT" {
@@ -94,6 +91,5 @@ variable "BOOTSTRAP_NODES" {
 }
 
 variable "NODE_LB_RULE_PRIORITY" {
-  type    = number
-  default = 4000
+  type = number
 }
diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf
index ef2efbd26d21..36e2cd02291e 100644
--- a/yarn-project/aztec/terraform/prover-node/main.tf
+++ b/yarn-project/aztec/terraform/prover-node/main.tf
@@ -338,42 +338,42 @@ resource "aws_lb_listener_rule" "api" {
   }
 }
 
-resource "aws_security_group_rule" "allow-node-tcp-in" {
-  count             = local.node_count
-  type              = "ingress"
-  from_port         = var.NODE_P2P_TCP_PORT + count.index
-  to_port           = var.NODE_P2P_TCP_PORT + count.index
-  protocol          = "tcp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
-}
-
-resource "aws_security_group_rule" "allow-node-tcp-out" {
-  count             = local.node_count
-  type              = "egress"
-  from_port         = var.NODE_P2P_TCP_PORT + count.index
-  to_port           = var.NODE_P2P_TCP_PORT + count.index
-  protocol          = "tcp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
-}
-
-resource "aws_security_group_rule" "allow-node-udp-in" {
-  count             = local.node_count
-  type              = "ingress"
-  from_port         = var.NODE_P2P_UDP_PORT
-  to_port           = var.NODE_P2P_UDP_PORT + count.index
-  protocol          = "udp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
-}
-
-resource "aws_security_group_rule" "allow-node-udp-out" {
-  count             = local.node_count
-  type              = "egress"
-  from_port         = var.NODE_P2P_UDP_PORT
-  to_port           = var.NODE_P2P_UDP_PORT + count.index
-  protocol          = "udp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
-}
+# resource "aws_security_group_rule" "allow-node-tcp-in" {
+#   count             = local.node_count
+#   type              = "ingress"
+#   from_port         = var.NODE_P2P_TCP_PORT + count.index
+#   to_port           = var.NODE_P2P_TCP_PORT + count.index
+#   protocol          = "tcp"
+#   cidr_blocks       = ["0.0.0.0/0"]
+#   security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+# }
+
+# resource "aws_security_group_rule" "allow-node-tcp-out" {
+#   count             = local.node_count
+#   type              = "egress"
+#   from_port         = var.NODE_P2P_TCP_PORT + count.index
+#   to_port           = var.NODE_P2P_TCP_PORT + count.index
+#   protocol          = "tcp"
+#   cidr_blocks       = ["0.0.0.0/0"]
+#   security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+# }
+
+# resource "aws_security_group_rule" "allow-node-udp-in" {
+#   count             = local.node_count
+#   type              = "ingress"
+#   from_port         = var.NODE_P2P_UDP_PORT
+#   to_port           = var.NODE_P2P_UDP_PORT + count.index
+#   protocol          = "udp"
+#   cidr_blocks       = ["0.0.0.0/0"]
+#   security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+# }
+
+# resource "aws_security_group_rule" "allow-node-udp-out" {
+#   count             = local.node_count
+#   type              = "egress"
+#   from_port         = var.NODE_P2P_UDP_PORT
+#   to_port           = var.NODE_P2P_UDP_PORT + count.index
+#   protocol          = "udp"
+#   cidr_blocks       = ["0.0.0.0/0"]
+#   security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+# }
diff --git a/yarn-project/aztec/terraform/prover-node/variables.tf b/yarn-project/aztec/terraform/prover-node/variables.tf
index 4f0fbb0ac7f1..2c8fc301f236 100644
--- a/yarn-project/aztec/terraform/prover-node/variables.tf
+++ b/yarn-project/aztec/terraform/prover-node/variables.tf
@@ -20,18 +20,15 @@ variable "NODE_P2P_PRIVATE_KEYS" {
 }
 
 variable "L1_CHAIN_ID" {
-  type    = string
-  default = 677692
+  type = string
 }
 
 variable "NODE_P2P_TCP_PORT" {
-  type    = number
-  default = 40000
+  type = number
 }
 
 variable "NODE_P2P_UDP_PORT" {
-  type    = number
-  default = 45000
+  type = number
 }
 
 variable "DOCKERHUB_ACCOUNT" {
@@ -69,6 +66,5 @@ variable "BOOTSTRAP_NODES" {
 }
 
 variable "PROVER_NODE_LB_RULE_PRIORITY" {
-  type    = number
-  default = 7000
+  type = number
 }
diff --git a/yarn-project/aztec/terraform/pxe/variables.tf b/yarn-project/aztec/terraform/pxe/variables.tf
index f154c4c0c99c..7ce4470f0c30 100644
--- a/yarn-project/aztec/terraform/pxe/variables.tf
+++ b/yarn-project/aztec/terraform/pxe/variables.tf
@@ -21,6 +21,5 @@ variable "PROVING_ENABLED" {
 }
 
 variable "PXE_LB_RULE_PRIORITY" {
-  type    = number
-  default = 5000
+  type = number
 }

From 4dbc97957a7280acca7b27cb85ec0fb0a18119c6 Mon Sep 17 00:00:00 2001
From: PhilWindle <philip.windle@gmail.com>
Date: Wed, 14 Aug 2024 09:02:46 +0000
Subject: [PATCH 10/10] Remove SEQ_SKIP_SUBMIT_PROOFS

---
 yarn-project/aztec/terraform/node/main.tf      | 4 ----
 yarn-project/aztec/terraform/node/variables.tf | 5 -----
 2 files changed, 9 deletions(-)

diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf
index fe08ebbd573f..4bbe91821693 100644
--- a/yarn-project/aztec/terraform/node/main.tf
+++ b/yarn-project/aztec/terraform/node/main.tf
@@ -240,10 +240,6 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "SEQ_PUBLISHER_PRIVATE_KEY"
           value = local.sequencer_private_keys[count.index]
         },
-        {
-          name  = "SEQ_SKIP_SUBMIT_PROOFS"
-          value = tostring(var.SEQ_SKIP_SUBMIT_PROOFS)
-        },
         {
           name  = "ROLLUP_CONTRACT_ADDRESS"
           value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address
diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf
index 0dcdfa4b24ff..6a3b24ff9c1a 100644
--- a/yarn-project/aztec/terraform/node/variables.tf
+++ b/yarn-project/aztec/terraform/node/variables.tf
@@ -55,11 +55,6 @@ variable "SEQ_MIN_SECONDS_BETWEEN_BLOCKS" {
   default = 30
 }
 
-variable "SEQ_SKIP_SUBMIT_PROOFS" {
-  type    = bool
-  default = true
-}
-
 variable "P2P_MIN_PEERS" {
   type    = string
   default = 5