diff --git a/yarn-project/aztec/terraform/bot/main.tf b/yarn-project/aztec/terraform/bot/main.tf deleted file mode 100644 index 6f67bc4c66fe..000000000000 --- a/yarn-project/aztec/terraform/bot/main.tf +++ /dev/null @@ -1,269 +0,0 @@ -terraform { - backend "s3" { - bucket = "aztec-terraform" - region = "eu-west-2" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.74.2" - } - } -} - -# Define provider and region -provider "aws" { - region = "eu-west-2" -} - -data "terraform_remote_state" "aztec2_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec2/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "setup_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "setup/setup-iac" - region = "eu-west-2" - } -} - -resource "aws_cloudwatch_log_group" "aztec-bot-log-group" { - name = "/fargate/service/${var.DEPLOY_TAG}/aztec-bot" - retention_in_days = 14 -} - -resource "aws_service_discovery_service" "aztec-bot" { - name = "${var.DEPLOY_TAG}-aztec-bot" - - health_check_custom_config { - failure_threshold = 1 - } - - dns_config { - namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id - - dns_records { - ttl = 60 - type = "A" - } - - dns_records { - ttl = 60 - type = "SRV" - } - - routing_policy = "MULTIVALUE" - } - - # Terraform just fails if this resource changes and you have registered instances. - provisioner "local-exec" { - when = destroy - command = "${path.module}/../servicediscovery-drain.sh ${self.id}" - } -} - -# Create a fleet. -data "template_file" "user_data" { - template = <> /etc/ecs/ecs.config -echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-bot"}' >> /etc/ecs/ecs.config -EOF -} - -resource "aws_launch_template" "bot_launch_template" { - name = "${var.DEPLOY_TAG}-launch-template" - image_id = "ami-0cd4858f2b923aa6b" - instance_type = "c6a.2xlarge" - vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - - iam_instance_profile { - name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name - } - - key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name - - user_data = base64encode(data.template_file.user_data.rendered) - - tag_specifications { - resource_type = "instance" - tags = { - Name = "${var.DEPLOY_TAG}-bot" - prometheus = "" - } - } -} - -resource "aws_ec2_fleet" "bot_fleet" { - launch_template_config { - launch_template_specification { - launch_template_id = aws_launch_template.bot_launch_template.id - version = aws_launch_template.bot_launch_template.latest_version - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id - availability_zone = "eu-west-2a" - max_price = "0.15" - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - availability_zone = "eu-west-2b" - max_price = "0.15" - } - } - - target_capacity_specification { - default_target_capacity_type = "on-demand" - total_target_capacity = var.BOT_COUNT - spot_target_capacity = 0 - on_demand_target_capacity = var.BOT_COUNT - } - - terminate_instances = true - terminate_instances_with_expiration = true -} - -locals { - api_prefix = "/${var.DEPLOY_TAG}/aztec-bot/${var.BOT_API_KEY}" -} - -resource "aws_ecs_task_definition" "aztec-bot" { - family = "${var.DEPLOY_TAG}-aztec-bot" - network_mode = "awsvpc" - requires_compatibilities = ["EC2"] - execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn - task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn - - container_definitions = jsonencode([ - { - name = "${var.DEPLOY_TAG}-aztec-bot" - image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}" - command = ["start", "--bot", "--pxe"] - essential = true - cpu = 8192 - memoryReservation = 15685 - portMappings = [ - { - containerPort = 80 - hostPort = 80 - } - ] - environment = [ - { name = "BOT_L1_PRIVATE_KEY", value = var.BOT_L1_PRIVATE_KEY }, - { name = "BOT_PRIVATE_KEY", value = var.BOT_PRIVATE_KEY }, - { name = "BOT_NO_START", value = var.BOT_NO_START }, - { name = "BOT_TX_INTERVAL_SECONDS", value = var.BOT_TX_INTERVAL_SECONDS }, - { name = "LOG_LEVEL", value = var.LOG_LEVEL }, - { name = "AZTEC_PORT", value = "80" }, - { name = "API_PREFIX", value = local.api_prefix }, - { name = "BOT_PRIVATE_TRANSFERS_PER_TX", value = var.BOT_PRIVATE_TRANSFERS_PER_TX }, - { name = "BOT_PUBLIC_TRANSFERS_PER_TX", value = var.BOT_PUBLIC_TRANSFERS_PER_TX }, - { name = "BOT_TX_MINED_WAIT_SECONDS", value = var.BOT_TX_MINED_WAIT_SECONDS }, - { name = "BOT_FOLLOW_CHAIN", value = var.BOT_FOLLOW_CHAIN }, - { name = "AZTEC_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-1.local/${var.DEPLOY_TAG}/aztec-node-1/${var.API_KEY}" }, - { name = "PXE_PROVER_ENABLED", value = tostring(var.PROVING_ENABLED) }, - { name = "NETWORK", value = var.DEPLOY_TAG }, - { name = "BOT_FLUSH_SETUP_TRANSACTIONS", value = tostring(var.BOT_FLUSH_SETUP_TRANSACTIONS) }, - { name = "BOT_MAX_PENDING_TXS", value = tostring(var.BOT_MAX_PENDING_TXS) }, - { name = "BOT_SKIP_PUBLIC_SIMULATION", value = tostring(var.BOT_SKIP_PUBLIC_SIMULATION) }, - { name = "BOT_L2_GAS_LIMIT", value = var.BOT_L2_GAS_LIMIT }, - { name = "BOT_DA_GAS_LIMIT", value = var.BOT_DA_GAS_LIMIT }, - { name = "LOG_JSON", value = "1" }, - { name = "BOT_TOKEN_CONTRACT", value = var.BOT_TOKEN_CONTRACT } - ] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.aztec-bot-log-group.name - "awslogs-region" = "eu-west-2" - "awslogs-stream-prefix" = "ecs" - } - } - } - ]) -} - -resource "aws_ecs_service" "aztec-bot" { - name = "${var.DEPLOY_TAG}-aztec-bot" - cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id - launch_type = "EC2" - desired_count = var.BOT_COUNT - deployment_maximum_percent = 100 - deployment_minimum_healthy_percent = 0 - force_new_deployment = true - enable_execute_command = true - - network_configuration { - subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, - data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - ] - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - } - - # load_balancer { - # target_group_arn = aws_alb_target_group.bot_http.arn - # container_name = "${var.DEPLOY_TAG}-aztec-bot" - # container_port = 80 - # } - - service_registries { - registry_arn = aws_service_discovery_service.aztec-bot.arn - container_name = "${var.DEPLOY_TAG}-aztec-bot" - container_port = 80 - } - - placement_constraints { - type = "memberOf" - expression = "attribute:group == ${var.DEPLOY_TAG}-bot" - } - - task_definition = aws_ecs_task_definition.aztec-bot.family -} - -# resource "aws_alb_target_group" "bot_http" { -# name = "${var.DEPLOY_TAG}-bot-http" -# port = 80 -# protocol = "HTTP" -# target_type = "ip" -# vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id -# deregistration_delay = 5 - -# health_check { -# path = "${local.api_prefix}/status" -# matcher = 200 -# interval = 10 -# healthy_threshold = 2 -# unhealthy_threshold = 5 -# timeout = 5 -# } - -# tags = { -# name = "${var.DEPLOY_TAG}-bot-http" -# } -# } - -# resource "aws_lb_listener_rule" "bot_api" { -# listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn -# priority = 700 - -# action { -# type = "forward" -# target_group_arn = aws_alb_target_group.bot_http.arn -# } - -# condition { -# path_pattern { -# values = ["${local.api_prefix}*"] -# } -# } -# } diff --git a/yarn-project/aztec/terraform/bot/variables.tf b/yarn-project/aztec/terraform/bot/variables.tf deleted file mode 100644 index 84b925f3cb2d..000000000000 --- a/yarn-project/aztec/terraform/bot/variables.tf +++ /dev/null @@ -1,92 +0,0 @@ -variable "DEPLOY_TAG" { - type = string -} - -variable "DOCKERHUB_ACCOUNT" { - type = string -} - -variable "API_KEY" { - type = string -} - -variable "BOT_API_KEY" { - type = string -} - -variable "BOT_L1_PRIVATE_KEY" { - type = string - default = "" -} - -variable "BOT_PRIVATE_KEY" { - type = string -} - -variable "BOT_NO_START" { - type = string -} - -variable "BOT_PRIVATE_TRANSFERS_PER_TX" { - type = string -} - -variable "BOT_PUBLIC_TRANSFERS_PER_TX" { - type = string -} -variable "LOG_LEVEL" { - type = string - default = "verbose" -} - -variable "BOT_TX_INTERVAL_SECONDS" { - type = string - default = "300" -} - -variable "BOT_TX_MINED_WAIT_SECONDS" { - type = string -} - -variable "BOT_FOLLOW_CHAIN" { - type = string -} - -variable "PROVING_ENABLED" { - type = bool - default = false -} - -variable "BOT_COUNT" { - type = string - default = "1" -} - -variable "BOT_FLUSH_SETUP_TRANSACTIONS" { - type = bool - default = false -} - -variable "BOT_MAX_PENDING_TXS" { - type = number - default = 1 -} - -variable "BOT_SKIP_PUBLIC_SIMULATION" { - type = bool - default = false -} - -variable "BOT_L2_GAS_LIMIT" { - type = string -} - -variable "BOT_DA_GAS_LIMIT" { - type = string -} - -variable "BOT_TOKEN_CONTRACT" { - type = string - default = "TokenContract" -} - diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf deleted file mode 100644 index 8dd3a5a887cf..000000000000 --- a/yarn-project/aztec/terraform/node/main.tf +++ /dev/null @@ -1,592 +0,0 @@ -terraform { - backend "s3" { - bucket = "aztec-terraform" - region = "eu-west-2" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.74.2" - } - } -} - -# Define provider and region -provider "aws" { - region = "eu-west-2" -} - -data "terraform_remote_state" "setup_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "setup/setup-iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec2_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec2/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec-network_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec-network/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "l1_contracts" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "${var.DEPLOY_TAG}/l1-contracts" - region = "eu-west-2" - } -} - -# Compute local variables -locals { - sequencer_private_keys = var.SEQUENCER_PRIVATE_KEYS - node_p2p_private_keys = var.NODE_P2P_PRIVATE_KEYS - node_count = length(local.sequencer_private_keys) - data_dir = "/usr/src/yarn-project/aztec" - eth_host = var.ETHEREUM_HOSTS != "" ? var.ETHEREUM_HOSTS : "https://${var.DEPLOY_TAG}-mainnet-fork.aztec.network:8545/admin-${var.FORK_ADMIN_API_KEY}" -} - -output "node_count" { - value = local.node_count -} - -resource "aws_cloudwatch_log_group" "aztec-node-log-group" { - count = local.node_count - name = "/fargate/service/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}" - retention_in_days = 14 -} - -resource "aws_service_discovery_service" "aztec-node" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - - health_check_custom_config { - failure_threshold = 1 - } - - dns_config { - namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id - - dns_records { - ttl = 60 - type = "A" - } - - dns_records { - ttl = 60 - type = "SRV" - } - - routing_policy = "MULTIVALUE" - } - - # Terraform just fails if this resource changes and you have registered instances. - provisioner "local-exec" { - when = destroy - command = "${path.module}/../servicediscovery-drain.sh ${self.id}" - } -} - -# Configure an EFS filesystem. -resource "aws_efs_file_system" "node_data_store" { - creation_token = "${var.DEPLOY_TAG}-node-data" - - tags = { - Name = "${var.DEPLOY_TAG}-node-data" - } - - lifecycle_policy { - transition_to_ia = "AFTER_14_DAYS" - } -} - -resource "aws_efs_mount_target" "public_az1" { - file_system_id = aws_efs_file_system.node_data_store.id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] -} - -resource "aws_efs_mount_target" "public_az2" { - file_system_id = aws_efs_file_system.node_data_store.id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] -} - - -data "template_file" "user_data" { - count = local.node_count - template = <> /etc/ecs/ecs.config -echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"}' >> /etc/ecs/ecs.config -EOF -} - -# Launch template for our prover agents -# 4 cores and 8 GB memory -resource "aws_launch_template" "aztec-node-launch-template" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-node-launch-template-${count.index + 1}" - image_id = "ami-0cd4858f2b923aa6b" - instance_type = "c6a.xlarge" - vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - - iam_instance_profile { - name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name - } - - key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name - - user_data = base64encode(data.template_file.user_data[count.index].rendered) - - tag_specifications { - resource_type = "instance" - tags = { - Name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - prometheus = "" - } - } -} - -resource "aws_ec2_fleet" "aztec_node_fleet" { - count = local.node_count - launch_template_config { - launch_template_specification { - launch_template_id = aws_launch_template.aztec-node-launch-template[count.index].id - version = aws_launch_template.aztec-node-launch-template[count.index].latest_version - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id - availability_zone = "eu-west-2a" - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - availability_zone = "eu-west-2b" - } - } - - target_capacity_specification { - default_target_capacity_type = "on-demand" - total_target_capacity = 1 - spot_target_capacity = 0 - on_demand_target_capacity = 1 - } - - terminate_instances = true - terminate_instances_with_expiration = true -} - -# Define task definitions for each node. -resource "aws_ecs_task_definition" "aztec-node" { - count = local.node_count - family = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - requires_compatibilities = ["EC2"] - network_mode = "awsvpc" - execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn - task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn - - volume { - name = "efs-data-store" - efs_volume_configuration { - root_directory = "/" - file_system_id = aws_efs_file_system.node_data_store.id - } - } - - container_definitions = jsonencode([ - { - name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" - command = ["start", "--node", "--archiver", "--sequencer"] - essential = true - cpu = 4096 - memoryReservation = 7790 - portMappings = [ - { - containerPort = 80 - }, - { - containerPort = var.NODE_P2P_TCP_PORT + count.index - protocol = "tcp" - }, - { - containerPort = var.NODE_P2P_UDP_PORT + count.index - protocol = "udp" - } - ] - environment = [ - { - name = "NODE_ENV" - value = "production" - }, - { - name = "DEPLOY_TAG" - value = var.DEPLOY_TAG - }, - { - name = "L1_CHAIN_ID" - value = var.L1_CHAIN_ID - }, - { - name = "DEPLOY_AZTEC_CONTRACTS" - value = "false" - }, - { - name = "AZTEC_PORT" - value = "80" - }, - { - name = "AZTEC_ADMIN_PORT" - value = "8880" - }, - { - name = "ETHEREUM_HOSTS" - value = "${local.eth_host}" - }, - { - name = "DATA_DIRECTORY" - value = "${local.data_dir}/node_${count.index + 1}/data" - }, - { - name = "ARCHIVER_POLLING_INTERVAL_MS" - value = "10000" - }, - { - name = "ARCHIVER_VIEM_POLLING_INTERVAL_MS" - value = "10000" - }, - { - name = "SEQ_VIEM_POLLING_INTERVAL_MS" - value = "10000" - }, - { - name = "SEQ_RETRY_INTERVAL" - value = "10000" - }, - { - name = "SEQ_MAX_TX_PER_BLOCK" - value = var.SEQ_MAX_TX_PER_BLOCK - }, - { - name = "SEQ_MIN_TX_PER_BLOCK" - value = var.SEQ_MIN_TX_PER_BLOCK - }, - { - name = "SEQ_PUBLISHER_PRIVATE_KEY" - value = local.sequencer_private_keys[count.index] - }, - { - name = "VALIDATOR_PRIVATE_KEY" - value = local.sequencer_private_keys[count.index] - }, - { - name = "ROLLUP_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address - }, - { - name = "INBOX_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.inbox_contract_address - }, - { - name = "OUTBOX_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.outbox_contract_address - }, - { - name = "REGISTRY_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.registry_contract_address - }, - { - name = "FEE_JUICE_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.fee_juice_contract_address - }, - { - name = "STAKING_ASSET_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.staking_asset_contract_address - }, - { - name = "FEE_JUICE_PORTAL_CONTRACT_ADDRESS" - value = data.terraform_remote_state.l1_contracts.outputs.FEE_JUICE_PORTAL_CONTRACT_ADDRESS - }, - { - name = "API_KEY" - value = var.API_KEY - }, - { - name = "API_PREFIX" - value = "/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" - }, - { - name = "P2P_TCP_LISTEN_ADDR" - value = "0.0.0.0:${var.NODE_P2P_TCP_PORT + count.index}" - }, - { - name = "P2P_UDP_LISTEN_ADDR" - value = "0.0.0.0:${var.NODE_P2P_UDP_PORT + count.index}" - }, - { - name = "P2P_TCP_ANNOUNCE_ADDR" - value = ":${var.NODE_P2P_TCP_PORT + count.index}" - }, - { - name = "P2P_UDP_ANNOUNCE_ADDR" - value = ":${var.NODE_P2P_UDP_PORT + count.index}" - }, - { - name = "P2P_QUERY_FOR_IP" - value = "true" - }, - { - name = "BOOTSTRAP_NODES" - value = var.BOOTSTRAP_NODES - }, - { - name = "P2P_ENABLED" - value = tostring(var.P2P_ENABLED) - }, - { - name = "PEER_ID_PRIVATE_KEY" - value = local.node_p2p_private_keys[count.index] - }, - { - name = "P2P_MAX_PEERS" - value = var.P2P_MAX_PEERS - }, - { - name = "P2P_BLOCK_CHECK_INTERVAL_MS" - value = "10000" - }, - { - name = "P2P_PEER_CHECK_INTERVAL_MS" - value = "2000" - }, - { - name = "P2P_TX_POOL_KEEP_PROVEN_FOR", - value = tostring(var.P2P_TX_POOL_KEEP_PROVEN_FOR) - }, - { - name = "P2P_DOUBLE_SPEND_SEVERE_PEER_PENALTY_WINDOW" - value = tostring(var.P2P_DOUBLE_SPEND_SEVERE_PEER_PENALTY_WINDOW) - }, - { - name = "P2P_GOSSIPSUB_INTERVAL_MS" - value = tostring(var.P2P_GOSSIPSUB_INTERVAL_MS) - }, - { - name = "P2P_GOSSIPSUB_D" - value = tostring(var.P2P_GOSSIPSUB_D) - }, - { - name = "P2P_GOSSIPSUB_DLO" - value = tostring(var.P2P_GOSSIPSUB_DLO) - }, - { - name = "P2P_GOSSIPSUB_DHI" - value = tostring(var.P2P_GOSSIPSUB_DHI) - }, - { - name = "P2P_GOSSIPSUB_MCACHE_LENGTH" - value = tostring(var.P2P_GOSSIPSUB_MCACHE_LENGTH) - }, - { - name = "P2P_GOSSIPSUB_MCACHE_GOSSIP" - value = tostring(var.P2P_GOSSIPSUB_MCACHE_GOSSIP) - }, - { - name = "PROVER_AGENT_ENABLED" - value = "false" - }, - { - name = "PROVER_REAL_PROOFS" - value = tostring(var.PROVING_ENABLED) - }, - { - name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" - value = "http://aztec-otel.local:4318/v1/metrics" - }, - { - name = "OTEL_SERVICE_NAME" - value = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - }, - { - name = "BB_WORKING_DIRECTORY" - value = "${local.data_dir}/node_${count.index + 1}/temp" - }, - { - name = "ACVM_WORKING_DIRECTORY" - value = "${local.data_dir}/node_${count.index + 1}/temp" - }, - { - name = "LOG_LEVEL" - value = "info" - }, - { - name = "LOG_JSON", - value = "1" - }, - { - name = "NETWORK_NAME", - value = "${var.DEPLOY_TAG}" - }, - { - name = "VALIDATOR_DISABLED", - value = "1" - }, - ] - mountPoints = [ - { - containerPath = "${local.data_dir}/node_${count.index + 1}" - sourceVolume = "efs-data-store" - } - ] - dependsOn = [ - ] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = "/fargate/service/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}" - "awslogs-region" = "eu-west-2" - "awslogs-stream-prefix" = "ecs" - } - } - }]) -} - -resource "aws_ecs_service" "aztec-node" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id - launch_type = "EC2" - desired_count = 1 - deployment_maximum_percent = 100 - deployment_minimum_healthy_percent = 0 - #platform_version = "1.4.0" - force_new_deployment = true - enable_execute_command = true - - - network_configuration { - #assign_public_ip = true - subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, - data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - ] - security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - } - - load_balancer { - target_group_arn = aws_alb_target_group.aztec-node-http[count.index].arn - container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - container_port = 80 - } - - service_registries { - registry_arn = aws_service_discovery_service.aztec-node[count.index].arn - container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - container_port = 80 - } - - placement_constraints { - type = "memberOf" - expression = "attribute:group == ${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - } - - task_definition = aws_ecs_task_definition.aztec-node[count.index].family -} - -# Configure ALB to route /aztec-node to server. -resource "aws_alb_target_group" "aztec-node-http" { - count = local.node_count - name = "${var.DEPLOY_TAG}-node-${count.index + 1}-http-target" - port = 80 - protocol = "HTTP" - target_type = "ip" - vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id - deregistration_delay = 5 - - health_check { - path = "/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}/status" - matcher = "200" - interval = 10 - healthy_threshold = 2 - unhealthy_threshold = 5 - timeout = 5 - } - - tags = { - name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - } -} - -resource "aws_lb_listener_rule" "api" { - count = local.node_count - listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn - priority = var.NODE_LB_RULE_PRIORITY + count.index - - action { - type = "forward" - target_group_arn = aws_alb_target_group.aztec-node-http[count.index].arn - } - - condition { - path_pattern { - values = ["/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}*"] - } - } -} - -resource "aws_security_group_rule" "allow-node-tcp-in" { - count = local.node_count - type = "ingress" - from_port = var.NODE_P2P_TCP_PORT + count.index - to_port = var.NODE_P2P_TCP_PORT + count.index - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -} - -resource "aws_security_group_rule" "allow-node-tcp-out" { - count = local.node_count - type = "egress" - from_port = var.NODE_P2P_TCP_PORT + count.index - to_port = var.NODE_P2P_TCP_PORT + count.index - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -} - -resource "aws_security_group_rule" "allow-node-udp-in" { - count = local.node_count - type = "ingress" - from_port = var.NODE_P2P_UDP_PORT - to_port = var.NODE_P2P_UDP_PORT + count.index - protocol = "udp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -} - -resource "aws_security_group_rule" "allow-node-udp-out" { - count = local.node_count - type = "egress" - from_port = var.NODE_P2P_UDP_PORT - to_port = var.NODE_P2P_UDP_PORT + count.index - protocol = "udp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -} diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf deleted file mode 100644 index ae0d39d4e565..000000000000 --- a/yarn-project/aztec/terraform/node/variables.tf +++ /dev/null @@ -1,120 +0,0 @@ -variable "DEPLOY_TAG" { - type = string -} - -variable "IMAGE_TAG" { - type = string - default = "latest" -} - -variable "API_KEY" { - type = string -} - -variable "FORK_ADMIN_API_KEY" { - type = string - default = "" -} - -variable "ETHEREUM_HOSTS" { - type = string - default = "" -} - -variable "SEQUENCER_PRIVATE_KEYS" { - type = list(string) -} - -variable "NODE_P2P_PRIVATE_KEYS" { - type = list(string) -} - -variable "L1_CHAIN_ID" { - type = string -} - -variable "NODE_P2P_TCP_PORT" { - type = number -} - -variable "NODE_P2P_UDP_PORT" { - type = number -} - -variable "DOCKERHUB_ACCOUNT" { - type = string -} - -variable "SEQ_MAX_TX_PER_BLOCK" { - type = string - default = 64 -} - -variable "SEQ_MIN_TX_PER_BLOCK" { - type = string - default = 2 -} - -variable "P2P_MAX_PEERS" { - type = string - default = 100 -} - -variable "P2P_ENABLED" { - type = bool - default = false -} - -variable "P2P_TX_POOL_KEEP_PROVEN_FOR" { - type = number - default = 64 -} - -variable "P2P_GOSSIPSUB_INTERVAL_MS" { - type = number - default = 1000 -} - -variable "P2P_GOSSIPSUB_D" { - type = number - default = 8 -} - -variable "P2P_GOSSIPSUB_DLO" { - type = number - default = 4 -} - -variable "P2P_GOSSIPSUB_DHI" { - type = number - default = 12 -} - -variable "P2P_GOSSIPSUB_MCACHE_LENGTH" { - type = number - default = 5 -} - -variable "P2P_GOSSIPSUB_MCACHE_GOSSIP" { - type = number - default = 3 -} - -variable "P2P_DOUBLE_SPEND_SEVERE_PEER_PENALTY_WINDOW" { - type = number - default = 30 -} - -variable "PROVING_ENABLED" { - type = bool - default = false -} - -variable "BOOTSTRAP_NODES" { - type = string - default = "" -} - -variable "NODE_LB_RULE_PRIORITY" { - type = number -} diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf deleted file mode 100644 index 75adb4a200ae..000000000000 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ /dev/null @@ -1,430 +0,0 @@ -terraform { - backend "s3" { - bucket = "aztec-terraform" - region = "eu-west-2" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.74.2" - } - } -} - -# Define provider and region -provider "aws" { - region = "eu-west-2" -} - -data "terraform_remote_state" "setup_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "setup/setup-iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec2_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec2/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec-network_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec-network/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "l1_contracts" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "${var.DEPLOY_TAG}/l1-contracts" - region = "eu-west-2" - } -} - -# Compute local variables -locals { - prover_private_keys = var.PROVER_PRIVATE_KEYS - node_p2p_private_keys = var.NODE_P2P_PRIVATE_KEYS - node_count = length(local.prover_private_keys) - data_dir = "/usr/src/yarn-project/aztec" - eth_host = var.ETHEREUM_HOSTS != "" ? var.ETHEREUM_HOSTS : "https://${var.DEPLOY_TAG}-mainnet-fork.aztec.network:8545/admin-${var.API_KEY}" -} - -output "node_count" { - value = local.node_count -} - -resource "aws_cloudwatch_log_group" "aztec-prover-node-log-group" { - count = local.node_count - name = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}" - retention_in_days = 14 -} - -resource "aws_service_discovery_service" "aztec-prover-node" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - - health_check_custom_config { - failure_threshold = 1 - } - - dns_config { - namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id - - dns_records { - ttl = 60 - type = "A" - } - - dns_records { - ttl = 60 - type = "SRV" - } - - routing_policy = "MULTIVALUE" - } - - # Terraform just fails if this resource changes and you have registered instances. - provisioner "local-exec" { - when = destroy - command = "${path.module}/../servicediscovery-drain.sh ${self.id}" - } -} - -# Configure an EFS filesystem. -resource "aws_efs_file_system" "prover_node_data_store" { - creation_token = "${var.DEPLOY_TAG}-prover-node-data" - - tags = { - Name = "${var.DEPLOY_TAG}-prover-node-data" - } - - lifecycle_policy { - transition_to_ia = "AFTER_14_DAYS" - } -} - -resource "aws_efs_mount_target" "public_az1" { - file_system_id = aws_efs_file_system.prover_node_data_store.id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] -} - -resource "aws_efs_mount_target" "public_az2" { - file_system_id = aws_efs_file_system.prover_node_data_store.id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] -} - -data "template_file" "user_data" { - count = local.node_count - template = <> /etc/ecs/ecs.config -echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-prover-node-${count.index + 1}"}' >> /etc/ecs/ecs.config -EOF -} - -# Launch template for our prover agents -# 4 cores and 8 GB memory -resource "aws_launch_template" "aztec-prover-node-launch-template" { - count = local.node_count - name = "${var.DEPLOY_TAG}-prover-node-launch-template-${count.index + 1}" - image_id = "ami-0cd4858f2b923aa6b" - instance_type = "c6a.xlarge" - vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - - iam_instance_profile { - name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name - } - - key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name - - user_data = base64encode(data.template_file.user_data[count.index].rendered) - - tag_specifications { - resource_type = "instance" - tags = { - Name = "${var.DEPLOY_TAG}-prover-node-${count.index + 1}" - prometheus = "" - } - } -} - -resource "aws_ec2_fleet" "aztec_prover_node_fleet" { - count = local.node_count - launch_template_config { - launch_template_specification { - launch_template_id = aws_launch_template.aztec-prover-node-launch-template[count.index].id - version = aws_launch_template.aztec-prover-node-launch-template[count.index].latest_version - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id - availability_zone = "eu-west-2a" - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - availability_zone = "eu-west-2b" - } - } - - target_capacity_specification { - default_target_capacity_type = "on-demand" - total_target_capacity = 1 - spot_target_capacity = 0 - on_demand_target_capacity = 1 - } - - terminate_instances = true - terminate_instances_with_expiration = true -} - -# Define task definitions for each node. -resource "aws_ecs_task_definition" "aztec-prover-node" { - count = local.node_count - family = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - requires_compatibilities = ["EC2"] - network_mode = "awsvpc" - execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn - task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn - - volume { - name = "efs-data-store" - efs_volume_configuration { - root_directory = "/" - file_system_id = aws_efs_file_system.prover_node_data_store.id - } - } - - container_definitions = jsonencode([ - { - name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" - command = ["start", "--prover-node", "--archiver"] - essential = true - cpu = 4096 - memoryReservation = 7790 - portMappings = [ - { - containerPort = 80 - }, - { - containerPort = var.NODE_P2P_TCP_PORT + count.index - protocol = "tcp" - }, - { - containerPort = var.NODE_P2P_UDP_PORT + count.index - protocol = "udp" - } - ] - environment = [ - // General - { name = "NODE_ENV", value = "production" }, - { name = "LOG_LEVEL", value = "verbose" }, - { name = "LOG_JSON", value = "1" }, - { name = "DEPLOY_TAG", value = var.DEPLOY_TAG }, - { name = "NETWORK_NAME", value = "${var.DEPLOY_TAG}" }, - { name = "ETHEREUM_HOSTS", value = "${local.eth_host}" }, - { name = "L1_CHAIN_ID", value = var.L1_CHAIN_ID }, - { name = "DATA_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/data" }, - { name = "DEPLOY_AZTEC_CONTRACTS", value = "false" }, - - // API - { name = "AZTEC_PORT", value = "80" }, - { name = "API_KEY", value = var.API_KEY }, - { name = "API_PREFIX", value = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}" }, - - // Archiver - { name = "ARCHIVER_POLLING_INTERVAL_MS", value = "10000" }, - { name = "ARCHIVER_VIEM_POLLING_INTERVAL_MS", value = "10000" }, - { name = "PROVER_VIEM_POLLING_INTERVAL_MS", value = "10000" }, - - // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection) - { name = "TX_PROVIDER_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" }, - - // Prover - { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] }, - { name = "PROVER_AGENT_ENABLED", value = "false" }, - { name = "PROVER_REAL_PROOFS", value = tostring(var.PROVING_ENABLED) }, - { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/temp" }, - { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/temp" }, - { name = "PROVER_NODE_MAX_PENDING_JOBS", value = tostring(var.PROVER_NODE_MAX_PENDING_JOBS) }, - - // Metrics - { name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", value = "http://aztec-otel.local:4318/v1/metrics" }, - { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" }, - - // L1 addresses - { name = "ROLLUP_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address }, - { name = "INBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.inbox_contract_address }, - { name = "OUTBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.outbox_contract_address }, - { name = "REGISTRY_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.registry_contract_address }, - { name = "FEE_JUICE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.fee_juice_contract_address }, - { name = "STAKING_ASSET_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.staking_asset_contract_address }, - { name = "FEE_JUICE_PORTAL_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.FEE_JUICE_PORTAL_CONTRACT_ADDRESS }, - - // P2P (disabled) - { name = "P2P_ENABLED", value = tostring(var.P2P_ENABLED) }, - { name = "BOOTSTRAP_NODES", value = var.BOOTSTRAP_NODES }, - { name = "PEER_ID_PRIVATE_KEY", value = local.node_p2p_private_keys[count.index] }, - { name = "P2P_IP", value = "0.0.0.0" }, - { name = "P2P_PORT", value = "${var.NODE_P2P_TCP_PORT + count.index}" }, - { name = "P2P_QUERY_FOR_IP", value = "true" }, - { name = "P2P_MAX_PEERS", value = var.P2P_MAX_PEERS }, - { name = "P2P_BLOCK_CHECK_INTERVAL_MS", value = "1000" }, - { name = "P2P_PEER_CHECK_INTERVAL_MS", value = "2000" }, - { name = "P2P_TX_POOL_KEEP_PROVEN_FOR", value = tostring(var.P2P_TX_POOL_KEEP_PROVEN_FOR) }, - ] - mountPoints = [ - { - containerPath = "${local.data_dir}/prover_node_${count.index + 1}" - sourceVolume = "efs-data-store" - } - ] - dependsOn = [ - ] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}" - "awslogs-region" = "eu-west-2" - "awslogs-stream-prefix" = "ecs" - } - } - }]) -} - -resource "aws_ecs_service" "aztec-prover-node" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id - launch_type = "EC2" - desired_count = 1 - deployment_maximum_percent = 100 - deployment_minimum_healthy_percent = 0 - #platform_version = "1.4.0" - force_new_deployment = true - enable_execute_command = true - - network_configuration { - #assign_public_ip = true - subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, - data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - ] - security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - } - - load_balancer { - target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn - container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - container_port = 80 - } - - service_registries { - registry_arn = aws_service_discovery_service.aztec-prover-node[count.index].arn - container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - container_port = 80 - } - - placement_constraints { - type = "memberOf" - expression = "attribute:group == ${var.DEPLOY_TAG}-prover-node-${count.index + 1}" - } - - task_definition = aws_ecs_task_definition.aztec-prover-node[count.index].family -} - -# Configure ALB to route /aztec-prover-node to server. -resource "aws_alb_target_group" "aztec-prover-node-http" { - count = local.node_count - name = "${var.DEPLOY_TAG}-prover-${count.index + 1}-target" - port = 80 - protocol = "HTTP" - target_type = "ip" - vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id - deregistration_delay = 5 - - health_check { - path = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}/status" - matcher = "200" - interval = 10 - healthy_threshold = 2 - unhealthy_threshold = 5 - timeout = 5 - } - - tags = { - name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - } -} - -resource "aws_lb_listener_rule" "api" { - count = local.node_count - listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn - priority = var.PROVER_NODE_LB_RULE_PRIORITY + count.index - - action { - type = "forward" - target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn - } - - condition { - path_pattern { - values = ["/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}*"] - } - } -} - -# resource "aws_security_group_rule" "allow-node-tcp-in" { -# count = local.node_count -# type = "ingress" -# from_port = var.NODE_P2P_TCP_PORT + count.index -# to_port = var.NODE_P2P_TCP_PORT + count.index -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -# } - -# resource "aws_security_group_rule" "allow-node-tcp-out" { -# count = local.node_count -# type = "egress" -# from_port = var.NODE_P2P_TCP_PORT + count.index -# to_port = var.NODE_P2P_TCP_PORT + count.index -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -# } - -# resource "aws_security_group_rule" "allow-node-udp-in" { -# count = local.node_count -# type = "ingress" -# from_port = var.NODE_P2P_UDP_PORT -# to_port = var.NODE_P2P_UDP_PORT + count.index -# protocol = "udp" -# cidr_blocks = ["0.0.0.0/0"] -# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -# } - -# resource "aws_security_group_rule" "allow-node-udp-out" { -# count = local.node_count -# type = "egress" -# from_port = var.NODE_P2P_UDP_PORT -# to_port = var.NODE_P2P_UDP_PORT + count.index -# protocol = "udp" -# cidr_blocks = ["0.0.0.0/0"] -# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id -# } diff --git a/yarn-project/aztec/terraform/prover-node/variables.tf b/yarn-project/aztec/terraform/prover-node/variables.tf deleted file mode 100644 index 834ea295cdf8..000000000000 --- a/yarn-project/aztec/terraform/prover-node/variables.tf +++ /dev/null @@ -1,76 +0,0 @@ -variable "DEPLOY_TAG" { - type = string -} - -variable "IMAGE_TAG" { - type = string - default = "latest" -} - -variable "ETHEREUM_HOSTS" { - type = string - default = "" -} - -variable "API_KEY" { - type = string - default = "" -} - -variable "PROVER_PRIVATE_KEYS" { - type = list(string) -} - -variable "NODE_P2P_PRIVATE_KEYS" { - type = list(string) -} - -variable "L1_CHAIN_ID" { - type = string -} - -variable "NODE_P2P_TCP_PORT" { - type = number -} - -variable "NODE_P2P_UDP_PORT" { - type = number -} - -variable "DOCKERHUB_ACCOUNT" { - type = string -} - -variable "P2P_MAX_PEERS" { - type = string - default = 100 -} - -variable "P2P_ENABLED" { - type = bool - default = false -} - -variable "P2P_TX_POOL_KEEP_PROVEN_FOR" { - type = number - default = 64 -} - -variable "PROVING_ENABLED" { - type = bool - default = false -} - -variable "PROVER_NODE_MAX_PENDING_JOBS" { - type = number - default = 16 -} - -variable "BOOTSTRAP_NODES" { - type = string - default = "" -} - -variable "PROVER_NODE_LB_RULE_PRIORITY" { - type = number -} diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf deleted file mode 100644 index 2f4a65c37648..000000000000 --- a/yarn-project/aztec/terraform/prover/main.tf +++ /dev/null @@ -1,431 +0,0 @@ -terraform { - backend "s3" { - bucket = "aztec-terraform" - region = "eu-west-2" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.74.2" - } - } -} - -# Define provider and region -provider "aws" { - region = "eu-west-2" -} - -data "terraform_remote_state" "setup_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "setup/setup-iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec2_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec2/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec-network_iac" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "aztec-network/iac" - region = "eu-west-2" - } -} - -data "terraform_remote_state" "aztec-network_prover-node" { - backend = "s3" - config = { - bucket = "aztec-terraform" - key = "${var.DEPLOY_TAG}/aztec-prover-node" - region = "eu-west-2" - } -} - -locals { - node_count = data.terraform_remote_state.aztec-network_prover-node.outputs.node_count - agents_per_prover = var.AGENTS_PER_PROVER -} - -resource "aws_cloudwatch_log_group" "aztec-proving-agent-log-group" { - count = local.node_count - name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-group-${count.index + 1}" - retention_in_days = 14 -} - -resource "aws_service_discovery_service" "aztec-proving-agent" { - count = local.node_count - name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}" - - health_check_custom_config { - failure_threshold = 1 - } - dns_config { - namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id - dns_records { - ttl = 60 - type = "A" - } - dns_records { - ttl = 60 - type = "SRV" - } - routing_policy = "MULTIVALUE" - } - # Terraform just fails if this resource changes and you have registered instances. - provisioner "local-exec" { - when = destroy - command = "${path.module}/../servicediscovery-drain.sh ${self.id}" - } -} - -# Create an autoscaling group for every sequencer. For each group we want 1 permanent on-demand instance to ensure liveness. -# We will fill the remaining desired instances from spot capacity. -data "template_file" "user_data" { - count = local.node_count - template = <> /etc/ecs/ecs.config -echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-proving-agent-group-${count.index + 1}"}' >> /etc/ecs/ecs.config -EOF -} - -# Launch template for our prover agents -# 16 cores and 128 GB memory -resource "aws_launch_template" "proving-agent-launch-template" { - count = local.node_count - name = "${var.DEPLOY_TAG}-proving-agent-launch-template-${count.index + 1}" - image_id = "ami-0cd4858f2b923aa6b" - instance_type = "r5a.4xlarge" - vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] - - iam_instance_profile { - name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name - } - - key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name - - user_data = base64encode(data.template_file.user_data[count.index].rendered) - - tag_specifications { - resource_type = "instance" - tags = { - Name = "${var.DEPLOY_TAG}-proving-agent-group-${count.index + 1}" - prometheus = "" - } - } -} - -resource "aws_ec2_fleet" "aztec_proving_agent_fleet" { - count = local.node_count - launch_template_config { - launch_template_specification { - launch_template_id = aws_launch_template.proving-agent-launch-template[count.index].id - version = aws_launch_template.proving-agent-launch-template[count.index].latest_version - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id - availability_zone = "eu-west-2a" - max_price = "0.7" - } - - override { - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - availability_zone = "eu-west-2b" - max_price = "0.7" - } - } - - target_capacity_specification { - default_target_capacity_type = "on-demand" - total_target_capacity = local.agents_per_prover - spot_target_capacity = 0 - on_demand_target_capacity = local.agents_per_prover - } - - terminate_instances = true - terminate_instances_with_expiration = true -} - -# Sets up the autoscaling groups -# resource "aws_autoscaling_group" "proving-agent-auto-scaling-group" { -# count = local.node_count -# min_size = 1 -# max_size = local.agents_per_prover -# desired_capacity = 1 -# vpc_zone_identifier = [data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id] - -# mixed_instances_policy { -# instances_distribution { -# on_demand_base_capacity = 1 -# on_demand_percentage_above_base_capacity = 100 -# spot_allocation_strategy = "lowest-price" -# spot_max_price = "0.7" # Current spot instance price for the m5.8xlarge instance type -# } - -# launch_template { -# launch_template_specification { -# launch_template_id = aws_launch_template.proving-agent-launch-template[count.index].id -# version = "$Latest" -# } -# } -# } - -# tag { -# key = "AmazonECSManaged" -# value = true -# propagate_at_launch = true -# } -# } - - -# # Capacity provider to manage the scaling of the EC2 instances -# resource "aws_ecs_capacity_provider" "proving-agent-capacity-provider" { -# count = local.node_count -# name = "${var.DEPLOY_TAG}-proving-agent-capacity-provider-${count.index + 1}" - - -# auto_scaling_group_provider { -# auto_scaling_group_arn = aws_autoscaling_group.proving-agent-auto-scaling-group[count.index].arn -# managed_termination_protection = "DISABLED" - -# managed_scaling { -# maximum_scaling_step_size = local.agents_per_prover -# minimum_scaling_step_size = 1 -# status = "ENABLED" -# target_capacity = 100 -# } -# } -# } - -# # Update the capacity providers on the cluster -# resource "aws_ecs_cluster_capacity_providers" "proving-agent-capacity-providers" { -# count = local.node_count -# cluster_name = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_name - -# #capacity_providers = [aws_ecs_capacity_provider.proving-agent-capacity-provider[count.index].name] - -# capacity_providers = local.enable_ecs_cluster_auto_scaling == true ? aws_ecs_capacity_provider.asg[*].name : [] - -# capacity_providers = (contains(capacity_providers, aws_ecs_capacity_provider.proving-agent-capacity-provider[count.index].name) == false ? concat(capacity_providers, [aws_ecs_capacity_provider.proving-agent-capacity-provider[count.index].name]) : capacity_providers) -# } - - -# Define task definitions for each node. -resource "aws_ecs_task_definition" "aztec-proving-agent" { - count = local.node_count - family = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}" - requires_compatibilities = ["EC2"] - network_mode = "awsvpc" - execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn - task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn - container_definitions = <" && exit 1 - -serviceId="--service-id=$1" - -echo "Draining servicediscovery instances from $1 ..." -ids="$(aws servicediscovery list-instances $serviceId --query 'Instances[].Id' --output text | tr '\t' ' ')" - -found= -for id in $ids; do - if [ -n "$id" ]; then - echo "Deregistering $1 / $id ..." - aws servicediscovery deregister-instance $serviceId --instance-id "$id" - found=1 - fi -done - -# Yes, I'm being lazy here... -[ -n "$found" ] && sleep 5 || true \ No newline at end of file