Skip to content

Commit

Permalink
feat: Aztec multi cloud deployment (#9980)
Browse files Browse the repository at this point in the history
# Change Log

- **Working Aztec multi cloud deployment**
- Additional firewall rules have been added to expose ethereum nodes in
AWS and Cloud.
- Test `multicloud-demo.yaml` helm values file is included for testing
deployments.
- Load balancers are renamed to not use Helm helper functions, due to
these values not being readable during terraform tests.
- Updated Google Cloud node sizes - so accommodate beefy provers testing

## Testing

These changes were testing by issuing a `terraform apply` command, and
connecting to nodes to confirm block confirmations were happening, as
well as prover bids were being sent from nodes in Google Cloud to AWS.
  • Loading branch information
stevenplatt authored Nov 15, 2024
1 parent 202870a commit d29bcf6
Show file tree
Hide file tree
Showing 15 changed files with 413 additions and 122 deletions.
4 changes: 2 additions & 2 deletions spartan/aztec-network/templates/boot-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-boot-node-lb-tcp
name: boot-node-lb-tcp
labels:
{{- include "aztec-network.labels" . | nindent 4 }}
spec:
Expand All @@ -252,7 +252,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-boot-node-lb-udp
name: boot-node-lb-udp
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
Expand Down
6 changes: 3 additions & 3 deletions spartan/aztec-network/templates/prover-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -182,11 +182,11 @@ spec:
name: p2p-udp
protocol: UDP
---
{{if .Values.network.public }}
{{if .Values.proverNode.public }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-prover-node-lb-tcp
name: prover-node-lb-tcp
labels:
{{- include "aztec-network.labels" . | nindent 4 }}
spec:
Expand All @@ -203,7 +203,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-prover-node-lb-udp
name: prover-node-lb-udp
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
Expand Down
2 changes: 1 addition & 1 deletion spartan/aztec-network/templates/pxe.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-pxe-lb
name: pxe-lb
labels:
{{- include "aztec-network.labels" . | nindent 4 }}
spec:
Expand Down
4 changes: 3 additions & 1 deletion spartan/aztec-network/templates/reth.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" . }}-ethereum-lb
name: ethereum-lb
labels:
{{- include "aztec-network.labels" . | nindent 4 }}
spec:
Expand All @@ -114,6 +114,7 @@ data:
genesis.json: |
{{ .Files.Get "files/config/genesis.json" | nindent 4 }}
---
{{- if gt (.Values.ethereum.replicas | int) 0 }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
Expand All @@ -125,4 +126,5 @@ spec:
resources:
requests:
storage: {{ .Values.ethereum.storage }}
{{- end }}
---
4 changes: 2 additions & 2 deletions spartan/aztec-network/templates/validator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }}-lb-tcp
name: validator-{{ $i }}-lb-tcp
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
Expand All @@ -244,7 +244,7 @@ spec:
apiVersion: v1
kind: Service
metadata:
name: {{ include "aztec-network.fullname" $ }}-validator-{{ $i }}-lb-udp
name: validator-{{ $i }}-lb-udp
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
Expand Down
1 change: 1 addition & 0 deletions spartan/aztec-network/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ validator:
storage: "8Gi"

proverNode:
public: false
externalTcpHost: ""
externalUdpHost: ""
replicas: 1
Expand Down
31 changes: 31 additions & 0 deletions spartan/aztec-network/values/multicloud-demo.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# deployment syntax: helm upgrade --install srp-test . -n srp-test --create-namespace --values "./values/multicloud-demo.yaml"

telemetry:
enabled: false
otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318

validator:
replicas: 1
validatorKeys:
- 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
validatorAddresses:
- 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
validator:
disabled: false

bootNode:
validator:
disabled: true

proverNode:
realProofs: false

proverAgent:
replicas: 1
realProofs: false
bb:
hardwareConcurrency: 16

aztec:
slotDuration: 36
epochDuration: 32
112 changes: 112 additions & 0 deletions spartan/terraform/eks-cluster/firewall.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# security_groups.tf

resource "aws_security_group" "node_traffic" {
name_prefix = "eks-node-traffic"
description = "Security group for EKS node UDP and TCP traffic"
vpc_id = module.vpc.vpc_id # Fixed VPC reference to use the vpc module output

# Ingress UDP rules
ingress {
from_port = 40400
to_port = 40499
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming UDP traffic for original port range"
}

ingress {
from_port = 8080
to_port = 8080
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming UDP traffic on port 8080"
}

ingress {
from_port = 8545
to_port = 8545
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming UDP traffic on port 8545"
}

# Ingress TCP rules
ingress {
from_port = 40400
to_port = 40499
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming TCP traffic for original port range"
}

ingress {
from_port = 8080
to_port = 8080
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming TCP traffic on port 8080"
}

ingress {
from_port = 8545
to_port = 8545
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming TCP traffic on port 8545"
}

# Egress UDP rules
egress {
from_port = 40400
to_port = 40499
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing UDP traffic for original port range"
}

egress {
from_port = 8080
to_port = 8080
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing UDP traffic on port 8080"
}

egress {
from_port = 8545
to_port = 8545
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing UDP traffic on port 8545"
}

# Egress TCP rules
egress {
from_port = 40400
to_port = 40499
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing TCP traffic for original port range"
}

egress {
from_port = 8080
to_port = 8080
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing TCP traffic on port 8080"
}

egress {
from_port = 8545
to_port = 8545
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing TCP traffic on port 8545"
}

tags = {
Name = "${var.cluster_name}-node-traffic"
Project = var.cluster_name
}
}
48 changes: 0 additions & 48 deletions spartan/terraform/eks-cluster/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -26,54 +26,6 @@ data "aws_availability_zones" "available" {
}
}

# Create security group for node traffic
resource "aws_security_group" "node_traffic" {
name_prefix = "eks-node-traffic"
description = "Security group for EKS node UDP and TCP traffic"
vpc_id = module.vpc.vpc_id

# Ingress UDP rule
ingress {
from_port = 40400
to_port = 40499
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming UDP traffic"
}

# Ingress TCP rule
ingress {
from_port = 40400
to_port = 40499
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow incoming TCP traffic"
}

# Egress UDP rule
egress {
from_port = 40400
to_port = 40499
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing UDP traffic"
}

# Egress TCP rule
egress {
from_port = 40400
to_port = 40499
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
description = "Allow outgoing TCP traffic"
}

tags = {
Name = "${var.cluster_name}-node-traffic"
Project = var.cluster_name
}
}

module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "5.8.1"
Expand Down
51 changes: 51 additions & 0 deletions spartan/terraform/gke-cluster/firewall.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Create ingress firewall rules for UDP
resource "google_compute_firewall" "udp_ingress" {
name = "allow-udp-ingress-custom"
network = "default"
allow {
protocol = "udp"
ports = ["40400-40499", "8080", "8545"]
}
direction = "INGRESS"
source_ranges = ["0.0.0.0/0"]
target_tags = ["gke-node"]
}

# Create egress firewall rules for UDP
resource "google_compute_firewall" "udp_egress" {
name = "allow-udp-egress-custom"
network = "default"
allow {
protocol = "udp"
ports = ["40400-40499", "8080", "8545"]
}
direction = "EGRESS"
destination_ranges = ["0.0.0.0/0"]
target_tags = ["gke-node"]
}

# Create ingress firewall rules for TCP
resource "google_compute_firewall" "tcp_ingress" {
name = "allow-tcp-ingress-custom"
network = "default"
allow {
protocol = "tcp"
ports = ["40400-40499", "8080", "8545"]
}
direction = "INGRESS"
source_ranges = ["0.0.0.0/0"]
target_tags = ["gke-node"]
}

# Create egress firewall rules for TCP
resource "google_compute_firewall" "tcp_egress" {
name = "allow-tcp-egress-custom"
network = "default"
allow {
protocol = "tcp"
ports = ["40400-40499", "8080", "8545"]
}
direction = "EGRESS"
destination_ranges = ["0.0.0.0/0"]
target_tags = ["gke-node"]
}
Loading

0 comments on commit d29bcf6

Please sign in to comment.