From 4606da83fd66ec18e2ee56d75a56b6b811540bbd Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Mon, 14 Jun 2021 18:04:04 +0200
Subject: [PATCH 1/6] Added prestop hook to fullnode template. Increased
 baklava memory

---
 .env.baklava                                       | 2 +-
 packages/helm-charts/common/templates/_helpers.tpl | 8 ++++++++
 packages/helm-charts/testnet/values.yaml           | 7 ++++---
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.env.baklava b/.env.baklava
index 8431b805312..cab6d5729d8 100644
--- a/.env.baklava
+++ b/.env.baklava
@@ -170,7 +170,7 @@ IN_MEMORY_DISCOVERY_TABLE=false
 CLUSTER_CREATION_FLAGS="--enable-autoscaling --min-nodes 3 --max-nodes 8 --machine-type=n1-standard-4"
 
 GETH_NODE_CPU_REQUEST=4
-GETH_NODE_MEMORY_REQUEST=4Gi
+GETH_NODE_MEMORY_REQUEST=8Gi
 
 VERIFICATION_POOL_URL="https://us-central1-celo-testnet.cloudfunctions.net/handleVerificationRequestbaklava/v0.1/sms/"
 VERIFICATION_REWARDS_URL="https://us-central1-celo-testnet.cloudfunctions.net/handleVerificationRequestbaklava/v0.1/rewards/"
diff --git a/packages/helm-charts/common/templates/_helpers.tpl b/packages/helm-charts/common/templates/_helpers.tpl
index 8a3fbbd989c..add3f1a8639 100644
--- a/packages/helm-charts/common/templates/_helpers.tpl
+++ b/packages/helm-charts/common/templates/_helpers.tpl
@@ -216,6 +216,7 @@ fi
       fieldRef:
         fieldPath: status.hostIP
 {{- end }}
+{{ include  "common.geth-prestop-hook" . | indent 2 -}}
 {{/* TODO: make this use IPC */}}
 {{- if .expose }}
   readinessProbe:
@@ -266,6 +267,13 @@ fi
 {{- end }}
 {{- end -}}
 
+{{- define "common.geth-prestop-hook" -}}
+lifecycle:
+  preStop:
+    exec:
+      command: ["/bin/sh","-c","killall -HUP geth; while killall -0 geth; do sleep 1; done"]
+{{- end -}}
+
 {{- define "common.geth-configmap" -}}
 apiVersion: v1
 kind: ConfigMap
diff --git a/packages/helm-charts/testnet/values.yaml b/packages/helm-charts/testnet/values.yaml
index 13ded98b310..7eaa5816773 100644
--- a/packages/helm-charts/testnet/values.yaml
+++ b/packages/helm-charts/testnet/values.yaml
@@ -32,9 +32,10 @@ geth:
     requests:
       memory: "256Mi"
       cpu: "500m"
-    limits:
-      memory: "4Gi"
-      cpu: "4"
+    limits: {}
+    # limits:
+    #   memory: "4Gi"
+    #   cpu: "4"
   fullnodeCheckBlockAge: true
 
 # UpdateStrategy for statefulsets only. Partition=0 is default rollingUpdate behaviour.

From 8ae379e3112fe64394d90ce6a42d51d725b1ea16 Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Mon, 21 Jun 2021 12:13:26 +0200
Subject: [PATCH 2/6] Added nginx sidecar to geth deployment to handle session
 affinity

---
 .../celo-fullnode/templates/_helpers.tpl      |  6 ++
 .../templates/nginx-geth-configmap.yaml       |  3 +
 .../celo-fullnode/templates/statefulset.yaml  |  6 ++
 .../helm-charts/celo-fullnode/values.yaml     |  1 +
 .../helm-charts/common/templates/_helpers.tpl | 94 +++++++++++++++++++
 .../testnet/templates/_helpers.tpl            | 10 ++
 .../templates/nginx-geth-configmap.yaml       |  3 +
 .../templates/txnode-private.statefulset.yaml |  2 +-
 8 files changed, 124 insertions(+), 1 deletion(-)
 create mode 100644 packages/helm-charts/celo-fullnode/templates/nginx-geth-configmap.yaml
 create mode 100644 packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml

diff --git a/packages/helm-charts/celo-fullnode/templates/_helpers.tpl b/packages/helm-charts/celo-fullnode/templates/_helpers.tpl
index d07b73ed4a6..7b131704acf 100644
--- a/packages/helm-charts/celo-fullnode/templates/_helpers.tpl
+++ b/packages/helm-charts/celo-fullnode/templates/_helpers.tpl
@@ -40,6 +40,12 @@ Create chart name and version as used by the chart label.
   targetPort: ws
   protocol: TCP
   name: ws
+{{- if .Values.geth.enable_nginx | default false }}
+- port: 3000
+  targetPort: nginx
+  protocol: TCP
+  name: nginx
+{{- end }}
 {{- end -}}
 
 {{/*
diff --git a/packages/helm-charts/celo-fullnode/templates/nginx-geth-configmap.yaml b/packages/helm-charts/celo-fullnode/templates/nginx-geth-configmap.yaml
new file mode 100644
index 00000000000..f18f1cb132c
--- /dev/null
+++ b/packages/helm-charts/celo-fullnode/templates/nginx-geth-configmap.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-configmap" . }}
+{{- end }}
diff --git a/packages/helm-charts/celo-fullnode/templates/statefulset.yaml b/packages/helm-charts/celo-fullnode/templates/statefulset.yaml
index c455afbe332..224f61e1a56 100644
--- a/packages/helm-charts/celo-fullnode/templates/statefulset.yaml
+++ b/packages/helm-charts/celo-fullnode/templates/statefulset.yaml
@@ -85,6 +85,9 @@ spec:
 {{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "http" "tcp_check_port" 8545 "server_port" 6000) | indent 6 }}
 {{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "ws" "tcp_check_port" 8546 "server_port" 6001) | indent 6 }}
 {{ end }}
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-sidecar" . | indent 6 }}
+{{- end }}
       terminationGracePeriodSeconds: 120 # 2 mins
       volumes:
       - name: data
@@ -93,3 +96,6 @@ spec:
         configMap:
           name: {{ template "common.fullname" . }}-health-check
           defaultMode: 0555
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-sidecar-volume" . | indent 6 }}
+{{- end }}
diff --git a/packages/helm-charts/celo-fullnode/values.yaml b/packages/helm-charts/celo-fullnode/values.yaml
index c57fdd7e6ea..32fb67fb698 100644
--- a/packages/helm-charts/celo-fullnode/values.yaml
+++ b/packages/helm-charts/celo-fullnode/values.yaml
@@ -93,6 +93,7 @@ geth:
     maxpeers: 1000
     serve: 90
   fullnodeCheckBlockAge: true
+  enable_nginx: true
 
 genesis:
   networkId: 31418
diff --git a/packages/helm-charts/common/templates/_helpers.tpl b/packages/helm-charts/common/templates/_helpers.tpl
index 83c24a5ef26..707b26110b4 100644
--- a/packages/helm-charts/common/templates/_helpers.tpl
+++ b/packages/helm-charts/common/templates/_helpers.tpl
@@ -530,3 +530,97 @@ prometheus.io/port: "{{ $pprof.port | default 6060 }}"
   - name: data
     mountPath: /root/.celo
 {{- end -}}
+
+{{- define "common.nginx-geth-sidecar" -}}
+- name: nginx
+  image: nginx:latest
+  imagePullPolicy: IfNotPresent
+  ports:
+  - containerPort: 3000
+    name: nginx
+    protocol: TCP
+  volumeMounts:
+  - name: nginx-geth-config
+    mountPath: /etc/nginx/nginx.conf
+    subPath: nginx.conf
+{{- end -}}
+
+{{- define "common.nginx-geth-sidecar-volume" -}}
+- name: nginx-geth-config
+  configMap:
+    name: {{ template "common.fullname" . }}-nginx-geth-config
+{{- end -}}
+
+{{- define "common.nginx-geth-configmap" -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "common.fullname" . }}-nginx-geth-config
+  labels:
+{{ include "common.standard.labels" .  | indent 4 }}
+data:
+  nginx.conf: |-
+    # /etc/nginx/nginx.conf
+    user nginx;
+    # Set number of worker processes automatically based on number of CPU cores.
+    worker_processes auto;
+    # Enables the use of JIT for regular expressions to speed-up their processing.
+    pcre_jit on;
+    # Configures default error logger.
+    error_log /var/log/nginx/error.log warn;
+    # Includes files with directives to load dynamic modules.
+    include /etc/nginx/modules/*.conf;
+    events {
+        # The maximum number of simultaneous connections that can be opened by
+        # a worker process.
+        worker_connections 1024;
+    }
+    http {
+    server {
+        listen 3000;
+        server_name tx-nodes-private;
+
+        location / {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://ws-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+
+        location /rpc {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://rpc-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+
+        location /ws {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://ws-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+    }
+    upstream rpc-backend {
+        server localhost:8545;
+    }
+
+    upstream ws-backend {
+        server localhost:8546;
+    }
+    }
+    # TIP: Uncomment if you use stream module.
+    #include /etc/nginx/stream.conf;
+{{- end -}}
diff --git a/packages/helm-charts/testnet/templates/_helpers.tpl b/packages/helm-charts/testnet/templates/_helpers.tpl
index ae7934b072d..c96049bb510 100644
--- a/packages/helm-charts/testnet/templates/_helpers.tpl
+++ b/packages/helm-charts/testnet/templates/_helpers.tpl
@@ -59,6 +59,10 @@ spec:
     name: rpc
   - port: 8546
     name: ws
+{{- if .enable_nginx_geth | default false }}
+  - port: 3000
+    name: nginx
+{{- end }}
   selector:
 {{- if .proxy | default false }}
 {{- $validatorProxied := printf "%s-validators-%d" .Release.Namespace .validator_index }}
@@ -151,6 +155,9 @@ spec:
 {{ end }}
       containers:
 {{ include "common.full-node-container" (dict "Values" .Values "Release" .Release "Chart" .Chart "proxy" .proxy "proxy_allow_private_ip_flag" .proxy_allow_private_ip_flag "unlock" .unlock "expose" .expose "syncmode" .syncmode "gcmode" .gcmode "pprof" (or (.Values.metrics) (.Values.pprof.enabled)) "pprof_port" (.Values.pprof.port) "metrics" .Values.metrics "public_ips" .public_ips "ethstats" (printf "%s-ethstats.%s" (include "common.fullname" .) .Release.Namespace))  | indent 6 }}
+{{- if .enable_nginx_geth | default false }}
+{{ include "common.nginx-geth-sidecar" . | indent 6 }}
+{{- end }}
       terminationGracePeriodSeconds: 120 # 2 mins
       volumes:
       - name: data
@@ -161,6 +168,9 @@ spec:
       - name: account
         secret:
           secretName: {{ template "common.fullname" . }}-geth-account
+{{- if .enable_nginx_geth | default false }}
+{{ include "common.nginx-geth-sidecar-volume" . | indent 6 }}
+{{- end }}
 {{- end -}}
 
 {{- /* This template puts a semicolon-separated pair of proxy enodes into $PROXY_ENODE_URL_PAIR. */ -}}
diff --git a/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml b/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml
new file mode 100644
index 00000000000..f18f1cb132c
--- /dev/null
+++ b/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-configmap" . }}
+{{- end }}
diff --git a/packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml b/packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml
index e3f536575b2..eae26834543 100644
--- a/packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml
+++ b/packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml
@@ -1 +1 @@
-{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses)) }}
+{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses) "enable_nginx_geth" true) }}

From 6596ddf3fcdaf4c531945720ca1cb7384d21ea6e Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Mon, 21 Jun 2021 15:59:17 +0200
Subject: [PATCH 3/6] Added celotool contexted option for deploying nginx geth
 sidecar

---
 .env.alfajores                                 |  1 +
 .env.baklava                                   |  3 +++
 .env.rc1                                       | 13 +++++++++----
 packages/celotool/src/lib/env-utils.ts         |  1 +
 packages/celotool/src/lib/fullnodes.ts         |  2 ++
 packages/celotool/src/lib/k8s-fullnode/base.ts |  2 ++
 6 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.env.alfajores b/.env.alfajores
index 30d9b117823..7854abd5092 100644
--- a/.env.alfajores
+++ b/.env.alfajores
@@ -51,6 +51,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_ROLLING_UPDATE_PARTITION=0
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # Temporarily point to celo-org repository to consume patched image.
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-org/geth"
diff --git a/.env.baklava b/.env.baklava
index 82840cda4e4..ec577f8d933 100644
--- a/.env.baklava
+++ b/.env.baklava
@@ -73,6 +73,7 @@ AZURE_ORACLE_WESTUS2_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_WESTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS2_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_CENTRALUS_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_CENTRALUS_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -90,6 +91,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Forno ----
 
@@ -118,6 +120,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-dc5e5dfa07231a4ff4664816a95eae606293eae9"
diff --git a/.env.rc1 b/.env.rc1
index a14aa4fc908..1a05000ea38 100644
--- a/.env.rc1
+++ b/.env.rc1
@@ -93,6 +93,7 @@ AZURE_ORACLE_WESTUS_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_WESTEUROPE_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_WESTEUROPE_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -109,6 +110,7 @@ AZURE_ORACLE_WESTEUROPE_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTEUROPE_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_EASTUS2_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_EASTUS2_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -125,6 +127,7 @@ AZURE_ORACLE_EASTUS2_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_EASTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_EASTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_EASTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_EASTUS2_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Forno ----
 
@@ -154,7 +157,7 @@ GCP_FORNO_US_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-west1
 GCP_FORNO_US_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_US_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_US_EAST1_GCP_ZONE=us-east1-b
@@ -170,7 +173,7 @@ GCP_FORNO_US_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-east1
 GCP_FORNO_US_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_ASIA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_ASIA_EAST1_GCP_ZONE=asia-east1-a
@@ -186,6 +189,7 @@ GCP_FORNO_ASIA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-asia-east1
 GCP_FORNO_ASIA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
+GCP_FORNO_ASIA_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_EUROPE_WEST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_EUROPE_WEST1_GCP_ZONE=europe-west1-b
@@ -201,7 +205,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_ZONE=southamerica-east1-a
@@ -217,7 +221,7 @@ GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-southameric
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Private txnodes context ----
 
@@ -230,6 +234,7 @@ GCP_PRIVATE_TXNODES_FULL_NODES_DISK_SIZE=300
 GCP_PRIVATE_TXNODES_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3,txpool,debug"
 GCP_PRIVATE_TXNODES_FULL_NODES_GETH_GC_MODE="archive"
 GCP_PRIVATE_TXNODES_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_PRIVATE_TXNODES_FULL_NODES_DEPLOY_NGINX_SIDECAR=true
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-f4a55e143932ea559cf4bcbd9bcccc14da43d6ed"
diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts
index 238d38ed7fc..390e9b37c7e 100644
--- a/packages/celotool/src/lib/env-utils.ts
+++ b/packages/celotool/src/lib/env-utils.ts
@@ -181,6 +181,7 @@ export enum DynamicEnvVar {
   AZURE_REGION_NAME = '{{ context }}_AZURE_REGION_NAME',
   AZURE_TENANT_ID = '{{ context }}_AZURE_TENANT_ID',
   FULL_NODES_COUNT = '{{ context }}_FULL_NODES_COUNT',
+  FULL_NODES_DEPLOY_NGINX = '{{ context }}_FULL_NODES_DEPLOY_NGINX',
   FULL_NODES_DISK_SIZE = '{{ context }}_FULL_NODES_DISK_SIZE',
   FULL_NODES_GETH_GC_MODE = '{{ context }}_FULL_NODES_GETH_GC_MODE',
   FULL_NODES_NODEKEY_DERIVATION_STRING = '{{ context }}_FULL_NODES_NODEKEY_DERIVATION_STRING',
diff --git a/packages/celotool/src/lib/fullnodes.ts b/packages/celotool/src/lib/fullnodes.ts
index 044311867be..60c56a51cfc 100644
--- a/packages/celotool/src/lib/fullnodes.ts
+++ b/packages/celotool/src/lib/fullnodes.ts
@@ -27,6 +27,7 @@ const contextFullNodeDeploymentEnvVars: {
   rpcApis: DynamicEnvVar.FULL_NODES_RPC_API_METHODS,
   gcMode: DynamicEnvVar.FULL_NODES_GETH_GC_MODE,
   useGstoreData: DynamicEnvVar.FULL_NODES_USE_GSTORAGE_DATA,
+  deployNginxSidecar: DynamicEnvVar.FULL_NODES_DEPLOY_NGINX,
 }
 
 /**
@@ -156,6 +157,7 @@ function getFullNodeDeploymentConfig(context: string): BaseFullNodeDeploymentCon
     rpcApis: fullNodeDeploymentEnvVarValues.rpcApis,
     gcMode: fullNodeDeploymentEnvVarValues.gcMode,
     useGstoreData: fullNodeDeploymentEnvVarValues.useGstoreData,
+    deployNginxSidecar: fullNodeDeploymentEnvVarValues.deployNginxSidecar,
   }
   return fullNodeDeploymentConfig
 }
diff --git a/packages/celotool/src/lib/k8s-fullnode/base.ts b/packages/celotool/src/lib/k8s-fullnode/base.ts
index 6dc19b704f1..0b12a2b3270 100644
--- a/packages/celotool/src/lib/k8s-fullnode/base.ts
+++ b/packages/celotool/src/lib/k8s-fullnode/base.ts
@@ -27,6 +27,7 @@ export interface BaseFullNodeDeploymentConfig {
   rpcApis: string
   gcMode: string
   useGstoreData: string
+  deployNginxSidecar: string
   // If undefined, node keys will not be predetermined and will be random
   nodeKeyGenerationInfo?: NodeKeyGenerationInfo
 }
@@ -117,6 +118,7 @@ export abstract class BaseFullNodeDeployer {
       `--set genesis.epoch_size=${fetchEnv(envVar.EPOCH)}`,
       `--set geth.use_gstorage_data=${this._deploymentConfig.useGstoreData}`,
       `--set geth.gstorage_data_bucket=${fetchEnvOrFallback('GSTORAGE_DATA_BUCKET', '')}`,
+      `--set geth.enable_nginx=${this._deploymentConfig.deployNginxSidecar}`,
       // Disable by default block age check in fullnode readinessProbe except for production envs
       `--set geth.fullnodeCheckBlockAge=${fetchEnvOrFallback(
         envVar.FULL_NODE_READINESS_CHECK_BLOCK_AGE,

From 2b2d1ddb978a41f5c9340f5a56350d8fcb92670f Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Mon, 21 Jun 2021 16:01:09 +0200
Subject: [PATCH 4/6] Use nginx proxy geth service for blockscout

---
 packages/celotool/src/lib/blockscout.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/celotool/src/lib/blockscout.ts b/packages/celotool/src/lib/blockscout.ts
index 956e3c79b4a..3b0388ce2af 100644
--- a/packages/celotool/src/lib/blockscout.ts
+++ b/packages/celotool/src/lib/blockscout.ts
@@ -140,12 +140,13 @@ async function helmParameters(
       `--set blockscout.jsonrpc_ws_url=${fetchEnv(envVar.BLOCKSCOUT_OVERRIDE_WS_ENDPOINT)}`
     )
   } else if (isVmBased()) {
+    // TODO: Deprecated. Remove soon.
     const txNodeLbIp = await getInternalTxNodeLoadBalancerIP(celoEnv)
     params.push(`--set blockscout.jsonrpc_http_url=http://${txNodeLbIp}:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://${txNodeLbIp}:8546`)
   } else if (privateNodes > 0) {
-    params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes-private:8545`)
-    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:8546`)
+    params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes-private:3000/rpc`)
+    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:3000/ws`)
   } else {
     params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes:8546`)

From f4bf283fdd1eab06c44853c0f532d1d57ae55c4d Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Mon, 21 Jun 2021 16:26:27 +0200
Subject: [PATCH 5/6] Update blockscout rc1 rpc/ws endpoints

---
 .env.rc1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.env.rc1 b/.env.rc1
index 1a05000ea38..88d1df407bf 100644
--- a/.env.rc1
+++ b/.env.rc1
@@ -27,8 +27,8 @@ BLOCKSCOUT_SUBNETWORK_NAME="Celo"
 BLOCKSCOUT_METADATA_CRAWLER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 BLOCKSCOUT_METADATA_CRAWLER_IMAGE_TAG="metadata-crawler-007f29dc8ebf837543e6e28a14818076e88c69cd"
 BLOCKSCOUT_METADATA_CRAWLER_SCHEDULE="0 */2 * * *"
-BLOCKSCOUT_OVERRIDE_RPC_ENDPOINT="http://rc1-fullnodes-rpc.rc1.svc.cluster.local:8545"
-BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:8546"
+BLOCKSCOUT_OVERRIDE_RPC_ENDPOINT="http://rc1-fullnodes-rpc.rc1.svc.cluster.local:3000/rpc"
+BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:3000/ws"
 METADATA_CRAWLER_DISCORD_CLUSTER_NAME="Mainnet"
 
 CELOSTATS_SERVER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celostats-server"

From ad82de9d8c946643fe611ca7c6d583c36163674a Mon Sep 17 00:00:00 2001
From: Javier Cortejoso <javier.cortejoso@gmail.com>
Date: Tue, 22 Jun 2021 11:17:43 +0200
Subject: [PATCH 6/6] Fix testnet nginx configmap

---
 .../helm-charts/testnet/templates/nginx-geth-configmap.yaml     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml b/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml
index f18f1cb132c..851e8b05fb9 100644
--- a/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml
+++ b/packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml
@@ -1,3 +1,3 @@
-{{- if .Values.geth.enable_nginx | default false }}
+{{- if gt .Values.geth.private_tx_nodes 0 | default true }}
 {{ include "common.nginx-geth-configmap" . }}
 {{- end }}