Add nginx sidecar to geth

.env.alfajores

@@ -51,6 +51,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_ROLLING_UPDATE_PARTITION=0
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # Temporarily point to celo-org repository to consume patched image.
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-org/geth"

.env.baklava

@@ -73,6 +73,7 @@ AZURE_ORACLE_WESTUS2_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_WESTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS2_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_CENTRALUS_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_CENTRALUS_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -90,6 +91,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Forno ----
 
@@ -118,6 +120,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-dc5e5dfa07231a4ff4664816a95eae606293eae9"

.env.rc1

@@ -27,8 +27,8 @@ BLOCKSCOUT_SUBNETWORK_NAME="Celo"
 BLOCKSCOUT_METADATA_CRAWLER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 BLOCKSCOUT_METADATA_CRAWLER_IMAGE_TAG="metadata-crawler-007f29dc8ebf837543e6e28a14818076e88c69cd"
 BLOCKSCOUT_METADATA_CRAWLER_SCHEDULE="0 */2 * * *"
-BLOCKSCOUT_OVERRIDE_RPC_ENDPOINT="http://rc1-fullnodes-rpc.rc1.svc.cluster.local:8545"
-BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:8546"
+BLOCKSCOUT_OVERRIDE_RPC_ENDPOINT="http://rc1-fullnodes-rpc.rc1.svc.cluster.local:3000/rpc"
+BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:3000/ws"
 METADATA_CRAWLER_DISCORD_CLUSTER_NAME="Mainnet"
 
 CELOSTATS_SERVER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celostats-server"
@@ -93,6 +93,7 @@ AZURE_ORACLE_WESTUS_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_WESTEUROPE_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_WESTEUROPE_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -109,6 +110,7 @@ AZURE_ORACLE_WESTEUROPE_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTEUROPE_DEPLOY_NGINX_SIDECAR=false
 
 AZURE_ORACLE_EASTUS2_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_EASTUS2_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -125,6 +127,7 @@ AZURE_ORACLE_EASTUS2_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_EASTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_EASTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_EASTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_EASTUS2_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Forno ----
 
@@ -154,7 +157,7 @@ GCP_FORNO_US_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-west1
 GCP_FORNO_US_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_US_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_US_EAST1_GCP_ZONE=us-east1-b
@@ -170,7 +173,7 @@ GCP_FORNO_US_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-east1
 GCP_FORNO_US_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_ASIA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_ASIA_EAST1_GCP_ZONE=asia-east1-a
@@ -186,6 +189,7 @@ GCP_FORNO_ASIA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-asia-east1
 GCP_FORNO_ASIA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
+GCP_FORNO_ASIA_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_EUROPE_WEST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_EUROPE_WEST1_GCP_ZONE=europe-west1-b
@@ -201,7 +205,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_ZONE=southamerica-east1-a
@@ -217,7 +221,7 @@ GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-southameric
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_DEPLOY_NGINX_SIDECAR=false
 
 # ---- Private txnodes context ----
 
@@ -230,6 +234,7 @@ GCP_PRIVATE_TXNODES_FULL_NODES_DISK_SIZE=300
 GCP_PRIVATE_TXNODES_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3,txpool,debug"
 GCP_PRIVATE_TXNODES_FULL_NODES_GETH_GC_MODE="archive"
 GCP_PRIVATE_TXNODES_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_PRIVATE_TXNODES_FULL_NODES_DEPLOY_NGINX_SIDECAR=true
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-f4a55e143932ea559cf4bcbd9bcccc14da43d6ed"

packages/celotool/src/lib/blockscout.ts

@@ -140,12 +140,13 @@ async function helmParameters(
       `--set blockscout.jsonrpc_ws_url=${fetchEnv(envVar.BLOCKSCOUT_OVERRIDE_WS_ENDPOINT)}`
     )
   } else if (isVmBased()) {
+    // TODO: Deprecated. Remove soon.
     const txNodeLbIp = await getInternalTxNodeLoadBalancerIP(celoEnv)
     params.push(`--set blockscout.jsonrpc_http_url=http://${txNodeLbIp}:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://${txNodeLbIp}:8546`)
   } else if (privateNodes > 0) {
-    params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes-private:8545`)
-    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:8546`)
+    params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes-private:3000/rpc`)
+    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:3000/ws`)
   } else {
     params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes:8546`)

packages/celotool/src/lib/env-utils.ts

@@ -181,6 +181,7 @@ export enum DynamicEnvVar {
   AZURE_REGION_NAME = '{{ context }}_AZURE_REGION_NAME',
   AZURE_TENANT_ID = '{{ context }}_AZURE_TENANT_ID',
   FULL_NODES_COUNT = '{{ context }}_FULL_NODES_COUNT',
+  FULL_NODES_DEPLOY_NGINX = '{{ context }}_FULL_NODES_DEPLOY_NGINX',
   FULL_NODES_DISK_SIZE = '{{ context }}_FULL_NODES_DISK_SIZE',
   FULL_NODES_GETH_GC_MODE = '{{ context }}_FULL_NODES_GETH_GC_MODE',
   FULL_NODES_NODEKEY_DERIVATION_STRING = '{{ context }}_FULL_NODES_NODEKEY_DERIVATION_STRING',

packages/celotool/src/lib/fullnodes.ts

@@ -27,6 +27,7 @@ const contextFullNodeDeploymentEnvVars: {
   rpcApis: DynamicEnvVar.FULL_NODES_RPC_API_METHODS,
   gcMode: DynamicEnvVar.FULL_NODES_GETH_GC_MODE,
   useGstoreData: DynamicEnvVar.FULL_NODES_USE_GSTORAGE_DATA,
+  deployNginxSidecar: DynamicEnvVar.FULL_NODES_DEPLOY_NGINX,
 }
 
 /**
@@ -156,6 +157,7 @@ function getFullNodeDeploymentConfig(context: string): BaseFullNodeDeploymentCon
     rpcApis: fullNodeDeploymentEnvVarValues.rpcApis,
     gcMode: fullNodeDeploymentEnvVarValues.gcMode,
     useGstoreData: fullNodeDeploymentEnvVarValues.useGstoreData,
+    deployNginxSidecar: fullNodeDeploymentEnvVarValues.deployNginxSidecar,
   }
   return fullNodeDeploymentConfig
 }

packages/celotool/src/lib/k8s-fullnode/base.ts

@@ -27,6 +27,7 @@ export interface BaseFullNodeDeploymentConfig {
   rpcApis: string
   gcMode: string
   useGstoreData: string
+  deployNginxSidecar: string
   // If undefined, node keys will not be predetermined and will be random
   nodeKeyGenerationInfo?: NodeKeyGenerationInfo
 }
@@ -117,6 +118,7 @@ export abstract class BaseFullNodeDeployer {
       `--set genesis.epoch_size=${fetchEnv(envVar.EPOCH)}`,
       `--set geth.use_gstorage_data=${this._deploymentConfig.useGstoreData}`,
       `--set geth.gstorage_data_bucket=${fetchEnvOrFallback('GSTORAGE_DATA_BUCKET', '')}`,
+      `--set geth.enable_nginx=${this._deploymentConfig.deployNginxSidecar}`,
       // Disable by default block age check in fullnode readinessProbe except for production envs
       `--set geth.fullnodeCheckBlockAge=${fetchEnvOrFallback(
         envVar.FULL_NODE_READINESS_CHECK_BLOCK_AGE,

packages/helm-charts/celo-fullnode/templates/_helpers.tpl

@@ -40,6 +40,12 @@ Create chart name and version as used by the chart label.
   targetPort: ws
   protocol: TCP
   name: ws
+{{- if .Values.geth.enable_nginx | default false }}
+- port: 3000
+  targetPort: nginx
+  protocol: TCP
+  name: nginx
+{{- end }}
 {{- end -}}
 
 {{/*

packages/helm-charts/celo-fullnode/templates/nginx-geth-configmap.yaml

@@ -0,0 +1,3 @@
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-configmap" . }}
+{{- end }}

packages/helm-charts/celo-fullnode/templates/statefulset.yaml

@@ -85,6 +85,9 @@ spec:
 {{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "http" "tcp_check_port" 8545 "server_port" 6000) | indent 6 }}
 {{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "ws" "tcp_check_port" 8546 "server_port" 6001) | indent 6 }}
 {{ end }}
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-sidecar" . | indent 6 }}
+{{- end }}
       terminationGracePeriodSeconds: 120 # 2 mins
       volumes:
       - name: data
@@ -93,3 +96,6 @@ spec:
         configMap:
           name: {{ template "common.fullname" . }}-health-check
           defaultMode: 0555
+{{- if .Values.geth.enable_nginx | default false }}
+{{ include "common.nginx-geth-sidecar-volume" . | indent 6 }}
+{{- end }}

packages/helm-charts/celo-fullnode/values.yaml

@@ -93,6 +93,7 @@ geth:
     maxpeers: 1000
     serve: 90
   fullnodeCheckBlockAge: true
+  enable_nginx: true
 
 genesis:
   networkId: 31418

packages/helm-charts/common/templates/_helpers.tpl

@@ -530,3 +530,97 @@ prometheus.io/port: "{{ $pprof.port | default 6060 }}"
   - name: data
     mountPath: /root/.celo
 {{- end -}}
+
+{{- define "common.nginx-geth-sidecar" -}}
+- name: nginx
+  image: nginx:latest
+  imagePullPolicy: IfNotPresent
+  ports:
+  - containerPort: 3000
+    name: nginx
+    protocol: TCP
+  volumeMounts:
+  - name: nginx-geth-config
+    mountPath: /etc/nginx/nginx.conf
+    subPath: nginx.conf
+{{- end -}}
+
+{{- define "common.nginx-geth-sidecar-volume" -}}
+- name: nginx-geth-config
+  configMap:
+    name: {{ template "common.fullname" . }}-nginx-geth-config
+{{- end -}}
+
+{{- define "common.nginx-geth-configmap" -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "common.fullname" . }}-nginx-geth-config
+  labels:
+{{ include "common.standard.labels" .  | indent 4 }}
+data:
+  nginx.conf: |-
+    # /etc/nginx/nginx.conf
+    user nginx;
+    # Set number of worker processes automatically based on number of CPU cores.
+    worker_processes auto;
+    # Enables the use of JIT for regular expressions to speed-up their processing.
+    pcre_jit on;
+    # Configures default error logger.
+    error_log /var/log/nginx/error.log warn;
+    # Includes files with directives to load dynamic modules.
+    include /etc/nginx/modules/*.conf;
+    events {
+        # The maximum number of simultaneous connections that can be opened by
+        # a worker process.
+        worker_connections 1024;
+    }
+    http {
+    server {
+        listen 3000;
+        server_name tx-nodes-private;
+
+        location / {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://ws-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+
+        location /rpc {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://rpc-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+
+        location /ws {
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+
+        proxy_pass http://ws-backend;
+
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        }
+    }
+    upstream rpc-backend {
+        server localhost:8545;
+    }
+
+    upstream ws-backend {
+        server localhost:8546;
+    }
+    }
+    # TIP: Uncomment if you use stream module.
+    #include /etc/nginx/stream.conf;
+{{- end -}}

packages/helm-charts/testnet/templates/_helpers.tpl

@@ -59,6 +59,10 @@ spec:
     name: rpc
   - port: 8546
     name: ws
+{{- if .enable_nginx_geth | default false }}
+  - port: 3000
+    name: nginx
+{{- end }}
   selector:
 {{- if .proxy | default false }}
 {{- $validatorProxied := printf "%s-validators-%d" .Release.Namespace .validator_index }}
@@ -151,6 +155,9 @@ spec:
 {{ end }}
       containers:
 {{ include "common.full-node-container" (dict "Values" .Values "Release" .Release "Chart" .Chart "proxy" .proxy "proxy_allow_private_ip_flag" .proxy_allow_private_ip_flag "unlock" .unlock "expose" .expose "syncmode" .syncmode "gcmode" .gcmode "pprof" (or (.Values.metrics) (.Values.pprof.enabled)) "pprof_port" (.Values.pprof.port) "metrics" .Values.metrics "public_ips" .public_ips "ethstats" (printf "%s-ethstats.%s" (include "common.fullname" .) .Release.Namespace))  | indent 6 }}
+{{- if .enable_nginx_geth | default false }}
+{{ include "common.nginx-geth-sidecar" . | indent 6 }}
+{{- end }}
       terminationGracePeriodSeconds: 120 # 2 mins
       volumes:
       - name: data
@@ -161,6 +168,9 @@ spec:
       - name: account
         secret:
           secretName: {{ template "common.fullname" . }}-geth-account
+{{- if .enable_nginx_geth | default false }}
+{{ include "common.nginx-geth-sidecar-volume" . | indent 6 }}
+{{- end }}
 {{- end -}}
 
 {{- /* This template puts a semicolon-separated pair of proxy enodes into $PROXY_ENODE_URL_PAIR. */ -}}

packages/helm-charts/testnet/templates/nginx-geth-configmap.yaml

@@ -0,0 +1,3 @@
+{{- if gt .Values.geth.private_tx_nodes 0 | default true }}
+{{ include "common.nginx-geth-configmap" . }}
+{{- end }}

packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml

@@ -1 +1 @@
-{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses)) }}
+{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses) "enable_nginx_geth" true) }}