Allow setting up a different Geth WS port. Private tx-nodes defualts to 8545

.env.alfajores

@@ -51,6 +51,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_ROLLING_UPDATE_PARTITION=0
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_WS_PORT="8546"
 
 # Temporarily point to celo-org repository to consume patched image.
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-org/geth"

.env.baklava

@@ -73,6 +73,7 @@ AZURE_ORACLE_WESTUS2_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_WESTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS2_FULL_NODES_WS_PORT="8546"
 
 AZURE_ORACLE_CENTRALUS_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_CENTRALUS_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -90,6 +91,7 @@ AZURE_ORACLE_CENTRALUS_FULL_NODES_DISK_SIZE=30
 AZURE_ORACLE_CENTRALUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_CENTRALUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_CENTRALUS_FULL_NODES_WS_PORT="8546"
 
 # ---- Forno ----
 
@@ -118,6 +120,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_WS_PORT="8546"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-dc5e5dfa07231a4ff4664816a95eae606293eae9"

.env.rc1

@@ -28,7 +28,7 @@ BLOCKSCOUT_METADATA_CRAWLER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 BLOCKSCOUT_METADATA_CRAWLER_IMAGE_TAG="metadata-crawler-007f29dc8ebf837543e6e28a14818076e88c69cd"
 BLOCKSCOUT_METADATA_CRAWLER_SCHEDULE="0 */2 * * *"
 BLOCKSCOUT_OVERRIDE_RPC_ENDPOINT="http://rc1-fullnodes-rpc.rc1.svc.cluster.local:8545"
-BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:8546"
+BLOCKSCOUT_OVERRIDE_WS_ENDPOINT="ws://rc1-fullnodes-rpc.rc1.svc.cluster.local:8545"
 METADATA_CRAWLER_DISCORD_CLUSTER_NAME="Mainnet"
 
 CELOSTATS_SERVER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celostats-server"
@@ -93,6 +93,7 @@ AZURE_ORACLE_WESTUS_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTUS_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTUS_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTUS_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTUS_FULL_NODES_WS_PORT="8546"
 
 AZURE_ORACLE_WESTEUROPE_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_WESTEUROPE_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -109,6 +110,7 @@ AZURE_ORACLE_WESTEUROPE_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_WESTEUROPE_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_WESTEUROPE_FULL_NODES_WS_PORT="8546"
 
 AZURE_ORACLE_EASTUS2_AZURE_SUBSCRIPTION_ID=7a6f5f20-bd43-4267-8c35-a734efca140c
 AZURE_ORACLE_EASTUS2_AZURE_TENANT_ID=7cb7628a-e37c-4afb-8332-2029e418980e
@@ -125,6 +127,7 @@ AZURE_ORACLE_EASTUS2_FULL_NODES_DISK_SIZE=100
 AZURE_ORACLE_EASTUS2_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 AZURE_ORACLE_EASTUS2_FULL_NODES_GETH_GC_MODE="full"
 AZURE_ORACLE_EASTUS2_FULL_NODES_USE_GSTORAGE_DATA=false
+AZURE_ORACLE_EASTUS2_FULL_NODES_WS_PORT="8546"
 
 # ---- Forno ----
 
@@ -154,7 +157,7 @@ GCP_FORNO_US_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-west1
 GCP_FORNO_US_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_WEST1_FULL_NODES_WS_PORT="8546"
 
 GCP_FORNO_US_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_US_EAST1_GCP_ZONE=us-east1-b
@@ -170,7 +173,7 @@ GCP_FORNO_US_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-us-east1
 GCP_FORNO_US_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_US_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_US_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_US_EAST1_FULL_NODES_WS_PORT="8546"
 
 GCP_FORNO_ASIA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_ASIA_EAST1_GCP_ZONE=asia-east1-a
@@ -186,6 +189,7 @@ GCP_FORNO_ASIA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-asia-east1
 GCP_FORNO_ASIA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_ASIA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
+GCP_FORNO_ASIA_EAST1_FULL_NODES_WS_PORT="8546"
 
 GCP_FORNO_EUROPE_WEST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_EUROPE_WEST1_GCP_ZONE=europe-west1-b
@@ -201,7 +205,7 @@ GCP_FORNO_EUROPE_WEST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-europe-west1
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_EUROPE_WEST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_EUROPE_WEST1_FULL_NODES_WS_PORT="8546"
 
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_PROJECT_NAME=celo-testnet-production
 GCP_FORNO_SOUTHAMERICA_EAST1_GCP_ZONE=southamerica-east1-a
@@ -217,7 +221,7 @@ GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_STATIC_NODES_FILE_SUFFIX=gcp-southameric
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_GETH_GC_MODE="full"
 GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_USE_GSTORAGE_DATA=true
-
+GCP_FORNO_SOUTHAMERICA_EAST1_FULL_NODES_WS_PORT="8546"
 
 # ---- Private txnodes context ----
 
@@ -230,6 +234,7 @@ GCP_PRIVATE_TXNODES_FULL_NODES_DISK_SIZE=300
 GCP_PRIVATE_TXNODES_FULL_NODES_RPC_API_METHODS="eth,net,rpc,web3,txpool,debug"
 GCP_PRIVATE_TXNODES_FULL_NODES_GETH_GC_MODE="archive"
 GCP_PRIVATE_TXNODES_FULL_NODES_USE_GSTORAGE_DATA=false
+GCP_PRIVATE_TXNODES_FULL_NODES_WS_PORT="8545"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-f4a55e143932ea559cf4bcbd9bcccc14da43d6ed"

packages/celotool/src/lib/blockscout.ts

@@ -140,12 +140,13 @@ async function helmParameters(
       `--set blockscout.jsonrpc_ws_url=${fetchEnv(envVar.BLOCKSCOUT_OVERRIDE_WS_ENDPOINT)}`
     )
   } else if (isVmBased()) {
+    // TODO: Deprecated
     const txNodeLbIp = await getInternalTxNodeLoadBalancerIP(celoEnv)
     params.push(`--set blockscout.jsonrpc_http_url=http://${txNodeLbIp}:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://${txNodeLbIp}:8546`)
   } else if (privateNodes > 0) {
     params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes-private:8545`)
-    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:8546`)
+    params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes-private:8545`)
   } else {
     params.push(`--set blockscout.jsonrpc_http_url=http://tx-nodes:8545`)
     params.push(`--set blockscout.jsonrpc_ws_url=ws://tx-nodes:8546`)

packages/celotool/src/lib/env-utils.ts

@@ -188,6 +188,7 @@ export enum DynamicEnvVar {
   FULL_NODES_RPC_API_METHODS = '{{ context }}_FULL_NODES_RPC_API_METHODS',
   FULL_NODES_STATIC_NODES_FILE_SUFFIX = '{{ context }}_FULL_NODES_STATIC_NODES_FILE_SUFFIX',
   FULL_NODES_USE_GSTORAGE_DATA = '{{ context }}_FULL_NODES_USE_GSTORAGE_DATA',
+  FULL_NODES_WS_PORT = '{{ context }}_FULL_NODES_WS_PORT',
   GCP_PROJECT_NAME = '{{ context }}_GCP_PROJECT_NAME',
   GCP_ZONE = '{{ context }}_GCP_ZONE',
   KUBERNETES_CLUSTER_NAME = '{{ context }}_KUBERNETES_CLUSTER_NAME',

packages/celotool/src/lib/fullnodes.ts

@@ -26,6 +26,7 @@ const contextFullNodeDeploymentEnvVars: {
   rollingUpdatePartition: DynamicEnvVar.FULL_NODES_ROLLING_UPDATE_PARTITION,
   rpcApis: DynamicEnvVar.FULL_NODES_RPC_API_METHODS,
   gcMode: DynamicEnvVar.FULL_NODES_GETH_GC_MODE,
+  wsPort: DynamicEnvVar.FULL_NODES_WS_PORT,
   useGstoreData: DynamicEnvVar.FULL_NODES_USE_GSTORAGE_DATA,
 }
 
@@ -155,6 +156,7 @@ function getFullNodeDeploymentConfig(context: string): BaseFullNodeDeploymentCon
     rollingUpdatePartition: parseInt(fullNodeDeploymentEnvVarValues.rollingUpdatePartition, 10),
     rpcApis: fullNodeDeploymentEnvVarValues.rpcApis,
     gcMode: fullNodeDeploymentEnvVarValues.gcMode,
+    wsPort: parseInt(fullNodeDeploymentEnvVarValues.wsPort, 10),
     useGstoreData: fullNodeDeploymentEnvVarValues.useGstoreData,
   }
   return fullNodeDeploymentConfig

packages/celotool/src/lib/k8s-fullnode/base.ts

@@ -27,6 +27,7 @@ export interface BaseFullNodeDeploymentConfig {
   rpcApis: string
   gcMode: string
   useGstoreData: string
+  wsPort: number
   // If undefined, node keys will not be predetermined and will be random
   nodeKeyGenerationInfo?: NodeKeyGenerationInfo
 }
@@ -116,6 +117,7 @@ export abstract class BaseFullNodeDeployer {
       `--set genesis.network=${this.celoEnv}`,
       `--set genesis.epoch_size=${fetchEnv(envVar.EPOCH)}`,
       `--set geth.use_gstorage_data=${this._deploymentConfig.useGstoreData}`,
+      `--set geth.ws_port=${this._deploymentConfig.wsPort}`,
       `--set geth.gstorage_data_bucket=${fetchEnvOrFallback('GSTORAGE_DATA_BUCKET', '')}`,
       // Disable by default block age check in fullnode readinessProbe except for production envs
       `--set geth.fullnodeCheckBlockAge=${fetchEnvOrFallback(

packages/helm-charts/celo-fullnode/templates/statefulset.yaml

@@ -83,7 +83,7 @@ spec:
 {{ include "common.full-node-container" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "expose" true "ip_addresses" .Values.geth.public_ip_per_node "syncmode" .Values.geth.syncmode "gcmode" .Values.geth.gcmode "rpc_apis" .Values.geth.rpc_apis "pprof" (or (.Values.prometheus) (.Values.pprof.enabled)) "pprof_port" (.Values.pprof.port) "metrics" (or (.Values.prometheus) (.Values.metrics)) "ports" .Values.geth.service_node_port_per_full_node "light_serve" .Values.geth.light.serve "light_maxpeers" .Values.geth.light.maxpeers "maxpeers" .Values.geth.maxpeers "geth_flags" (.Values.geth.flags | default "") "extra_setup" ((include "celo-fullnode.aws-subnet-specific-nat-ip" .) | indent 4)) | indent 6 }}
 {{ if .Values.geth.create_network_endpoint_group }}
 {{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "http" "tcp_check_port" 8545 "server_port" 6000) | indent 6 }}
-{{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "ws" "tcp_check_port" 8546 "server_port" 6001) | indent 6 }}
+{{ include "celo-fullnode.health-checker-server" (dict "protocol_name" "ws" "tcp_check_port" .Values.geth.ws_port "server_port" 6001) | indent 6 }}
 {{ end }}
       terminationGracePeriodSeconds: {{ .Values.geth.terminationGracePeriodSeconds | default 300 }}
       volumes:

packages/helm-charts/celo-fullnode/values.yaml

@@ -93,6 +93,7 @@ geth:
     maxpeers: 1000
     serve: 90
   fullnodeCheckBlockAge: true
+  ws_port: 8546
 
 genesis:
   networkId: 31418

packages/helm-charts/common/templates/_helpers.tpl

@@ -142,7 +142,8 @@ fi
     {{- end }}
     {{- if .expose }}
     RPC_APIS="{{ .rpc_apis | default "eth,net,web3,debug,txpool" }}"
-    ADDITIONAL_FLAGS="${ADDITIONAL_FLAGS} --rpc --rpcaddr 0.0.0.0 --rpcapi=${RPC_APIS} --rpccorsdomain='*' --rpcvhosts=* --ws --wsaddr 0.0.0.0 --wsorigins=* --wsapi=${RPC_APIS}"
+    ADDITIONAL_FLAGS="${ADDITIONAL_FLAGS} --rpc --rpcaddr 0.0.0.0 --rpcapi=${RPC_APIS} --rpccorsdomain='*' --rpcvhosts=*"
+    ADDITIONAL_FLAGS="${ADDITIONAL_FLAGS} --ws --wsaddr 0.0.0.0 --wsorigins=* --wsapi=${RPC_APIS} --wsport={{ default .Values.geth.ws_port .ws_port }}"
     {{- end }}
     {{- if .ping_ip_from_packet | default false }}
     ADDITIONAL_FLAGS="${ADDITIONAL_FLAGS} --ping-ip-from-packet"
@@ -252,7 +253,7 @@ fi
   - name: rpc
     containerPort: 8545
   - name: ws
-    containerPort: 8546
+    containerPort: {{ default .Values.geth.ws_port .ws_port }}
 {{ end }}
 {{- if .pprof }}
   - name: pprof

packages/helm-charts/common/values.yaml

@@ -26,6 +26,7 @@ geth:
   use_static_ips: true
   azure_mixed_protocols: true
   fullnodeCheckBlockAge: true
+  ws_port: 8546
 
 # Genesis
 genesis:

packages/helm-charts/testnet/templates/_helpers.tpl

@@ -57,7 +57,7 @@ spec:
   ports:
   - port: 8545
     name: rpc
-  - port: 8546
+  - port: {{ .ws_port | default .Values.geth.ws_port }}
     name: ws
   selector:
 {{- if .proxy | default false }}
@@ -82,7 +82,7 @@ spec:
   ports:
   - port: 8545
     name: rpc
-  - port: 8546
+  - port: {{ .ws_port | default .Values.geth.ws_port }}
     name: ws
   selector:
 {{- if .proxy | default false }}
@@ -150,7 +150,7 @@ spec:
 {{ include "common.import-geth-account-container" .  | indent 6 }}
 {{ end }}
       containers:
-{{ include "common.full-node-container" (dict "Values" .Values "Release" .Release "Chart" .Chart "proxy" .proxy "proxy_allow_private_ip_flag" .proxy_allow_private_ip_flag "unlock" .unlock "expose" .expose "syncmode" .syncmode "gcmode" .gcmode "pprof" (or (.Values.metrics) (.Values.pprof.enabled)) "pprof_port" (.Values.pprof.port) "metrics" .Values.metrics "public_ips" .public_ips "ethstats" (printf "%s-ethstats.%s" (include "common.fullname" .) .Release.Namespace))  | indent 6 }}
+{{ include "common.full-node-container" (dict "Values" .Values "Release" .Release "Chart" .Chart "proxy" .proxy "proxy_allow_private_ip_flag" .proxy_allow_private_ip_flag "unlock" .unlock "expose" .expose "syncmode" .syncmode "gcmode" .gcmode "ws_port" (default .Values.geth.ws_port .ws_port) "pprof" (or (.Values.metrics) (.Values.pprof.enabled)) "pprof_port" (.Values.pprof.port) "metrics" .Values.metrics "public_ips" .public_ips "ethstats" (printf "%s-ethstats.%s" (include "common.fullname" .) .Release.Namespace))  | indent 6 }}
       terminationGracePeriodSeconds:  {{ .Values.geth.terminationGracePeriodSeconds | default 300 }}
       volumes:
       - name: data

packages/helm-charts/testnet/templates/txnode-private.statefulset.yaml

@@ -1 +1 @@
-{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses)) }}
+{{ include "celo.full-node-statefulset" (dict "Values" $.Values "Release" $.Release "Chart" $.Chart "name" "tx-nodes-private" "component_label" "tx_nodes_private" "replicas" .Values.geth.private_tx_nodes "mnemonic_account_type" "tx_node_private" "expose" true "syncmode" "full" "gcmode" "archive" "rpc_apis" "eth,net,web3,debug,txpool" "ws_port" "8545" "ip_addresses" (join "/" .Values.geth.private_tx_node_ip_addresses)) }}

packages/helm-charts/testnet/values.yaml

@@ -37,6 +37,7 @@ geth:
     #   memory: "4Gi"
     #   cpu: "4"
   fullnodeCheckBlockAge: true
+  ws_port: 8546
 
 # UpdateStrategy for statefulsets only. Partition=0 is default rollingUpdate behaviour.
 updateStrategy: