diff --git a/.github/tests/mix-public.yaml b/.github/tests/mix-public.yaml new file mode 100644 index 000000000..01fe74a7b --- /dev/null +++ b/.github/tests/mix-public.yaml @@ -0,0 +1,18 @@ +participants: + - el_type: geth + cl_type: teku + - el_type: nethermind + cl_type: prysm + - el_type: erigon + cl_type: nimbus + - el_type: besu + cl_type: lighthouse + - el_type: reth + cl_type: lodestar + - el_type: nimbus + cl_type: teku + - el_type: ethereumjs + cl_type: grandine +additional_services: [] +port_publisher: + public_port_start: 50000 diff --git a/.github/tests/mix.yaml b/.github/tests/mix.yaml index 04c945aa1..b79ebdc15 100644 --- a/.github/tests/mix.yaml +++ b/.github/tests/mix.yaml @@ -13,4 +13,4 @@ participants: cl_type: teku - el_type: ethereumjs cl_type: grandine -additional_services: [] +additional_services: [] \ No newline at end of file diff --git a/.github/workflows/per-pr.yml b/.github/workflows/per-pr.yml index 4fe5f5ca3..8ac427491 100644 --- a/.github/workflows/per-pr.yml +++ b/.github/workflows/per-pr.yml @@ -33,6 +33,7 @@ jobs: "./.github/tests/mev-mock.yaml", "./.github/tests/mix-with-tools.yaml", "./.github/tests/mix-persistence.yaml", + "./.github/tests/mix-public.yaml", "./network_params.yaml" ] runs-on: ubuntu-latest diff --git a/README.md b/README.md index 97973e9a1..d204813ab 100644 --- a/README.md +++ b/README.md @@ -752,6 +752,17 @@ global_node_selectors: {} # This will open up http ports to your validator services! # Defaults to false keymanager_enabled: false + +# Global paarameter to set the exit ip address of services and public ports +port_publisher: + # if you have a service that you want to expose on a specific interfact; set that IP here + # if you set it to auto it gets the public ip from ident.me and sets it + # Defaults to constants.PRIVATE_IP_ADDRESS_PLACEHOLDER + # The default value just means its the IP address of the container in which the service is running + nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER + # The start value gets used as a seed for TCP and UDP discovery ports for el/cl client + # Defaults to None - no public ports + public_port_start: None ``` #### Example configurations diff --git a/main.star b/main.star index db802750f..a5a71ab9e 100644 --- a/main.star +++ b/main.star @@ -115,6 +115,7 @@ def run(plan, args={}): global_node_selectors, keymanager_enabled, parallel_keystore_generation, + args_with_right_defaults.port_publisher, ) plan.print( diff --git a/network_params.yaml b/network_params.yaml index c02da9c53..8a8786726 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -138,3 +138,6 @@ xatu_sentry_params: global_tolerations: [] global_node_selectors: {} keymanager_enabled: false +port_publisher: + public_port_start: null + nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER diff --git a/src/blobber/blobber_launcher.star b/src/blobber/blobber_launcher.star index 0927a1190..1af3cdee2 100644 --- a/src/blobber/blobber_launcher.star +++ b/src/blobber/blobber_launcher.star @@ -1,5 +1,6 @@ shared_utils = import_module("../shared_utils/shared_utils.star") input_parser = import_module("../package_io/input_parser.star") +constants = import_module("../package_io/constants.star") cl_context = import_module("../cl/cl_context.star") blobber_context = import_module("../blobber/blobber_context.star") @@ -10,8 +11,6 @@ BLOBBER_BEACON_PORT_UDP_ID = "discovery-udp" BLOBBER_VALIDATOR_PROXY_PORT_NUM = 5000 BLOBBER_VALIDATOR_PROXY_PORT_ID = "http" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - DEFAULT_BLOBBER_IMAGE = "ethpandaops/blobber:1.1.0" VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" @@ -76,7 +75,8 @@ def get_config( "--cl={0}".format(beacon_http_url), "--validator-key-folder={0}".format(validator_root_dirpath), "--enable-unsafe-mode", - "--external-ip={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER), + # Does this get affected by public ip address changes? + "--external-ip={0}".format(constants.PRIVATE_IP_ADDRESS_PLACEHOLDER), "--validator-proxy-port-start={0}".format(BLOBBER_VALIDATOR_PROXY_PORT_NUM), ] @@ -90,7 +90,7 @@ def get_config( VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid }, cmd=cmd, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=MIN_CPU, max_cpu=MAX_CPU, min_memory=MIN_MEMORY, diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index a1c6320e1..5b4373771 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -31,6 +31,7 @@ def launch( validator_data, prysm_password_relative_filepath, prysm_password_artifact_uuid, + port_publisher, ): plan.print("Launching CL network") @@ -175,6 +176,7 @@ def launch( node_selectors, participant.use_separate_vc, participant.keymanager_enabled, + port_publisher, ) else: boot_cl_client_ctx = all_cl_contexts @@ -208,6 +210,7 @@ def launch( node_selectors, participant.use_separate_vc, participant.keymanager_enabled, + port_publisher, ) # Add participant cl additional prometheus labels diff --git a/src/cl/grandine/grandine_launcher.star b/src/cl/grandine/grandine_launcher.star index dfbdbfc1a..b2c24c951 100644 --- a/src/cl/grandine/grandine_launcher.star +++ b/src/cl/grandine/grandine_launcher.star @@ -29,22 +29,23 @@ BEACON_METRICS_PATH = "/metrics" MIN_PEERS = 1 -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -BEACON_USED_PORTS = { - BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( - BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port): + beacon_used_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( + BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return beacon_used_ports ENTRYPOINT_ARGS = ["sh", "-c"] @@ -88,6 +89,7 @@ def launch( node_selectors, use_separate_vc, keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) log_level = input_parser.get_client_log_level_or_default( @@ -148,6 +150,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(service_name, config) @@ -227,6 +230,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -250,6 +254,23 @@ def get_beacon_config( el_context.ip_addr, el_context.engine_rpc_port_num, ) + + public_ports = {} + discovery_port = BEACON_DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if bootnode_contexts and len(bootnode_contexts) > 0: + discovery_port = discovery_port + len(bootnode_contexts) + public_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--network={0}".format( network if network in constants.PUBLIC_NETWORKS else "custom" @@ -257,15 +278,15 @@ def get_beacon_config( "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--http-address=0.0.0.0", "--http-port={0}".format(BEACON_HTTP_PORT_NUM), - "--libp2p-port={0}".format(BEACON_DISCOVERY_PORT_NUM), - "--discovery-port={0}".format(BEACON_DISCOVERY_PORT_NUM), + "--libp2p-port={0}".format(discovery_port), + "--discovery-port={0}".format(discovery_port), "--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, "--eth1-rpc-urls=" + EXECUTION_ENGINE_ENDPOINT, # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv "--disable-enr-auto-update", - "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, - "--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), - "--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), + "--enr-address=" + port_publisher.nat_exit_ip, + "--enr-udp-port={0}".format(discovery_port), + "--enr-tcp-port={0}".format(discovery_port), # ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^ # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--metrics", @@ -353,7 +374,7 @@ def get_beacon_config( } ports = {} - ports.update(BEACON_USED_PORTS) + ports.update(used_ports) if node_keystore_files != None and not use_separate_vc: cmd.extend(validator_default_cmd) files[ @@ -373,10 +394,11 @@ def get_beacon_config( return ServiceConfig( image=image, ports=ports, + public_ports=public_ports, cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index 1c404c966..1e1ee4582 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -32,26 +32,28 @@ BEACON_MIN_MEMORY = 256 METRICS_PATH = "/metrics" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -BEACON_USED_PORTS = { - BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - BEACON_HTTP_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( - BEACON_METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), -} + +def get_used_ports(discovery_port): + beacon_used_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + BEACON_HTTP_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( + BEACON_METRICS_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + } + return beacon_used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "error", @@ -90,8 +92,9 @@ def launch( participant_tolerations, global_tolerations, node_selectors, - use_separate_vc=True, - keymanager_enabled=False, + use_separate_vc, + keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) @@ -148,6 +151,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -242,6 +246,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): # If snooper is enabled use the snooper engine context, otherwise use the execution client context if snooper_enabled: @@ -255,6 +260,22 @@ def get_beacon_config( el_context.engine_rpc_port_num, ) + public_ports = {} + discovery_port = BEACON_DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if boot_cl_client_ctxs and len(boot_cl_client_ctxs) > 0: + discovery_port = discovery_port + len(boot_cl_client_ctxs) + public_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + # NOTE: If connecting to the merge devnet remotely we DON'T want the following flags; when they're not set, the node's external IP address is auto-detected # from the peers it communicates with but when they're set they basically say "override the autodetection and # use what I specify instead." This requires having a know external IP address and port, which we definitely won't @@ -270,13 +291,13 @@ def get_beacon_config( "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER, # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv "--disable-enr-auto-update", - "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, - "--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), - "--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), + "--enr-address=" + port_publisher.nat_exit_ip, + "--enr-udp-port={0}".format(discovery_port), + "--enr-tcp-port={0}".format(discovery_port), # ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^ "--listen-address=0.0.0.0", "--port={0}".format( - BEACON_DISCOVERY_PORT_NUM + discovery_port ), # NOTE: Remove for connecting to external net! "--http", "--http-address=0.0.0.0", @@ -368,11 +389,12 @@ def get_beacon_config( env.update(extra_env_vars) return ServiceConfig( image=image, - ports=BEACON_USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, files=files, env_vars=env, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index d6f422392..4d2b97e9a 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -25,22 +25,24 @@ BEACON_MIN_MEMORY = 256 METRICS_PATH = "/metrics" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -BEACON_USED_PORTS = { - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port): + beacon_used_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return beacon_used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "error", @@ -79,8 +81,9 @@ def launch( participant_tolerations, global_tolerations, node_selectors, - use_separate_vc=True, - keymanager_enabled=False, + use_separate_vc, + keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) log_level = input_parser.get_client_log_level_or_default( @@ -136,6 +139,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -234,6 +238,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): el_client_rpc_url_str = "http://{0}:{1}".format( el_context.ip_addr, @@ -252,11 +257,27 @@ def get_beacon_config( el_context.engine_rpc_port_num, ) + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if bootnode_contexts and len(bootnode_contexts) > 0: + discovery_port = discovery_port + len(bootnode_contexts) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "beacon", "--logLevel=" + log_level, - "--port={0}".format(DISCOVERY_PORT_NUM), - "--discoveryPort={0}".format(DISCOVERY_PORT_NUM), + "--port={0}".format(discovery_port), + "--discoveryPort={0}".format(discovery_port), "--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--eth1.depositContractDeployBlock=0", "--network.connectToDiscv5Bootnodes=true", @@ -269,9 +290,9 @@ def get_beacon_config( "--rest.namespace=*", "--rest.port={0}".format(HTTP_PORT_NUM), "--nat=true", - "--enr.ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, - "--enr.tcp={0}".format(DISCOVERY_PORT_NUM), - "--enr.udp={0}".format(DISCOVERY_PORT_NUM), + "--enr.ip=" + port_publisher.nat_exit_ip, + "--enr.tcp={0}".format(discovery_port), + "--enr.udp={0}".format(discovery_port), # Set per Pari's recommendation to reduce noise in the logs "--subscribeAllSubnets=true", "--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, @@ -348,11 +369,12 @@ def get_beacon_config( ) return ServiceConfig( image=image, - ports=BEACON_USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index 712a9d050..9750425aa 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -42,25 +42,27 @@ VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/nimbus/validator-keys" # ---------------------------------- Used Ports ---------------------------------- -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" -BEACON_USED_PORTS = { - BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - BEACON_HTTP_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( - BEACON_METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), -} +def get_used_ports(discovery_port): + used_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + BEACON_HTTP_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( + BEACON_METRICS_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + } + return used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "ERROR", @@ -103,6 +105,7 @@ def launch( node_selectors, use_separate_vc, keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) @@ -163,6 +166,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -242,6 +246,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -266,11 +271,27 @@ def get_beacon_config( el_context.engine_rpc_port_num, ) + public_ports = {} + discovery_port = BEACON_DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if bootnode_contexts and len(bootnode_contexts) > 0: + discovery_port = discovery_port + len(bootnode_contexts) + public_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--non-interactive=true", "--log-level=" + log_level, - "--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), - "--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), + "--udp-port={0}".format(discovery_port), + "--tcp-port={0}".format(discovery_port), "--network={0}".format( network if network in constants.PUBLIC_NETWORKS @@ -278,7 +299,7 @@ def get_beacon_config( ), "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--web3-url=" + EXECUTION_ENGINE_ENDPOINT, - "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, + "--nat=extip:" + port_publisher.nat_exit_ip, "--enr-auto-update=false", "--history={0}".format("archive" if constants.ARCHIVE_MODE else "prune"), "--rest", @@ -340,7 +361,7 @@ def get_beacon_config( constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, } ports = {} - ports.update(BEACON_USED_PORTS) + ports.update(used_ports) if node_keystore_files != None and not use_separate_vc: cmd.extend(validator_default_cmd) files[ @@ -360,11 +381,12 @@ def get_beacon_config( return ServiceConfig( image=image, - ports=ports, + ports=used_ports, + public_ports=public_ports, cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 818543573..02179e06b 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -31,23 +31,27 @@ METRICS_PATH = "/metrics" MIN_PEERS = 1 -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -BEACON_NODE_USED_PORTS = { - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_TCP_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_UDP_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - RPC_PORT_ID: shared_utils.new_port_spec(RPC_PORT_NUM, shared_utils.TCP_PROTOCOL), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_MONITORING_PORT_ID: shared_utils.new_port_spec( - BEACON_MONITORING_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port): + used_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + BEACON_MONITORING_PORT_ID: shared_utils.new_port_spec( + BEACON_MONITORING_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "error", @@ -86,8 +90,9 @@ def launch( participant_tolerations, global_tolerations, node_selectors, - use_separate_vc=True, - keymanager_enabled=False, + use_separate_vc, + keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) log_level = input_parser.get_client_log_level_or_default( @@ -142,6 +147,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -219,6 +225,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): # If snooper is enabled use the snooper engine context, otherwise use the execution client context if snooper_enabled: @@ -232,6 +239,22 @@ def get_beacon_config( el_context.engine_rpc_port_num, ) + public_ports = {} + discovery_port = DISCOVERY_TCP_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if bootnode_contexts and len(bootnode_contexts) > 0: + discovery_port = discovery_port + len(bootnode_contexts) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--accept-terms-of-use=true", # it's mandatory in order to run the node "--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, @@ -241,9 +264,9 @@ def get_beacon_config( "--grpc-gateway-host=0.0.0.0", "--grpc-gateway-corsdomain=*", "--grpc-gateway-port={0}".format(HTTP_PORT_NUM), - "--p2p-host-ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, - "--p2p-tcp-port={0}".format(DISCOVERY_TCP_PORT_NUM), - "--p2p-udp-port={0}".format(DISCOVERY_UDP_PORT_NUM), + "--p2p-host-ip=" + port_publisher.nat_exit_ip, + "--p2p-tcp-port={0}".format(discovery_port), + "--p2p-udp-port={0}".format(discovery_port), "--min-sync-peers={0}".format(MIN_PEERS), "--verbosity=" + log_level, "--slots-per-archive-point={0}".format(32 if constants.ARCHIVE_MODE else 8192), @@ -331,11 +354,12 @@ def get_beacon_config( return ServiceConfig( image=beacon_image, - ports=BEACON_NODE_USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index b365d857b..90d6e3a05 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -31,22 +31,23 @@ BEACON_METRICS_PATH = "/metrics" MIN_PEERS = 1 -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -BEACON_USED_PORTS = { - BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( - BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( - BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port): + used_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + BEACON_HTTP_PORT_ID: shared_utils.new_port_spec( + BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + BEACON_METRICS_PORT_ID: shared_utils.new_port_spec( + BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports ENTRYPOINT_ARGS = ["sh", "-c"] @@ -90,6 +91,7 @@ def launch( node_selectors, use_separate_vc, keymanager_enabled, + port_publisher, ): beacon_service_name = "{0}".format(service_name) log_level = input_parser.get_client_log_level_or_default( @@ -151,6 +153,7 @@ def launch( cl_volume_size, tolerations, node_selectors, + port_publisher, ) beacon_service = plan.add_service(service_name, config) @@ -232,6 +235,7 @@ def get_beacon_config( cl_volume_size, tolerations, node_selectors, + port_publisher, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -255,6 +259,23 @@ def get_beacon_config( el_context.ip_addr, el_context.engine_rpc_port_num, ) + + public_ports = {} + discovery_port = BEACON_DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.cl_start + if bootnode_contexts and len(bootnode_contexts) > 0: + discovery_port = discovery_port + len(bootnode_contexts) + public_ports = { + BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--logging=" + log_level, "--log-destination=CONSOLE", @@ -271,8 +292,9 @@ def get_beacon_config( # Set per Pari's recommendation, to reduce noise in the logs "--p2p-subscribe-all-subnets-enabled=true", "--p2p-peer-lower-bound={0}".format(MIN_PEERS), - "--p2p-advertised-ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, + "--p2p-advertised-ip=" + port_publisher.nat_exit_ip, "--p2p-discovery-site-local-addresses-enabled=true", + "--p2p-port={0}".format(discovery_port), "--rest-api-enabled=true", "--rest-api-docs-enabled=true", "--rest-api-interface=0.0.0.0", @@ -374,7 +396,7 @@ def get_beacon_config( constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, } ports = {} - ports.update(BEACON_USED_PORTS) + ports.update(used_ports) if node_keystore_files != None and not use_separate_vc: cmd.extend(validator_default_cmd) files[ @@ -395,10 +417,11 @@ def get_beacon_config( return ServiceConfig( image=image, ports=ports, + public_ports=public_ports, cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID ), diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star index 2e44540b3..1dcd17a50 100644 --- a/src/el/besu/besu_launcher.star +++ b/src/el/besu/besu_launcher.star @@ -29,26 +29,31 @@ UDP_DISCOVERY_PORT_ID = "udp-discovery" ENGINE_HTTP_RPC_PORT_ID = "engine-rpc" METRICS_PORT_ID = "metrics" JAVA_OPTS = {"JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n"} -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" -USED_PORTS = { - RPC_PORT_ID: shared_utils.new_port_spec( - RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL - ), - WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_HTTP_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_HTTP_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_HTTP_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_HTTP_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + ENTRYPOINT_ARGS = ["sh", "-c"] @@ -80,6 +85,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -129,6 +135,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -174,7 +181,22 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "besu", "--logging=" + log_level, @@ -190,8 +212,8 @@ def get_config( "--rpc-ws-port={0}".format(WS_PORT_NUM), "--rpc-ws-api=ADMIN,CLIQUE,ETH,NET,DEBUG,TXPOOL,ENGINE,TRACE,WEB3", "--p2p-enabled=true", - "--p2p-host=" + PRIVATE_IP_ADDRESS_PLACEHOLDER, - "--p2p-port={0}".format(DISCOVERY_PORT_NUM), + "--p2p-host=" + port_publisher.nat_exit_ip, + "--p2p-port={0}".format(discovery_port), "--engine-rpc-enabled=true", "--engine-jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, "--engine-host-allowlist=*", @@ -255,12 +277,13 @@ def get_config( ) return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=[cmd_str], files=files, env_vars=extra_env_vars, entrypoint=ENTRYPOINT_ARGS, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/el_launcher.star b/src/el/el_launcher.star index 14d5226d9..f08a4379f 100644 --- a/src/el/el_launcher.star +++ b/src/el/el_launcher.star @@ -23,6 +23,7 @@ def launch( persistent, network_id, num_participants, + port_publisher, ): el_launchers = { constants.EL_TYPE.geth: { @@ -147,6 +148,7 @@ def launch( participant.el_volume_size, tolerations, node_selectors, + port_publisher, ) # Add participant el additional prometheus metrics for metrics_info in el_context.el_metrics_info: diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star index 23a82b7c8..9c96b7744 100644 --- a/src/el/erigon/erigon_launcher.star +++ b/src/el/erigon/erigon_launcher.star @@ -28,27 +28,28 @@ ENGINE_RPC_PORT_ID = "engine-rpc" METRICS_PORT_ID = "metrics" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + WS_RPC_PORT_ID: shared_utils.new_port_spec( + WS_RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports -USED_PORTS = { - WS_RPC_PORT_ID: shared_utils.new_port_spec( - WS_RPC_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} ENTRYPOINT_ARGS = ["sh", "-c"] @@ -80,6 +81,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -131,6 +133,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -180,12 +183,27 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): init_datadir_cmd_str = "erigon init --datadir={0} {1}".format( EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json", ) + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "erigon", "{0}".format( @@ -196,12 +214,12 @@ def get_config( "--networkid={0}".format(networkid), "--log.console.verbosity=" + verbosity_level, "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - "--port={0}".format(DISCOVERY_PORT_NUM), + "--port={0}".format(discovery_port), "--http.api=eth,erigon,engine,web3,net,debug,trace,txpool,admin", "--http.vhosts=*", "--ws", "--allow-insecure-unlock", - "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, + "--nat=extip:" + port_publisher.nat_exit_ip, "--http", "--http.addr=0.0.0.0", "--http.corsdomain=*", @@ -257,11 +275,12 @@ def get_config( return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=[command_arg_str], files=files, entrypoint=ENTRYPOINT_ARGS, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star index 570bb716e..187e422e9 100644 --- a/src/el/ethereumjs/ethereumjs_launcher.star +++ b/src/el/ethereumjs/ethereumjs_launcher.star @@ -30,27 +30,31 @@ METRICS_PATH = "/metrics" # The dirpath of the execution data directory on the client container EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/ethereumjs/execution-data" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" -USED_PORTS = { - RPC_PORT_ID: shared_utils.new_port_spec( - RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL - ), - WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), - WS_PORT_ENGINE_ID: shared_utils.new_port_spec( - WS_PORT_ENGINE_NUM, shared_utils.TCP_PROTOCOL - ), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - # METRICS_PORT_ID: shared_utils.new_port_spec(METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL) -} +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + WS_PORT_ENGINE_ID: shared_utils.new_port_spec( + WS_PORT_ENGINE_NUM, shared_utils.TCP_PROTOCOL + ), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + # METRICS_PORT_ID: shared_utils.new_port_spec(METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL) + } + return used_ports + ENTRYPOINT_ARGS = [] @@ -82,6 +86,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -131,6 +136,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -175,10 +181,25 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--dataDir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, - "--port={0}".format(DISCOVERY_PORT_NUM), + "--port={0}".format(discovery_port), "--rpc", "--rpcAddr=0.0.0.0", "--rpcPort={0}".format(RPC_PORT_NUM), @@ -192,7 +213,7 @@ def get_config( "--wsEnginePort={0}".format(WS_PORT_ENGINE_NUM), "--wsEngineAddr=0.0.0.0", "--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, - "--extIP={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER), + "--extIP={0}".format(port_publisher.nat_exit_ip), "--sync=full", "--isSingleNode=true", "--logLevel={0}".format(verbosity_level), @@ -242,11 +263,12 @@ def get_config( ) return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, files=files, entrypoint=ENTRYPOINT_ARGS, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/geth/geth_launcher.star b/src/el/geth/geth_launcher.star index a62597e3d..cff68c758 100644 --- a/src/el/geth/geth_launcher.star +++ b/src/el/geth/geth_launcher.star @@ -36,29 +36,31 @@ METRICS_PATH = "/debug/metrics/prometheus" # The dirpath of the execution data directory on the client container EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" -USED_PORTS = { - RPC_PORT_ID: shared_utils.new_port_spec( - RPC_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, - shared_utils.TCP_PROTOCOL, - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + ENTRYPOINT_ARGS = ["sh", "-c"] @@ -94,6 +96,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -146,6 +149,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -196,6 +200,7 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): if "--gcmode=archive" in extra_params or "--gcmode archive" in extra_params: gcmode_archive = True @@ -232,6 +237,20 @@ def get_config( constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json", ) + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "geth", # Disable path based storage scheme for electra fork and verkle @@ -271,7 +290,7 @@ def get_config( "--ws.api=admin,engine,net,eth,web3,debug", "--ws.origins=*", "--allow-insecure-unlock", - "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, + "--nat=extip:" + port_publisher.nat_exit_ip, "--verbosity=" + verbosity_level, "--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM), "--authrpc.addr=0.0.0.0", @@ -282,6 +301,8 @@ def get_config( "--metrics", "--metrics.addr=0.0.0.0", "--metrics.port={0}".format(METRICS_PORT_NUM), + "--discovery.port={0}".format(discovery_port), + "--port={0}".format(discovery_port), ] if BUILDER_IMAGE_STR in image: @@ -352,11 +373,12 @@ def get_config( ) return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=[command_str], files=files, entrypoint=ENTRYPOINT_ARGS, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/nethermind/nethermind_launcher.star b/src/el/nethermind/nethermind_launcher.star index e18838c6b..4e8ae8dc9 100644 --- a/src/el/nethermind/nethermind_launcher.star +++ b/src/el/nethermind/nethermind_launcher.star @@ -29,26 +29,30 @@ UDP_DISCOVERY_PORT_ID = "udp-discovery" ENGINE_RPC_PORT_ID = "engine-rpc" METRICS_PORT_ID = "metrics" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -USED_PORTS = { - RPC_PORT_ID: shared_utils.new_port_spec( - RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL - ), - WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "ERROR", @@ -78,6 +82,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -127,6 +132,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -173,7 +179,22 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--log=" + log_level, "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, @@ -185,9 +206,9 @@ def get_config( "--JsonRpc.WebSocketsPort={0}".format(WS_PORT_NUM), "--JsonRpc.EngineHost=0.0.0.0", "--JsonRpc.EnginePort={0}".format(ENGINE_RPC_PORT_NUM), - "--Network.ExternalIp={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER), - "--Network.DiscoveryPort={0}".format(DISCOVERY_PORT_NUM), - "--Network.P2PPort={0}".format(DISCOVERY_PORT_NUM), + "--Network.ExternalIp={0}".format(port_publisher.nat_exit_ip), + "--Network.DiscoveryPort={0}".format(discovery_port), + "--Network.P2PPort={0}".format(discovery_port), "--JsonRpc.JwtSecretFile=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, "--Metrics.Enabled=true", "--Metrics.ExposePort={0}".format(METRICS_PORT_NUM), @@ -250,10 +271,11 @@ def get_config( return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/nimbus-eth1/nimbus_launcher.star b/src/el/nimbus-eth1/nimbus_launcher.star index ef339b814..30fdd0966 100644 --- a/src/el/nimbus-eth1/nimbus_launcher.star +++ b/src/el/nimbus-eth1/nimbus_launcher.star @@ -27,31 +27,33 @@ METRICS_PATH = "/metrics" # The dirpath of the execution data directory on the client container EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/nimbus/execution-data" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" -USED_PORTS = { - WS_RPC_PORT_ID: shared_utils.new_port_spec( - WS_RPC_PORT_NUM, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, - shared_utils.TCP_PROTOCOL, - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, - shared_utils.UDP_PROTOCOL, - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, - shared_utils.TCP_PROTOCOL, - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, - shared_utils.TCP_PROTOCOL, - ), -} +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + WS_RPC_PORT_ID: shared_utils.new_port_spec( + WS_RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, + shared_utils.TCP_PROTOCOL, + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, + shared_utils.UDP_PROTOCOL, + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, + shared_utils.TCP_PROTOCOL, + ), + } + return used_ports + VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "ERROR", @@ -82,6 +84,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -131,6 +134,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -176,7 +180,22 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "--log-level={0}".format(verbosity_level), "--data-dir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, @@ -193,7 +212,8 @@ def get_config( "--metrics", "--metrics-address=0.0.0.0", "--metrics-port={0}".format(METRICS_PORT_NUM), - "--nat=extip:{0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER), + "--nat=extip:{0}".format(port_publisher.nat_exit_ip), + "--tcp-port={0}".format(discovery_port), ] if ( network not in constants.PUBLIC_NETWORKS @@ -243,10 +263,11 @@ def get_config( return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=cmd, files=files, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/el/reth/reth_launcher.star b/src/el/reth/reth_launcher.star index 3a9592d09..3608c9680 100644 --- a/src/el/reth/reth_launcher.star +++ b/src/el/reth/reth_launcher.star @@ -29,26 +29,30 @@ METRICS_PATH = "/metrics" # The dirpath of the execution data directory on the client container EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/reth/execution-data" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - -USED_PORTS = { - RPC_PORT_ID: shared_utils.new_port_spec( - RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL - ), - WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), - TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( - DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL - ), - ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( - ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL - ), - METRICS_PORT_ID: shared_utils.new_port_spec( - METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL - ), -} + +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + ENTRYPOINT_ARGS = ["sh", "-c"] @@ -81,6 +85,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -130,6 +135,7 @@ def launch( el_volume_size, tolerations, node_selectors, + port_publisher, ) service = plan.add_service(service_name, config) @@ -175,12 +181,27 @@ def get_config( el_volume_size, tolerations, node_selectors, + port_publisher, ): init_datadir_cmd_str = "reth init --datadir={0} --chain={1}".format( EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json", ) + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + if port_publisher.public_port_start: + discovery_port = port_publisher.el_start + len(existing_el_clients) + public_ports = { + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + } + used_ports = get_used_ports(discovery_port) + cmd = [ "reth", "node", @@ -203,11 +224,13 @@ def get_config( "--ws.port={0}".format(WS_PORT_NUM), "--ws.api=net,eth", "--ws.origins=*", - "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, + "--nat=extip:" + port_publisher.nat_exit_ip, "--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM), "--authrpc.jwtsecret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, "--authrpc.addr=0.0.0.0", "--metrics=0.0.0.0:{0}".format(METRICS_PORT_NUM), + "--discovery.port={0}".format(discovery_port), + "--port={0}".format(discovery_port), ] if network == constants.NETWORK_NAME.kurtosis: if len(existing_el_clients) > 0: @@ -256,11 +279,12 @@ def get_config( return ServiceConfig( image=image, - ports=USED_PORTS, + ports=used_ports, + public_ports=public_ports, cmd=[command_str], files=files, entrypoint=ENTRYPOINT_ARGS, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, min_memory=el_min_mem, diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 373d9b2e7..8573d7aaf 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -67,6 +67,8 @@ KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = ( DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest" +PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" + GENESIS_FORK_VERSION = "0x10000038" BELLATRIX_FORK_VERSION = "0x30000038" CAPELLA_FORK_VERSION = "0x40000038" diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index f791d4184..99c3baa5a 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -65,6 +65,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = ( "tx_spammer_params", "custom_flood_params", "xatu_sentry_params", + "port_publisher", ) @@ -135,6 +136,10 @@ def input_parser(plan, input_args): for sub_attr in input_args["xatu_sentry_params"]: sub_value = input_args["xatu_sentry_params"][sub_attr] result["xatu_sentry_params"][sub_attr] = sub_value + elif attr == "port_publisher": + for sub_attr in input_args["port_publisher"]: + sub_value = input_args["port_publisher"][sub_attr] + result["port_publisher"][sub_attr] = sub_value if result.get("disable_peer_scoring"): result = enrich_disable_peer_scoring(result) @@ -147,6 +152,14 @@ def input_parser(plan, input_args): result.get("mev_type"), ) + if result["port_publisher"]["nat_exit_ip"] == "auto": + result["port_publisher"]["nat_exit_ip"] = get_public_ip(plan) + + if result["port_publisher"]["public_port_start"] != None: + start = result["port_publisher"]["public_port_start"] + result["port_publisher"]["el_start"] = start + result["port_publisher"]["cl_start"] = start + len(result["participants"]) + return struct( participants=[ struct( @@ -303,6 +316,12 @@ def input_parser(plan, input_args): global_tolerations=result["global_tolerations"], global_node_selectors=result["global_node_selectors"], keymanager_enabled=result["keymanager_enabled"], + port_publisher=struct( + public_port_start=result["port_publisher"]["public_port_start"], + nat_exit_ip=result["port_publisher"]["nat_exit_ip"], + el_start=result["port_publisher"].get("el_start"), + cl_start=result["port_publisher"].get("cl_start"), + ), ) @@ -611,6 +630,10 @@ def default_input_args(): "global_tolerations": [], "global_node_selectors": {}, "keymanager_enabled": False, + "port_publisher": { + "nat_exit_ip": constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, + "public_port_start": None, + }, } @@ -878,3 +901,10 @@ def deep_copy_participant(participant): else: part[k] = v return part + + +def get_public_ip(plan): + response = plan.run_sh( + run="curl -s https://ident.me", + ) + return response.output diff --git a/src/participant_network.star b/src/participant_network.star index 3adb99437..54fe05027 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -39,7 +39,8 @@ def launch_participant_network( global_tolerations, global_node_selectors, keymanager_enabled, - parallel_keystore_generation=False, + parallel_keystore_generation, + port_publisher, ): network_id = network_params.network_id latest_block = "" @@ -145,6 +146,7 @@ def launch_participant_network( persistent, network_id, num_participants, + port_publisher, ) # Launch all consensus layer clients @@ -180,6 +182,7 @@ def launch_participant_network( validator_data, prysm_password_relative_filepath, prysm_password_artifact_uuid, + port_publisher, ) ethereum_metrics_exporter_context = None diff --git a/src/snooper/snooper_beacon_launcher.star b/src/snooper/snooper_beacon_launcher.star index fd0cc05b0..630600c3f 100644 --- a/src/snooper/snooper_beacon_launcher.star +++ b/src/snooper/snooper_beacon_launcher.star @@ -8,9 +8,6 @@ SNOOPER_BEACON_RPC_PORT_NUM = 8562 SNOOPER_BEACON_RPC_PORT_ID = "http" SNOOPER_BINARY_COMMAND = "./json_rpc_snoop" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - - SNOOPER_USED_PORTS = { SNOOPER_BEACON_RPC_PORT_ID: shared_utils.new_port_spec( SNOOPER_BEACON_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s" @@ -51,7 +48,6 @@ def get_config(service_name, cl_context, node_selectors): image=constants.DEFAULT_SNOOPER_IMAGE, ports=SNOOPER_USED_PORTS, cmd=cmd, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=MIN_CPU, max_cpu=MAX_CPU, min_memory=MIN_MEMORY, diff --git a/src/snooper/snooper_engine_launcher.star b/src/snooper/snooper_engine_launcher.star index 4f818e201..6d2ef91c2 100644 --- a/src/snooper/snooper_engine_launcher.star +++ b/src/snooper/snooper_engine_launcher.star @@ -8,8 +8,6 @@ SNOOPER_ENGINE_RPC_PORT_NUM = 8561 SNOOPER_ENGINE_RPC_PORT_ID = "http" SNOOPER_BINARY_COMMAND = "./json_rpc_snoop" -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - SNOOPER_USED_PORTS = { SNOOPER_ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( SNOOPER_ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s" @@ -52,7 +50,6 @@ def get_config(service_name, el_context, node_selectors): image=constants.DEFAULT_SNOOPER_IMAGE, ports=SNOOPER_USED_PORTS, cmd=cmd, - private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=MIN_CPU, max_cpu=MAX_CPU, min_memory=MIN_MEMORY, diff --git a/src/vc/lodestar.star b/src/vc/lodestar.star index 343cca4a7..fb16f443a 100644 --- a/src/vc/lodestar.star +++ b/src/vc/lodestar.star @@ -99,7 +99,6 @@ def get_config( cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=vc_min_cpu, max_cpu=vc_max_cpu, min_memory=vc_min_mem, diff --git a/src/vc/nimbus.star b/src/vc/nimbus.star index 94569d658..d84fd161a 100644 --- a/src/vc/nimbus.star +++ b/src/vc/nimbus.star @@ -77,7 +77,6 @@ def get_config( cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=vc_min_cpu, max_cpu=vc_max_cpu, min_memory=vc_min_mem, diff --git a/src/vc/prysm.star b/src/vc/prysm.star index f853925d0..3c7368c8e 100644 --- a/src/vc/prysm.star +++ b/src/vc/prysm.star @@ -92,7 +92,6 @@ def get_config( cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=vc_min_cpu, max_cpu=vc_max_cpu, min_memory=vc_min_mem, diff --git a/src/vc/shared.star b/src/vc/shared.star index e5d405603..aa38feddd 100644 --- a/src/vc/shared.star +++ b/src/vc/shared.star @@ -1,7 +1,5 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" - VALIDATOR_HTTP_PORT_NUM = 5056 VALIDATOR_HTTP_PORT_ID = "vc-http" diff --git a/src/vc/teku.star b/src/vc/teku.star index be6512ce4..91065cdc2 100644 --- a/src/vc/teku.star +++ b/src/vc/teku.star @@ -88,7 +88,6 @@ def get_config( cmd=cmd, env_vars=extra_env_vars, files=files, - private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=vc_min_cpu, max_cpu=vc_max_cpu, min_memory=vc_min_mem,