diff --git a/.github/tests/blobber.yaml b/.github/tests/blobber.yaml index 6e01ca8f5..78fffdd7f 100644 --- a/.github/tests/blobber.yaml +++ b/.github/tests/blobber.yaml @@ -3,14 +3,28 @@ participants: el_image: ethpandaops/geth:master cl_type: lighthouse blobber_enabled: true + blobber_image: ethpandaops/blobber:latest blobber_extra_params: - --proposal-action-frequency=1 - - "--proposal-action={\"name\": \"blob_gossip_delay\", \"delay_milliseconds\": 1500}" + - "--proposal-action={\"name\": \"blob_gossip_delay\", \"delay_milliseconds\": 3000}" count: 1 - el_type: geth el_image: ethpandaops/geth:master cl_type: lodestar count: 1 + - el_type: geth + el_image: ethpandaops/geth:master + cl_type: prysm + blobber_enabled: true + blobber_image: ethpandaops/blobber:latest + blobber_extra_params: + - --proposal-action-frequency=1 + - "--proposal-action={\"name\": \"blob_gossip_delay\", \"delay_milliseconds\": 30000}" + count: 1 + additional_services: - dora - spamoor + +# - "--proposal-action={\"name\": \"blob_gossip_delay\", \"delay_milliseconds\": 30000}" +# - "--proposal-action={\"name\": \"conflicting_blobs\"}" diff --git a/README.md b/README.md index 29d2922a7..ef2342314 100644 --- a/README.md +++ b/README.md @@ -446,6 +446,10 @@ participants: # Defaults to empty blobber_extra_params: [] + # Blobber image to be used for the blobber container + # Defaults to empty + blobber_image: ethpandaops/blobber:latest + # A set of parameters the node needs to reach an external block building network # If `null` then the builder infrastructure will not be instantiated # Example: diff --git a/src/blobber/blobber_launcher.star b/src/blobber/blobber_launcher.star index 1af3cdee2..92f3bdebf 100644 --- a/src/blobber/blobber_launcher.star +++ b/src/blobber/blobber_launcher.star @@ -39,7 +39,7 @@ def launch( service_name, node_keystore_files, beacon_http_url, - extra_params, + participant, node_selectors, ): blobber_service_name = "{0}".format(service_name) @@ -48,14 +48,14 @@ def launch( service_name, node_keystore_files, beacon_http_url, - extra_params, + participant, node_selectors, ) blobber_service = plan.add_service(blobber_service_name, blobber_config) return blobber_context.new_blobber_context( blobber_service.ip_address, - blobber_service.ports[BLOBBER_VALIDATOR_PROXY_PORT_NUM], + blobber_service.ports[BLOBBER_VALIDATOR_PROXY_PORT_ID].number, ) @@ -63,7 +63,7 @@ def get_config( service_name, node_keystore_files, beacon_http_url, - extra_params, + participant, node_selectors, ): validator_root_dirpath = shared_utils.path_join( @@ -75,16 +75,15 @@ def get_config( "--cl={0}".format(beacon_http_url), "--validator-key-folder={0}".format(validator_root_dirpath), "--enable-unsafe-mode", - # Does this get affected by public ip address changes? "--external-ip={0}".format(constants.PRIVATE_IP_ADDRESS_PLACEHOLDER), "--validator-proxy-port-start={0}".format(BLOBBER_VALIDATOR_PROXY_PORT_NUM), ] - if len(extra_params) > 0: - cmd.extend([param for param in extra_params]) + if len(participant.blobber_extra_params) > 0: + cmd.extend([param for param in participant.blobber_extra_params]) return ServiceConfig( - image=DEFAULT_BLOBBER_IMAGE, + image=participant.blobber_image, ports=BLOBBER_USED_PORTS, files={ VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index ca72c47c4..fb5212a80 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -76,6 +76,7 @@ def launch( all_snooper_el_engine_contexts = [] all_cl_contexts = [] + blobber_configs_with_contexts = [] preregistered_validator_keys_for_nodes = ( validator_data.per_node_keystores if network_params.network == constants.NETWORK_NAME.kurtosis @@ -165,7 +166,7 @@ def launch( all_snooper_el_engine_contexts.append(snooper_el_engine_context) full_name = "{0}-{1}-{2}".format(index_str, el_type, cl_type) if index == 0: - cl_context = launch_method( + result = launch_method( plan, cl_launcher, cl_service_name, @@ -187,7 +188,7 @@ def launch( ) else: boot_cl_client_ctx = all_cl_contexts - cl_context = launch_method( + result = launch_method( plan, cl_launcher, cl_service_name, @@ -208,6 +209,21 @@ def launch( network_params, ) + # Handle both old (single return) and new (tuple) format + if type(result) == "tuple": + cl_context, blobber_config = result + if blobber_config != None: + blobber_configs_with_contexts.append( + struct( + cl_context=cl_context, + blobber_config=blobber_config, + participant=participant, + ) + ) + else: + # Backward compatibility for CL clients that don't support blobbers + cl_context = result + # Add participant cl additional prometheus labels for metrics_info in cl_context.cl_nodes_metrics_info: if metrics_info != None: @@ -219,4 +235,5 @@ def launch( all_snooper_el_engine_contexts, preregistered_validator_keys_for_nodes, global_other_index, + blobber_configs_with_contexts, ) diff --git a/src/cl/grandine/grandine_launcher.star b/src/cl/grandine/grandine_launcher.star index c453159ab..98d111d40 100644 --- a/src/cl/grandine/grandine_launcher.star +++ b/src/cl/grandine/grandine_launcher.star @@ -85,6 +85,9 @@ def launch( beacon_service.ip_address, beacon_http_port.number ) + # Grandine doesn't support blobbers, return None for blobber config + blobber_config = None + beacon_metrics_port = beacon_service.ports[constants.METRICS_PORT_ID] beacon_metrics_url = "{0}:{1}".format( beacon_service.ip_address, beacon_metrics_port.number @@ -110,7 +113,7 @@ def launch( beacon_service_name, BEACON_METRICS_PATH, beacon_metrics_url ) nodes_metrics_info = [beacon_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="grandine", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -128,6 +131,9 @@ def launch( supernode=participant.supernode, ) + # Return tuple of cl_context and blobber_config + return (cl_context_obj, blobber_config) + def get_beacon_config( plan, diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index a5229e3b5..4f6e13573 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -89,26 +89,16 @@ def launch( beacon_service.ip_address, beacon_http_port.number ) - # Blobber config + # Prepare blobber config without launching + blobber_config = None if participant.blobber_enabled: - blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name) - blobber_config = blobber_launcher.get_config( - blobber_service_name, - node_keystore_files, - beacon_http_url, - participant.blobber_extra_params, - node_selectors, + blobber_config = struct( + service_name="{0}-{1}".format("blobber", beacon_service_name), + beacon_http_url=beacon_http_url, + node_keystore_files=node_keystore_files, + node_selectors=node_selectors, ) - blobber_service = plan.add_service(blobber_service_name, blobber_config) - blobber_http_port = blobber_service.ports[ - blobber_launcher.BLOBBER_VALIDATOR_PROXY_PORT_ID - ] - blobber_http_url = "http://{0}:{1}".format( - blobber_service.ip_address, blobber_http_port.number - ) - beacon_http_url = blobber_http_url - # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( endpoint="/eth/v1/node/identity", @@ -134,7 +124,7 @@ def launch( beacon_service_name, METRICS_PATH, beacon_metrics_url ) nodes_metrics_info = [beacon_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="lighthouse", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -152,6 +142,9 @@ def launch( supernode=participant.supernode, ) + # Return tuple of cl_context and blobber_config + return (cl_context_obj, blobber_config) + def get_beacon_config( plan, diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index 91fe46485..12d35f9f0 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -79,25 +79,16 @@ def launch( beacon_service.ip_address, beacon_http_port.number ) - # Blobber config + # Prepare blobber config without launching + blobber_config = None if participant.blobber_enabled: - blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name) - blobber_config = blobber_launcher.get_config( - blobber_service_name, - node_keystore_files, - beacon_http_url, - participant.blobber_extra_params, + blobber_config = struct( + service_name="{0}-{1}".format("blobber", beacon_service_name), + beacon_http_url=beacon_http_url, + node_keystore_files=node_keystore_files, + node_selectors=node_selectors, ) - blobber_service = plan.add_service(blobber_service_name, blobber_config) - blobber_http_port = blobber_service.ports[ - blobber_launcher.BLOBBER_VALIDATOR_PROXY_PORT_ID - ] - blobber_http_url = "http://{0}:{1}".format( - blobber_service.ip_address, blobber_http_port.number - ) - beacon_http_url = blobber_http_url - # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -126,7 +117,7 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="lodestar", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -144,6 +135,9 @@ def launch( supernode=participant.supernode, ) + # Return tuple of cl_context and blobber_config + return (cl_context_obj, blobber_config) + def get_beacon_config( plan, diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index f7276e4f5..e8a72c517 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -122,7 +122,7 @@ def launch( ) nodes_metrics_info = [nimbus_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="nimbus", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -140,6 +140,9 @@ def launch( supernode=participant.supernode, ) + # Nimbus doesn't support blobbers, return None for blobber config + return (cl_context_obj, None) + def get_beacon_config( plan, diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 6861c85c3..79d454928 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -83,6 +83,16 @@ def launch( ) beacon_grpc_url = "{0}:{1}".format(beacon_service.ip_address, RPC_PORT_NUM) + # Prepare blobber config without launching + blobber_config = None + if participant.blobber_enabled: + blobber_config = struct( + service_name="{0}-{1}".format("blobber", beacon_service_name), + beacon_http_url=beacon_http_url, + node_keystore_files=node_keystore_files, + node_selectors=node_selectors, + ) + # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( endpoint="/eth/v1/node/identity", @@ -109,7 +119,7 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="prysm", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -128,6 +138,9 @@ def launch( supernode=participant.supernode, ) + # Return tuple of cl_context and blobber_config + return (cl_context_obj, blobber_config) + def get_beacon_config( plan, diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index 0f3a927f8..4d21a355c 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -111,7 +111,7 @@ def launch( ) nodes_metrics_info = [beacon_node_metrics_info] - return cl_context.new_cl_context( + cl_context_obj = cl_context.new_cl_context( client_name="teku", enr=beacon_node_enr, ip_addr=beacon_service.ip_address, @@ -129,6 +129,9 @@ def launch( supernode=participant.supernode, ) + # Teku doesn't support blobbers, return None for blobber config + return (cl_context_obj, None) + def get_beacon_config( plan, diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 3325cf24f..eae038723 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -330,6 +330,7 @@ def input_parser(plan, input_args): ), blobber_enabled=participant["blobber_enabled"], blobber_extra_params=participant["blobber_extra_params"], + blobber_image=participant["blobber_image"], keymanager_enabled=participant["keymanager_enabled"], ) for participant in result["participants"] @@ -809,8 +810,13 @@ def parse_network_params(plan, input_args): blobber_enabled = participant["blobber_enabled"] if blobber_enabled: - # unless we are running lighthouse, we don't support blobber - if participant["cl_type"] != constants.CL_TYPE.lighthouse: + # lighthouse, lodestar, prysm, and grandine support blobber + if participant["cl_type"] not in [ + constants.CL_TYPE.lighthouse, + constants.CL_TYPE.lodestar, + constants.CL_TYPE.prysm, + constants.CL_TYPE.grandine, + ]: fail( "blobber is not supported for {0} client".format( participant["cl_type"] @@ -1219,6 +1225,7 @@ def default_participant(): }, "blobber_enabled": False, "blobber_extra_params": [], + "blobber_image": "ethpandaops/blobber:latest", "builder_network_params": None, "keymanager_enabled": None, } @@ -1621,6 +1628,7 @@ def docker_cache_image_override(plan, result): "cl_image", "vc_image", "remote_signer_image", + "blobber_image", ] tooling_overridable_image = [ "dora_params.image", diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 5b9fe611e..9734a8185 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -58,6 +58,7 @@ PARTICIPANT_CATEGORIES = { "prometheus_config", "blobber_enabled", "blobber_extra_params", + "blobber_image", "builder_network_params", "keymanager_enabled", ], diff --git a/src/participant_network.star b/src/participant_network.star index 538a90d80..a30d38abf 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -27,6 +27,8 @@ remote_signer = import_module("./remote_signer/remote_signer_launcher.star") beacon_snooper = import_module("./snooper/snooper_beacon_launcher.star") snooper_el_launcher = import_module("./snooper/snooper_el_launcher.star") +blobber_launcher = import_module("./blobber/blobber_launcher.star") +cl_context_module = import_module("./cl/cl_context.star") def launch_participant_network( @@ -163,6 +165,7 @@ def launch_participant_network( all_snooper_el_engine_contexts, preregistered_validator_keys_for_nodes, global_other_index, + blobber_configs_with_contexts, ) = cl_client_launcher.launch( plan, network_params, @@ -181,6 +184,57 @@ def launch_participant_network( global_other_index, ) + # Launch all blobbers after all CLs are up + cl_context_to_blobber_url = {} + if len(blobber_configs_with_contexts) > 0: + plan.print("Launching blobbers for CL clients that have them enabled") + for config in blobber_configs_with_contexts: + blobber = blobber_launcher.launch( + plan, + config.blobber_config.service_name, + config.blobber_config.node_keystore_files, + config.blobber_config.beacon_http_url, + config.participant, + config.blobber_config.node_selectors, + ) + + # Store the blobber URL mapping + blobber_http_url = "http://{0}:{1}".format( + blobber.ip_addr, blobber.port_num + ) + cl_context_to_blobber_url[ + config.cl_context.beacon_service_name + ] = blobber_http_url + + # Helper function to get cl_context with blobber URL if available + def get_cl_context_with_blobber_url(cl_context): + beacon_service_name = cl_context.beacon_service_name + effective_beacon_url = cl_context_to_blobber_url.get( + beacon_service_name, cl_context.beacon_http_url + ) + + if effective_beacon_url == cl_context.beacon_http_url: + # No blobber, return original context + return cl_context + + # Create a new cl_context with the blobber URL + return cl_context_module.new_cl_context( + client_name=cl_context.client_name, + enr=cl_context.enr, + ip_addr=cl_context.ip_addr, + http_port=cl_context.http_port, + beacon_http_url=effective_beacon_url, + cl_nodes_metrics_info=cl_context.cl_nodes_metrics_info, + beacon_service_name=cl_context.beacon_service_name, + beacon_grpc_url=cl_context.beacon_grpc_url, + multiaddr=cl_context.multiaddr, + peer_id=cl_context.peer_id, + snooper_enabled=cl_context.snooper_enabled, + snooper_el_engine_context=cl_context.snooper_el_engine_context, + validator_keystore_files_artifact_uuid=cl_context.validator_keystore_files_artifact_uuid, + supernode=cl_context.supernode, + ) + ethereum_metrics_exporter_context = None all_ethereum_metrics_exporter_contexts = [] all_xatu_sentry_contexts = [] @@ -227,7 +281,7 @@ def launch_participant_network( pair_name, ethereum_metrics_exporter_service_name, el_context, - cl_context, + get_cl_context_with_blobber_url(cl_context), node_selectors, args_with_right_defaults.port_publisher, global_other_index, @@ -255,7 +309,7 @@ def launch_participant_network( xatu_sentry_context = xatu_sentry.launch( plan, xatu_sentry_service_name, - cl_context, + get_cl_context_with_blobber_url(cl_context), xatu_sentry_params, network_params, pair_name, @@ -335,7 +389,7 @@ def launch_participant_network( snooper_beacon_context = beacon_snooper.launch( plan, snooper_service_name, - cl_context, + get_cl_context_with_blobber_url(cl_context), node_selectors, args_with_right_defaults.port_publisher, global_other_index, @@ -396,7 +450,7 @@ def launch_participant_network( vc_type=vc_type, image=participant.vc_image, global_log_level=args_with_right_defaults.global_log_level, - cl_context=cl_context, + cl_context=get_cl_context_with_blobber_url(cl_context), el_context=el_context, remote_signer_context=remote_signer_context, full_name=full_name, diff --git a/src/vc/prysm.star b/src/vc/prysm.star index a65472955..4b765a777 100644 --- a/src/vc/prysm.star +++ b/src/vc/prysm.star @@ -41,7 +41,6 @@ def get_config( + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, - "--beacon-rpc-provider=" + cl_context.beacon_grpc_url, "--beacon-rest-api-provider=" + beacon_http_url, # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv "--disable-monitoring=false", @@ -50,6 +49,11 @@ def get_config( # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^ ] + # Only add RPC provider if we're not using a blobber (blobber doesn't proxy RPC) + # Blobber uses port 5000, so check if that's in the URL + if ":5000" not in beacon_http_url: + cmd.append("--beacon-rpc-provider=" + cl_context.beacon_grpc_url) + if remote_signer_context == None: cmd.extend( [ @@ -77,8 +81,13 @@ def get_config( "--keymanager-token-file=" + constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER, ] - if cl_context.client_name != constants.CL_TYPE.prysm: - # Use Beacon API if a Prysm VC wants to connect to a non-Prysm BN + # Check if we're using a blobber by checking for port 5000 + is_using_blobber = ":5000" in beacon_http_url + + if cl_context.client_name != constants.CL_TYPE.prysm or is_using_blobber: + # Use Beacon API if: + # 1. Prysm VC wants to connect to a non-Prysm BN, OR + # 2. Blobber is enabled (since blobber only proxies REST, not RPC) cmd.append("--enable-beacon-rest-api") if len(participant.vc_extra_params) > 0: