diff --git a/.github/tests/binary.norun..yaml b/.github/tests/binary.norun..yaml new file mode 100644 index 000000000..5db618362 --- /dev/null +++ b/.github/tests/binary.norun..yaml @@ -0,0 +1,10 @@ +participants: + - cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + count: 4 + - el_type: geth + cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + cl_binary_path: binary/lighthouse +additional_services: + - dora diff --git a/.github/tests/binary.yaml b/.github/tests/binary.yaml deleted file mode 100644 index 82cea939c..000000000 --- a/.github/tests/binary.yaml +++ /dev/null @@ -1,17 +0,0 @@ -participants: - - el_type: geth - cl_type: lighthouse - cl_image: ethpandaops/lighthouse:stable - cl_binary_path: binary/release/lighthouse - - cl_type: lighthouse -additional_services: - - dora - -port_publisher: - cl: - enabled: true - public_port_start: 33000 - additional_services: - enabled: true - public_port_start: 34000 - diff --git a/.github/tests/binary_force.norun.yaml b/.github/tests/binary_force.norun.yaml new file mode 100644 index 000000000..3ea6e2d56 --- /dev/null +++ b/.github/tests/binary_force.norun.yaml @@ -0,0 +1,11 @@ +participants: + - cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + count: 4 + - el_type: geth + cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + cl_binary_path: binary/lighthouse + cl_force_restart: true +additional_services: + - dora diff --git a/.github/tests/force.yaml b/.github/tests/force.yaml new file mode 100644 index 000000000..c2f9c3bf1 --- /dev/null +++ b/.github/tests/force.yaml @@ -0,0 +1,8 @@ +participants: + - cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + count: 2 + - el_type: geth + cl_type: lighthouse + cl_image: ethpandaops/lighthouse:unstable + cl_force_restart: true diff --git a/.github/tests/minimal.yaml b/.github/tests/minimal.yaml index 053e9e483..aea361d67 100644 --- a/.github/tests/minimal.yaml +++ b/.github/tests/minimal.yaml @@ -3,13 +3,13 @@ participants: cl_type: teku - el_type: geth cl_type: prysm - # - el_type: erigon - # cl_type: nimbus - #- el_type: besu - # cl_type: lighthouse + - el_type: erigon + cl_type: nimbus + - el_type: besu + cl_type: lighthouse - el_type: reth cl_type: lodestar - # - el_type: geth - # cl_type: grandine + - el_type: geth + cl_type: grandine network_params: preset: minimal diff --git a/README.md b/README.md index 975a31549..d68c26e5f 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ participants: # When set, the binary will be uploaded and mounted into the container, # replacing the default binary from the Docker image # Useful for rapid debugging with locally compiled binaries + # IMPORTANT: el_force_restart must be set to true when using this option # IMPORTANT: The binary file must live inside the ethereum-package directory # Build the client in its own repo, then copy ONLY the binary to ethereum-package # Do not run builds inside ethereum-package or copy build dependencies - only the final binary @@ -271,6 +272,12 @@ participants: el_min_mem: 0 el_max_mem: 0 + # Force container recreation on next run (Docker only) + # When set to true, the container will be recreated even if the image tag hasn't changed + # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name + # Defaults to false + el_force_restart: false + # CL(Consensus Layer) Specific flags # The type of CL client that should be started # Valid values are nimbus, lighthouse, lodestar, teku, prysm, and grandine @@ -290,6 +297,7 @@ participants: # When set, the binary will be uploaded and mounted into the container, # replacing the default binary from the Docker image # Useful for rapid debugging with locally compiled binaries + # IMPORTANT: cl_force_restart must be set to true when using this option # IMPORTANT: The binary file must live inside the ethereum-package directory # Build the client in its own repo, then copy ONLY the binary to ethereum-package # Do not run builds inside ethereum-package or copy build dependencies - only the final binary @@ -355,6 +363,12 @@ participants: cl_min_mem: 0 cl_max_mem: 0 + # Force container recreation on next run (Docker only) + # When set to true, the container will be recreated even if the image tag hasn't changed + # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name + # Defaults to false + cl_force_restart: false + # Whether to act as a supernode for the network # Supernodes will subscribe to all subnet topics # This flag should only be used with peerdas @@ -386,6 +400,7 @@ participants: # When set, the binary will be uploaded and mounted into the container, # replacing the default binary from the Docker image # Useful for rapid debugging with locally compiled binaries + # IMPORTANT: vc_force_restart must be set to true when using this option # IMPORTANT: The binary file must live inside the ethereum-package directory # Build the client in its own repo, then copy ONLY the binary to ethereum-package # Do not run builds inside ethereum-package or copy build dependencies - only the final binary @@ -446,6 +461,12 @@ participants: vc_min_mem: 0 vc_max_mem: 0 + # Force container recreation on next run (Docker only) + # When set to true, the container will be recreated even if the image tag hasn't changed + # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name + # Defaults to false + vc_force_restart: false + # A list of indices of the beacon nodes that the validator client should connect to # Defaults to null vc_beacon_node_indices: null diff --git a/main.star b/main.star index 1f8d3f65e..a65a86ae6 100644 --- a/main.star +++ b/main.star @@ -100,38 +100,19 @@ def run(plan, args={}): artifact = plan.render_templates(template_data, name + "_artifact") extra_files_artifacts[name] = artifact - # Process binary injection - upload local binaries for participants - # NOTE: Binary injection is only supported with Docker backend - binary_artifacts = {} - for index, participant in enumerate(args_with_right_defaults.participants): - participant_binaries = {} - for bin_type, bin_path in [ - ("el", participant.el_binary_path), - ("cl", participant.cl_binary_path), - ("vc", participant.vc_binary_path), + # Validate binary injection - only supported with Docker backend + for participant in args_with_right_defaults.participants: + for bin_path in [ + participant.el_binary_path, + participant.cl_binary_path, + participant.vc_binary_path, ]: - if bin_path: - if detected_backend != "docker": - fail( - "Binary injection (*_binary_path) is only supported with Docker backend, detected: {0}".format( - detected_backend - ) - ) - plan.print( - "Uploading {0} binary for participant {1}: {2}".format( - bin_type.upper(), index + 1, bin_path + if bin_path and detected_backend != "docker": + fail( + "Binary injection (*_binary_path) is only supported with Docker backend, detected: {0}".format( + detected_backend ) ) - # Extract filename from path and store both artifact and filename - filename = bin_path.split("/")[-1] - participant_binaries[bin_type] = struct( - artifact=plan.upload_files( - src=bin_path, name="{0}-binary-{1}".format(bin_type, index + 1) - ), - filename=filename, - ) - if participant_binaries: - binary_artifacts[index] = participant_binaries mev_params = args_with_right_defaults.mev_params parallel_keystore_generation = args_with_right_defaults.parallel_keystore_generation @@ -291,7 +272,6 @@ def run(plan, args={}): extra_files_artifacts, tempo_otlp_grpc_url, detected_backend, - binary_artifacts, ) plan.print( diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index 3cbc641fc..783649e5f 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -227,10 +227,7 @@ def launch( all_snooper_el_engine_contexts.append(snooper_el_engine_context) full_name = "{0}-{1}-{2}".format(index_str, el_type, cl_type) - # Get binary artifact for this participant if it exists - cl_binary_artifact = None - if index in binary_artifacts and "cl" in binary_artifacts[index]: - cl_binary_artifact = binary_artifacts[index]["cl"] + cl_binary_artifact = binary_artifacts.get(index, {}).get("cl", None) if index == 0: cl_context = launch_method( @@ -319,12 +316,13 @@ def launch( "get_cl_context": get_cl_context, "get_blobber_config": get_blobber_config, "participant_index": index, + "cl_type": cl_type, } # add rest of cl's in parallel to speed package execution - cl_services = {} - if len(cl_service_configs) > 0: - cl_services = plan.add_services(cl_service_configs) + cl_services = shared_utils.add_services_with_force_restart( + plan, cl_service_configs, cl_participant_info, "cl_force_restart" + ) # Create CL contexts ordered by participant index cl_contexts_temp = {} diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index b8ab8cc94..427a4b620 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -63,7 +63,6 @@ def launch( bootnode_enr_override=None, cl_binary_artifact=None, ): - # Launch Beacon node beacon_config = get_beacon_config( plan, launcher, @@ -90,7 +89,9 @@ def launch( cl_binary_artifact, ) - beacon_service = plan.add_service(beacon_service_name, beacon_config) + beacon_service = plan.add_service( + beacon_service_name, beacon_config, force_update=participant.cl_force_restart + ) cl_context_obj = get_cl_context( plan, diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star index 8e70c3cb4..885d3941f 100644 --- a/src/el/besu/besu_launcher.star +++ b/src/el/besu/besu_launcher.star @@ -68,7 +68,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/dummy/dummy_launcher.star b/src/el/dummy/dummy_launcher.star index 952649fe8..aa1e7ef12 100644 --- a/src/el/dummy/dummy_launcher.star +++ b/src/el/dummy/dummy_launcher.star @@ -58,7 +58,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/el_launcher.star b/src/el/el_launcher.star index f7fc45b22..38ed56012 100644 --- a/src/el/el_launcher.star +++ b/src/el/el_launcher.star @@ -175,11 +175,7 @@ def launch( index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants)))) el_service_name = "el-{0}-{1}-{2}".format(index_str, el_type, cl_type) - - # Get binary artifact for this participant if it exists - el_binary_artifact = None - if index in binary_artifacts and "el" in binary_artifacts[index]: - el_binary_artifact = binary_artifacts[index]["el"] + el_binary_artifact = binary_artifacts.get(index, {}).get("el", None) if index == 0: el_context = launch_method( @@ -234,9 +230,9 @@ def launch( } # add remainder of el's in parallel to speed package execution - el_services = {} - if len(el_service_configs) > 0: - el_services = plan.add_services(el_service_configs) + el_services = shared_utils.add_services_with_force_restart( + plan, el_service_configs, el_participant_info, "el_force_restart" + ) # Create contexts ordered by participant index el_contexts_temp = {} diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star index beb6bb148..5957a06af 100644 --- a/src/el/erigon/erigon_launcher.star +++ b/src/el/erigon/erigon_launcher.star @@ -66,7 +66,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star index fb4e548fd..e45d7cd86 100644 --- a/src/el/ethereumjs/ethereumjs_launcher.star +++ b/src/el/ethereumjs/ethereumjs_launcher.star @@ -68,7 +68,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/ethrex/ethrex_launcher.star b/src/el/ethrex/ethrex_launcher.star index 927db1146..5c5802f55 100644 --- a/src/el/ethrex/ethrex_launcher.star +++ b/src/el/ethrex/ethrex_launcher.star @@ -83,7 +83,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/geth/geth_launcher.star b/src/el/geth/geth_launcher.star index 7d96adb14..d9c977dd2 100644 --- a/src/el/geth/geth_launcher.star +++ b/src/el/geth/geth_launcher.star @@ -72,9 +72,12 @@ def launch( network_params, extra_files_artifacts, bootnodoor_enode, + el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/nethermind/nethermind_launcher.star b/src/el/nethermind/nethermind_launcher.star index 8407765af..13875e148 100644 --- a/src/el/nethermind/nethermind_launcher.star +++ b/src/el/nethermind/nethermind_launcher.star @@ -64,7 +64,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/nimbus-eth1/nimbus_launcher.star b/src/el/nimbus-eth1/nimbus_launcher.star index f067f655b..deeec790c 100644 --- a/src/el/nimbus-eth1/nimbus_launcher.star +++ b/src/el/nimbus-eth1/nimbus_launcher.star @@ -64,7 +64,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/el/reth/reth_launcher.star b/src/el/reth/reth_launcher.star index 94123aa06..563f8f066 100644 --- a/src/el/reth/reth_launcher.star +++ b/src/el/reth/reth_launcher.star @@ -73,7 +73,9 @@ def launch( el_binary_artifact, ) - service = plan.add_service(service_name, config) + service = plan.add_service( + service_name, config, force_update=participant.el_force_restart + ) return get_el_context( plan, diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index bf94b67bc..4d2085a5e 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -525,14 +525,17 @@ def input_parser(plan, input_args): el_max_cpu=participant["el_max_cpu"], el_min_mem=participant["el_min_mem"], el_max_mem=participant["el_max_mem"], + el_force_restart=participant["el_force_restart"], cl_min_cpu=participant["cl_min_cpu"], cl_max_cpu=participant["cl_max_cpu"], cl_min_mem=participant["cl_min_mem"], cl_max_mem=participant["cl_max_mem"], + cl_force_restart=participant["cl_force_restart"], vc_min_cpu=participant["vc_min_cpu"], vc_max_cpu=participant["vc_max_cpu"], vc_min_mem=participant["vc_min_mem"], vc_max_mem=participant["vc_max_mem"], + vc_force_restart=participant["vc_force_restart"], remote_signer_min_cpu=participant["remote_signer_min_cpu"], remote_signer_max_cpu=participant["remote_signer_max_cpu"], remote_signer_min_mem=participant["remote_signer_min_mem"], @@ -1509,6 +1512,7 @@ def default_participant(): "el_max_cpu": 0, "el_min_mem": 0, "el_max_mem": 0, + "el_force_restart": False, "cl_type": "lighthouse", "cl_image": "", "cl_binary_path": "", @@ -1524,6 +1528,7 @@ def default_participant(): "cl_max_cpu": 0, "cl_min_mem": 0, "cl_max_mem": 0, + "cl_force_restart": False, "supernode": False, "use_separate_vc": None, "vc_type": "", @@ -1540,6 +1545,7 @@ def default_participant(): "vc_max_cpu": 0, "vc_min_mem": 0, "vc_max_mem": 0, + "vc_force_restart": False, "use_remote_signer": None, "remote_signer_type": "web3signer", "remote_signer_image": "", diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index e785f28d3..a2e720ad4 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -16,6 +16,7 @@ PARTICIPANT_CATEGORIES = { "el_max_cpu", "el_min_mem", "el_max_mem", + "el_force_restart", "cl_type", "cl_image", "cl_binary_path", @@ -31,6 +32,7 @@ PARTICIPANT_CATEGORIES = { "cl_max_cpu", "cl_min_mem", "cl_max_mem", + "cl_force_restart", "supernode", "use_separate_vc", "vc_type", @@ -47,6 +49,7 @@ PARTICIPANT_CATEGORIES = { "vc_max_cpu", "vc_min_mem", "vc_max_mem", + "vc_force_restart", "validator_count", "use_remote_signer", "remote_signer_type", @@ -96,6 +99,7 @@ PARTICIPANT_MATRIX_PARAMS = { "el_max_cpu", "el_min_mem", "el_max_mem", + "el_force_restart", ], "cl": [ "cl_type", @@ -127,11 +131,13 @@ PARTICIPANT_MATRIX_PARAMS = { "vc_max_cpu", "vc_min_mem", "vc_max_mem", + "vc_force_restart", "validator_count", "count", "supernode", "vc_beacon_node_indices", "checkpoint_sync_enabled", + "cl_force_restart", ], "vc": [ "vc_type", @@ -148,6 +154,7 @@ PARTICIPANT_MATRIX_PARAMS = { "vc_max_cpu", "vc_min_mem", "vc_max_mem", + "vc_force_restart", "validator_count", ], "remote_signer": [ diff --git a/src/participant_network.star b/src/participant_network.star index a72a98f46..2de8a970e 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -50,7 +50,6 @@ def launch_participant_network( extra_files_artifacts, tempo_otlp_grpc_url, backend, - binary_artifacts={}, ): network_id = network_params.network_id num_participants = len(args_with_right_defaults.participants) @@ -161,6 +160,26 @@ def launch_participant_network( plan.print("Bootnodoor launched with ENR: {0}".format(bootnodoor_enr)) plan.print("Bootnodoor launched with ENODE: {0}".format(bootnodoor_enode)) + # Upload binary artifacts when both binary_path and force_restart are enabled + binary_artifacts = {} + for index, participant in enumerate(args_with_right_defaults.participants): + participant_binaries = {} + for bin_type, bin_path, force_restart in [ + ("el", participant.el_binary_path, participant.el_force_restart), + ("cl", participant.cl_binary_path, participant.cl_force_restart), + ("vc", participant.vc_binary_path, participant.vc_force_restart), + ]: + if bin_path and force_restart: + participant_binaries[bin_type] = struct( + artifact=plan.upload_files( + src="../" + bin_path, + name="{0}-binary-{1}".format(bin_type, index + 1), + ), + filename=bin_path.split("/")[-1], + ) + if participant_binaries: + binary_artifacts[index] = participant_binaries + # Launch all execution layer clients all_el_contexts = el_client_launcher.launch( plan, @@ -500,9 +519,7 @@ def launch_participant_network( remote_signer_context.metrics_info["config"] = participant.prometheus_config service_name = "vc-{0}".format(full_name) - vc_binary_artifact = None - if index in binary_artifacts and "vc" in binary_artifacts[index]: - vc_binary_artifact = binary_artifacts[index]["vc"] + vc_binary_artifact = binary_artifacts.get(index, {}).get("vc", None) vc_service_config = vc.get_vc_config( plan=plan, launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), @@ -538,13 +555,14 @@ def launch_participant_network( vc_service_info[service_name] = { "client_name": vc_type, "participant_index": index, + "participant": participant, } current_vc_index += 1 # add vc's in parallel to speed package execution - vc_services = {} - if len(vc_service_configs) > 0: - vc_services = plan.add_services(vc_service_configs) + vc_services = shared_utils.add_services_with_force_restart( + plan, vc_service_configs, vc_service_info, "vc_force_restart" + ) # Create VC contexts ordered by participant index vc_contexts_temp = {} diff --git a/src/shared_utils/shared_utils.star b/src/shared_utils/shared_utils.star index ee4023442..83147f81d 100644 --- a/src/shared_utils/shared_utils.star +++ b/src/shared_utils/shared_utils.star @@ -492,3 +492,29 @@ def get_tolerations( ) ) return toleration_list + + +def add_services_with_force_restart( + plan, service_configs, participant_info, force_restart_attr +): + force_restart_names = [ + name + for name in service_configs.keys() + if getattr(participant_info[name]["participant"], force_restart_attr, False) + == True + ] + + regular_configs = { + k: v for k, v in service_configs.items() if k not in force_restart_names + } + + services = {} + if len(regular_configs) > 0: + services = plan.add_services(regular_configs) + + for service_name in force_restart_names: + services[service_name] = plan.add_service( + service_name, service_configs[service_name], force_update=True + ) + + return services