feat: add force_restart - to be able to replace images/binaries without killing enclave

.github/tests/binary.norun..yaml

@@ -0,0 +1,10 @@
+participants:
+  - cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    count: 4
+  - el_type: geth
+    cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    cl_binary_path: binary/lighthouse
+additional_services:
+  - dora

.github/tests/binary_force.norun.yaml

@@ -0,0 +1,11 @@
+participants:
+  - cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    count: 4
+  - el_type: geth
+    cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    cl_binary_path: binary/lighthouse
+    cl_force_restart: true
+additional_services:
+  - dora

.github/tests/force.yaml

@@ -0,0 +1,8 @@
+participants:
+  - cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    count: 2
+  - el_type: geth
+    cl_type: lighthouse
+    cl_image: ethpandaops/lighthouse:unstable
+    cl_force_restart: true

.github/tests/minimal.yaml

@@ -3,13 +3,13 @@ participants:
     cl_type: teku
   - el_type: geth
     cl_type: prysm
- # - el_type: erigon
-  # cl_type: nimbus
-  #- el_type: besu
-   # cl_type: lighthouse
+  - el_type: erigon
+    cl_type: nimbus
+  - el_type: besu
+    cl_type: lighthouse
   - el_type: reth
     cl_type: lodestar
- # - el_type: geth
-  #  cl_type: grandine
+  - el_type: geth
+    cl_type: grandine
 network_params:
   preset: minimal

README.md

@@ -197,6 +197,7 @@ participants:
     # When set, the binary will be uploaded and mounted into the container,
     # replacing the default binary from the Docker image
     # Useful for rapid debugging with locally compiled binaries
+    # IMPORTANT: el_force_restart must be set to true when using this option
     # IMPORTANT: The binary file must live inside the ethereum-package directory
     # Build the client in its own repo, then copy ONLY the binary to ethereum-package
     # Do not run builds inside ethereum-package or copy build dependencies - only the final binary
@@ -271,6 +272,12 @@ participants:
     el_min_mem: 0
     el_max_mem: 0
 
+    # Force container recreation on next run (Docker only)
+    # When set to true, the container will be recreated even if the image tag hasn't changed
+    # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name
+    # Defaults to false
+    el_force_restart: false
+
   # CL(Consensus Layer) Specific flags
     # The type of CL client that should be started
     # Valid values are nimbus, lighthouse, lodestar, teku, prysm, and grandine
@@ -290,6 +297,7 @@ participants:
     # When set, the binary will be uploaded and mounted into the container,
     # replacing the default binary from the Docker image
     # Useful for rapid debugging with locally compiled binaries
+    # IMPORTANT: cl_force_restart must be set to true when using this option
     # IMPORTANT: The binary file must live inside the ethereum-package directory
     # Build the client in its own repo, then copy ONLY the binary to ethereum-package
     # Do not run builds inside ethereum-package or copy build dependencies - only the final binary
@@ -355,6 +363,12 @@ participants:
     cl_min_mem: 0
     cl_max_mem: 0
 
+    # Force container recreation on next run (Docker only)
+    # When set to true, the container will be recreated even if the image tag hasn't changed
+    # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name
+    # Defaults to false
+    cl_force_restart: false
+
     # Whether to act as a supernode for the network
     # Supernodes will subscribe to all subnet topics
     # This flag should only be used with peerdas
@@ -386,6 +400,7 @@ participants:
     # When set, the binary will be uploaded and mounted into the container,
     # replacing the default binary from the Docker image
     # Useful for rapid debugging with locally compiled binaries
+    # IMPORTANT: vc_force_restart must be set to true when using this option
     # IMPORTANT: The binary file must live inside the ethereum-package directory
     # Build the client in its own repo, then copy ONLY the binary to ethereum-package
     # Do not run builds inside ethereum-package or copy build dependencies - only the final binary
@@ -446,6 +461,12 @@ participants:
     vc_min_mem: 0
     vc_max_mem: 0
 
+    # Force container recreation on next run (Docker only)
+    # When set to true, the container will be recreated even if the image tag hasn't changed
+    # Useful when rebuilding Docker images with the same tag or recompiling binaries with the same name
+    # Defaults to false
+    vc_force_restart: false
+
     # A list of indices of the beacon nodes that the validator client should connect to
     # Defaults to null
     vc_beacon_node_indices: null

main.star

@@ -100,38 +100,19 @@ def run(plan, args={}):
             artifact = plan.render_templates(template_data, name + "_artifact")
             extra_files_artifacts[name] = artifact
 
-    # Process binary injection - upload local binaries for participants
-    # NOTE: Binary injection is only supported with Docker backend
-    binary_artifacts = {}
-    for index, participant in enumerate(args_with_right_defaults.participants):
-        participant_binaries = {}
-        for bin_type, bin_path in [
-            ("el", participant.el_binary_path),
-            ("cl", participant.cl_binary_path),
-            ("vc", participant.vc_binary_path),
+    # Validate binary injection - only supported with Docker backend
+    for participant in args_with_right_defaults.participants:
+        for bin_path in [
+            participant.el_binary_path,
+            participant.cl_binary_path,
+            participant.vc_binary_path,
         ]:
-            if bin_path:
-                if detected_backend != "docker":
-                    fail(
-                        "Binary injection (*_binary_path) is only supported with Docker backend, detected: {0}".format(
-                            detected_backend
-                        )
-                    )
-                plan.print(
-                    "Uploading {0} binary for participant {1}: {2}".format(
-                        bin_type.upper(), index + 1, bin_path
+            if bin_path and detected_backend != "docker":
+                fail(
+                    "Binary injection (*_binary_path) is only supported with Docker backend, detected: {0}".format(
+                        detected_backend
                     )
                 )
-                # Extract filename from path and store both artifact and filename
-                filename = bin_path.split("/")[-1]
-                participant_binaries[bin_type] = struct(
-                    artifact=plan.upload_files(
-                        src=bin_path, name="{0}-binary-{1}".format(bin_type, index + 1)
-                    ),
-                    filename=filename,
-                )
-        if participant_binaries:
-            binary_artifacts[index] = participant_binaries
 
     mev_params = args_with_right_defaults.mev_params
     parallel_keystore_generation = args_with_right_defaults.parallel_keystore_generation
@@ -291,7 +272,6 @@ def run(plan, args={}):
         extra_files_artifacts,
         tempo_otlp_grpc_url,
         detected_backend,
-        binary_artifacts,
     )
 
     plan.print(

src/cl/cl_launcher.star

@@ -227,10 +227,7 @@ def launch(
         all_snooper_el_engine_contexts.append(snooper_el_engine_context)
         full_name = "{0}-{1}-{2}".format(index_str, el_type, cl_type)
 
-        # Get binary artifact for this participant if it exists
-        cl_binary_artifact = None
-        if index in binary_artifacts and "cl" in binary_artifacts[index]:
-            cl_binary_artifact = binary_artifacts[index]["cl"]
+        cl_binary_artifact = binary_artifacts.get(index, {}).get("cl", None)
 
         if index == 0:
             cl_context = launch_method(
@@ -319,12 +316,13 @@ def launch(
                 "get_cl_context": get_cl_context,
                 "get_blobber_config": get_blobber_config,
                 "participant_index": index,
+                "cl_type": cl_type,
             }
 
     # add rest of cl's in parallel to speed package execution
-    cl_services = {}
-    if len(cl_service_configs) > 0:
-        cl_services = plan.add_services(cl_service_configs)
+    cl_services = shared_utils.add_services_with_force_restart(
+        plan, cl_service_configs, cl_participant_info, "cl_force_restart"
+    )
 
     # Create CL contexts ordered by participant index
     cl_contexts_temp = {}

src/cl/lighthouse/lighthouse_launcher.star

@@ -63,7 +63,6 @@ def launch(
     bootnode_enr_override=None,
     cl_binary_artifact=None,
 ):
-    # Launch Beacon node
     beacon_config = get_beacon_config(
         plan,
         launcher,
@@ -90,7 +89,9 @@ def launch(
         cl_binary_artifact,
     )
 
-    beacon_service = plan.add_service(beacon_service_name, beacon_config)
+    beacon_service = plan.add_service(
+        beacon_service_name, beacon_config, force_update=participant.cl_force_restart
+    )
 
     cl_context_obj = get_cl_context(
         plan,

src/el/besu/besu_launcher.star

@@ -68,7 +68,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/dummy/dummy_launcher.star

@@ -58,7 +58,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/el_launcher.star

@@ -175,11 +175,7 @@ def launch(
         index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
 
         el_service_name = "el-{0}-{1}-{2}".format(index_str, el_type, cl_type)
-
-        # Get binary artifact for this participant if it exists
-        el_binary_artifact = None
-        if index in binary_artifacts and "el" in binary_artifacts[index]:
-            el_binary_artifact = binary_artifacts[index]["el"]
+        el_binary_artifact = binary_artifacts.get(index, {}).get("el", None)
 
         if index == 0:
             el_context = launch_method(
@@ -234,9 +230,9 @@ def launch(
             }
 
     # add remainder of el's in parallel to speed package execution
-    el_services = {}
-    if len(el_service_configs) > 0:
-        el_services = plan.add_services(el_service_configs)
+    el_services = shared_utils.add_services_with_force_restart(
+        plan, el_service_configs, el_participant_info, "el_force_restart"
+    )
 
     # Create contexts ordered by participant index
     el_contexts_temp = {}

src/el/erigon/erigon_launcher.star

@@ -66,7 +66,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/ethereumjs/ethereumjs_launcher.star

@@ -68,7 +68,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/ethrex/ethrex_launcher.star

@@ -83,7 +83,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/geth/geth_launcher.star

@@ -72,9 +72,12 @@ def launch(
         network_params,
         extra_files_artifacts,
         bootnodoor_enode,
+        el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/nethermind/nethermind_launcher.star

@@ -64,7 +64,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/nimbus-eth1/nimbus_launcher.star

@@ -64,7 +64,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,

src/el/reth/reth_launcher.star

@@ -73,7 +73,9 @@ def launch(
         el_binary_artifact,
     )
 
-    service = plan.add_service(service_name, config)
+    service = plan.add_service(
+        service_name, config, force_update=participant.el_force_restart
+    )
 
     return get_el_context(
         plan,