chore: Support multiple supervisors: Decouple challengers from networks [4/N]

README.md

@@ -410,23 +410,6 @@ optimism_package:
         # A list of optional extra params that will be passed to the batcher container for modifying its behaviour
         extra_params: []
 
-      # Default challenger configuration
-      challenger_params:
-        # Whether or not to enable the challenger
-        enabled: true
-
-        # The Docker image that should be used for the challenger; leave blank to use the default op-challenger image
-        image: ""
-
-        # A list of optional extra params that will be passed to the challenger container for modifying its behaviour
-        extra_params: []
-
-        # Path to folder containing cannon prestate-proof.json file
-        cannon_prestates_path: "static_files/prestates"
-
-        # Base URL to absolute prestates to use when generating trace data.
-        cannon_prestates_url: ""
-
       # Default proposer configuration
       proposer_params:
         # The Docker image that should be used for the proposer; leave blank to use the default op-proposer image
@@ -477,6 +460,36 @@ optimism_package:
           - "--port=3100"
           - "--log.level=debug"
 
+  challengers:
+    my-challenger:
+      # Whether this challenger is active
+      enabled: true
+
+      # The Docker image that should be used for the challenger; leave blank to use the default op-challenger image
+      image: ""
+
+      # List of L2 chains that this challenger is connected to
+      # 
+      # This field accepts several configuration types:
+      # 
+      # A list of network IDs, in which case the challenger will connect to all the nodes in these network
+      participants: ["2151908"]
+
+      # OR "*" meaning the challenger will connect to all nodes of all L2 networks
+      participants: "*"
+
+      # A list of optional extra params that will be passed to the challenger container for modifying its behaviour
+      extra_params: []
+
+      # Path to folder containing cannon prestate-proof.json file
+      cannon_prestates_path: "static_files/prestates"
+
+      # OR Base URL to absolute prestates to use when generating trace data.
+      cannon_prestates_url: ""
+
+      # Directory in which the challenger will store its data
+      datadir: "/data/op-challenger/op-challenger-data"
+
   # L2 contract deployer configuration - used for all L2 networks.
   # The docker image that should be used for the L2 contract deployer.
   # Locators can be http(s) URLs, or point to an enclave artifact with

main.star

@@ -2,9 +2,7 @@ ethereum_package = import_module("github.com/ethpandaops/ethereum-package/main.s
 contract_deployer = import_module("./src/contracts/contract_deployer.star")
 l2_launcher = import_module("./src/l2.star")
 op_supervisor_launcher = import_module("./src/interop/op-supervisor/launcher.star")
-op_challenger_launcher = import_module(
-    "./src/challenger/op-challenger/op_challenger_launcher.star"
-)
+op_challenger_launcher = import_module("./src/challenger/op-challenger/launcher.star")
 
 faucet = import_module("./src/faucet/op-faucet/op_faucet_launcher.star")
 observability = import_module("./src/observability/observability.star")
@@ -123,8 +121,9 @@ def run(plan, args={}):
             )
         )
 
+    supervisor = None
     if interop_params.enabled:
-        op_supervisor_launcher.launch(
+        supervisor = op_supervisor_launcher.launch(
             plan,
             l1_config_env_vars,
             optimism_args.chains,
@@ -134,29 +133,16 @@ def run(plan, args={}):
             observability_helper,
         )
 
-    # challenger must launch after supervisor because it depends on it for interop
-    for l2_num, l2 in enumerate(l2s):
-        chain = optimism_args.chains[l2_num]
-        op_challenger_image = (
-            chain.challenger_params.image
-            if chain.challenger_params.image != ""
-            else input_parser.DEFAULT_CHALLENGER_IMAGES["op-challenger"]
+    for challenger_params in optimism_args.challengers:
+        op_challenger_launcher.launch(
+            plan=plan,
+            params=challenger_params,
+            l2s=l2s,
+            supervisor=supervisor,
+            l1_config_env_vars=l1_config_env_vars,
+            deployment_output=deployment_output,
+            observability_helper=observability_helper,
         )
-        if chain.challenger_params.enabled:
-            op_challenger_launcher.launch(
-                plan,
-                l2_num,
-                "op-challenger-{0}".format(chain.network_params.name),
-                chain.challenger_params.image,
-                l2.participants[0].el_context,
-                l2.participants[0].cl_context,
-                l1_config_env_vars,
-                deployment_output,
-                chain.network_params,
-                chain.challenger_params,
-                interop_params,
-                observability_helper,
-            )
 
     if optimism_args.faucet.enabled:
         _install_faucet(

network_params.yaml

@@ -26,6 +26,8 @@ optimism_package:
         node_selectors: {}
         tolerations: []
         count: 1
+      - el_type: op-reth
+        cl_type: op-node
       network_params:
         network: "kurtosis"
         network_id: "2151908"
@@ -36,9 +38,6 @@ optimism_package:
         fund_dev_accounts: true
       batcher_params:
         extra_params: []
-      challenger_params:
-        enabled: true
-        extra_params: []
       mev_params:
         builder_host: ""
         builder_port: ""
@@ -53,6 +52,14 @@ optimism_package:
   global_node_selectors: {}
   global_tolerations: []
   persistent: false
+  challengers:
+    challenger-0: &x-challenger
+      participants: "*"
+      cannon_trace_types: ["super-cannon", "super-permissioned"]
+    challenger-1:
+      <<: *x-challenger
+  interop:
+    enabled: True
 ethereum_package:
   participants:
   - el_type: geth

src/challenger/input_parser.star

@@ -0,0 +1,83 @@
+_expansion = import_module("/src/util/expansion.star")
+_filter = import_module("/src/util/filter.star")
+
+_DEFAULT_ARGS = {
+    "enabled": True,
+    "image": "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-challenger:develop",
+    "extra_params": [],
+    "participants": "*",
+    "cannon_prestate_path": "",
+    "cannon_prestates_url": "https://storage.googleapis.com/oplabs-network-data/proofs/op-program/cannon",
+    "cannon_trace_types": [],
+    "datadir": "/data/op-challenger/op-challenger-data",
+}
+
+
+def parse(args, chains):
+    return _filter.remove_none(
+        [
+            _parse_instance(challenger_args or {}, challenger_name, chains)
+            for challenger_name, challenger_args in (args or {}).items()
+        ]
+    )
+
+
+def _parse_instance(challenger_args, challenger_name, chains):
+    # Any extra attributes will cause an error
+    extra_keys = _filter.remove_keys(challenger_args, _DEFAULT_ARGS.keys())
+    if len(extra_keys) > 0:
+        fail(
+            "Invalid attributes in challenger configuration for {}: {}".format(
+                challenger_name, ",".join(extra_keys)
+            )
+        )
+
+    # We filter the None values so that we can merge dicts easily
+    # and merge the config with the defaults
+    challenger_params = _DEFAULT_ARGS | _filter.remove_none(challenger_args)
+
+    if not challenger_params["enabled"]:
+        return None
+
+    # We expand the list of participants since we support a special "*" value to include all networks
+    network_ids = [c["network_params"]["network_id"] for c in chains]
+    challenger_params["participants"] = _expansion.expand_asterisc(
+        challenger_params["participants"],
+        network_ids,
+        missing_value_message="network ID {0} does not exist, please check challenger configuration for challenger "
+        + challenger_name,
+    )
+
+    # No participants means that the challenger is disabled
+    if len(challenger_params["participants"]) == 0:
+        return None
+
+    # We add name & service name
+    challenger_params["name"] = challenger_name
+    challenger_params["service_name"] = "op-challenger-{}".format(challenger_name)
+
+    # Now we make sure to cover the prestate arg combinations
+    #
+    # First we check we only have one of them defined
+    if (
+        challenger_params["cannon_prestate_path"]
+        and challenger_params["cannon_prestates_url"]
+    ):
+        fail(
+            "Only one of cannon_prestate_path and cannon_prestates_url can be set for challenger {}".format(
+                challenger_name
+            )
+        )
+
+    # And we also need to make sure we have at least one of them defined
+    if (
+        not challenger_params["cannon_prestate_path"]
+        and not challenger_params["cannon_prestates_url"]
+    ):
+        fail(
+            "At least one of cannon_prestate_path and cannon_prestates_url must be set for challenger {}".format(
+                challenger_name
+            )
+        )
+
+    return struct(**challenger_params)