feat(proxyd): add proxyd support

README.md

@@ -385,6 +385,17 @@ optimism_package:
         # Defaults to True
         fund_dev_accounts: true
 
+      # Default proxyd configuration
+      proxyd_params:
+        # The Docker image that should be used for proxyd; leave blank to use the default image
+        image: "us-docker.pkg.dev/oplabs-tools-artifacts/images/proxyd"
+
+        # The Docker tag that should be used for proxyd; leave blank to use the default tag
+        tag: ""
+
+        # A list of optional extra params that will be passed to the proxyd container
+        extra_params: []
+
       # Default batcher configuration
       batcher_params:
         # The Docker image that should be used for the batcher; leave blank to use the default op-batcher image
@@ -653,7 +664,7 @@ Compile [tx-fuzz](https://github.com/MariusVanDerWijden/tx-fuzz) locally per ins
 
 Install the latest [contender](https://github.com/flashbots/contender) version via cargo:
 ```bash
-cargo install --git https://github.com/flashbots/contender --bin contender --force 
+cargo install --git https://github.com/flashbots/contender --bin contender --force
 ```
 
 Browse the available [scenarios](https://github.com/flashbots/contender/tree/main/scenarios) and pick one that fits your needs. For example, to download the `stress` scenario:

src/l2.star

@@ -24,6 +24,7 @@ def launch_l2(
     interop_params,
 ):
     network_params = l2_args.network_params
+    proxyd_params = l2_args.proxyd_params
     batcher_params = l2_args.batcher_params
     challenger_params = l2_args.challenger_params
     proposer_params = l2_args.proposer_params
@@ -50,6 +51,7 @@ def launch_l2(
         l2_args.participants,
         jwt_file,
         network_params,
+        proxyd_params,
         batcher_params,
         challenger_params,
         proposer_params,

src/observability/loki/loki_launcher.star

@@ -11,11 +11,12 @@ GRPC_PORT_NUMBER = 9096
 
 TEMPLATES_FILEPATH = "./templates"
 
-CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/loki-config.yaml.tmpl"
-CONFIG_REL_FILEPATH = "loki-config.yaml"
+CONFIG_FILE_NAME = "loki-config.yaml"
+CONFIG_TEMPLATE_FILEPATH = "{0}/{1}.tmpl".format(TEMPLATES_FILEPATH, CONFIG_FILE_NAME)
 
 CONFIG_DIRPATH_ON_SERVICE = "/config"
 
+
 USED_PORTS = {
     constants.HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
         HTTP_PORT_NUMBER,
@@ -70,7 +71,7 @@ def create_config_artifact(
     )
 
     template_and_data_by_rel_dest_filepath = {
-        CONFIG_REL_FILEPATH: config_template_and_data,
+        CONFIG_FILE_NAME: config_template_and_data,
     }
 
     config_artifact_name = plan.render_templates(
@@ -89,9 +90,7 @@ def get_service_config(
         image=loki_params.image,
         ports=USED_PORTS,
         cmd=[
-            "-config.file={0}/{1}".format(
-                CONFIG_DIRPATH_ON_SERVICE, CONFIG_REL_FILEPATH
-            ),
+            "-config.file={0}/{1}".format(CONFIG_DIRPATH_ON_SERVICE, CONFIG_FILE_NAME),
         ],
         files={
             CONFIG_DIRPATH_ON_SERVICE: config_artifact_name,

src/observability/observability.star

@@ -40,7 +40,9 @@ def new_metrics_info(helper, service, metrics_path=METRICS_PATH):
 
 def expose_metrics_port(ports, port_id=METRICS_PORT_ID, port_num=METRICS_PORT_NUM):
     ports[port_id] = ethereum_package_shared_utils.new_port_spec(
-        port_num, ethereum_package_shared_utils.TCP_PROTOCOL
+        port_num,
+        ethereum_package_shared_utils.TCP_PROTOCOL,
+        ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
     )
 
 

src/package_io/input_parser.star

@@ -211,6 +211,11 @@ def input_parser(plan, input_args):
                     interop_time_offset=result["network_params"]["interop_time_offset"],
                     fund_dev_accounts=result["network_params"]["fund_dev_accounts"],
                 ),
+                proxyd_params=struct(
+                    image=result["proxyd_params"]["image"],
+                    tag=result["proxyd_params"]["tag"],
+                    extra_params=result["proxyd_params"]["extra_params"],
+                ),
                 batcher_params=struct(
                     image=result["batcher_params"]["image"],
                     extra_params=result["batcher_params"]["extra_params"],
@@ -321,6 +326,9 @@ def parse_network_params(plan, input_args):
         network_params = default_network_params()
         network_params.update(chain.get("network_params", {}))
 
+        proxyd_params = default_proxyd_params()
+        proxyd_params.update(chain.get("proxyd_params", {}))
+
         batcher_params = default_batcher_params()
         batcher_params.update(chain.get("batcher_params", {}))
 
@@ -406,6 +414,7 @@ def parse_network_params(plan, input_args):
         result = {
             "participants": participants,
             "network_params": network_params,
+            "proxyd_params": proxyd_params,
             "batcher_params": batcher_params,
             "challenger_params": challenger_params,
             "proposer_params": proposer_params,
@@ -531,6 +540,7 @@ def default_chains():
         {
             "participants": [default_participant()],
             "network_params": default_network_params(),
+            "proxyd_params": default_proxyd_params(),
             "batcher_params": default_batcher_params(),
             "proposer_params": default_proposer_params(),
             "challenger_params": default_challenger_params(),
@@ -563,6 +573,14 @@ def default_batcher_params():
     }
 
 
+def default_proxyd_params():
+    return {
+        "image": "us-docker.pkg.dev/oplabs-tools-artifacts/images/proxyd",
+        "tag": "v4.14.2",
+        "extra_params": [],
+    }
+
+
 def default_challenger_params():
     return {
         "enabled": True,

src/package_io/sanity_check.star

@@ -143,6 +143,7 @@ SUBCATEGORY_PARAMS = {
         "interop_time_offset",
         "fund_dev_accounts",
     ],
+    "proxyd_params": ["image", "tag", "extra_params"],
     "batcher_params": ["image", "extra_params"],
     "proposer_params": ["image", "extra_params", "game_type", "proposal_interval"],
     "challenger_params": [

src/participant_network.star

@@ -6,6 +6,7 @@ op_challenger_launcher = import_module(
     "./challenger/op-challenger/op_challenger_launcher.star"
 )
 op_proposer_launcher = import_module("./proposer/op-proposer/op_proposer_launcher.star")
+proxyd_launcher = import_module("./proxyd/proxyd_launcher.star")
 util = import_module("./util.star")
 
 
@@ -14,6 +15,7 @@ def launch_participant_network(
     participants,
     jwt_file,
     network_params,
+    proxyd_params,
     batcher_params,
     challenger_params,
     proposer_params,
@@ -70,6 +72,14 @@ def launch_participant_network(
 
         all_participants.append(participant_entry)
 
+    proxyd_launcher.launch(
+        plan,
+        proxyd_params,
+        network_params,
+        all_el_contexts,
+        observability_helper,
+    )
+
     batcher_key = util.read_network_config_value(
         plan,
         deployment_output,

src/proxyd/proxyd_launcher.star

@@ -0,0 +1,133 @@
+ethereum_package_shared_utils = import_module(
+    "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star"
+)
+
+ethereum_package_constants = import_module(
+    "github.com/ethpandaops/ethereum-package/src/package_io/constants.star"
+)
+
+constants = import_module("../package_io/constants.star")
+util = import_module("../util.star")
+
+observability = import_module("../observability/observability.star")
+prometheus = import_module("../observability/prometheus/prometheus_launcher.star")
+
+# Port nums
+HTTP_PORT_NUM = 8080
+METRICS_PORT_NUM = 7300
+
+TEMPLATES_FILEPATH = "./templates"
+
+CONFIG_FILE_NAME = "proxyd.toml"
+CONFIG_TEMPLATE_FILEPATH = "{0}/{1}.tmpl".format(TEMPLATES_FILEPATH, CONFIG_FILE_NAME)
+
+CONFIG_DIRPATH_ON_SERVICE = "/etc/proxyd"
+
+
+def get_used_ports():
+    used_ports = {
+        constants.HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
+            HTTP_PORT_NUM,
+            ethereum_package_shared_utils.TCP_PROTOCOL,
+            ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ),
+    }
+    return used_ports
+
+
+def launch(
+    plan,
+    proxyd_params,
+    network_params,
+    el_contexts,
+    observability_helper,
+):
+    config_template = read_file(CONFIG_TEMPLATE_FILEPATH)
+
+    config_artifact_name = create_config_artifact(
+        plan,
+        config_template,
+        el_contexts,
+        observability_helper,
+    )
+
+    config = get_proxyd_config(
+        plan,
+        proxyd_params,
+        config_artifact_name,
+        observability_helper,
+    )
+
+    service = plan.add_service("proxyd-{0}".format(network_params.network), config)
+    service_url = util.make_service_http_url(service)
+
+    observability.register_op_service_metrics_job(
+        observability_helper, service, network_params.network
+    )
+
+    return service_url
+
+
+def create_config_artifact(
+    plan,
+    config_template,
+    el_contexts,
+    observability_helper,
+):
+    config_data = {
+        "Ports": {
+            "rpc": HTTP_PORT_NUM,
+        },
+        "Metrics": {
+            "enabled": observability_helper.enabled,
+            "port": METRICS_PORT_NUM,
+        },
+        "Replicas": {
+            "{0}-{1}".format(el_context.client_name, num): el_context.rpc_http_url
+            for num, el_context in enumerate(el_contexts)
+        },
+    }
+
+    config_template_and_data = ethereum_package_shared_utils.new_template_and_data(
+        config_template, config_data
+    )
+
+    config_artifact_name = plan.render_templates(
+        {
+            CONFIG_FILE_NAME: config_template_and_data,
+        },
+        name="proxyd-config",
+    )
+
+    return config_artifact_name
+
+
+def get_proxyd_config(
+    plan,
+    proxyd_params,
+    config_artifact_name,
+    observability_helper,
+):
+    ports = dict(get_used_ports())
+
+    cmd = [
+        "proxyd",
+        "{0}/{1}".format(CONFIG_DIRPATH_ON_SERVICE, CONFIG_FILE_NAME),
+    ]
+
+    # apply customizations
+
+    if observability_helper.enabled:
+        observability.expose_metrics_port(ports, port_num=METRICS_PORT_NUM)
+
+    cmd += proxyd_params.extra_params
+
+    return ServiceConfig(
+        image="{0}:{1}".format(proxyd_params.image, proxyd_params.tag),
+        ports=ports,
+        cmd=cmd,
+        files={
+            CONFIG_DIRPATH_ON_SERVICE: config_artifact_name,
+        },
+        private_ip_address_placeholder=ethereum_package_constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
+    )

src/proxyd/templates/proxyd.toml.tmpl

@@ -0,0 +1,56 @@
+[server]
+rpc_host = "0.0.0.0"
+rpc_port = {{ .Ports.rpc }}
+
+{{- if .Metrics.enabled }}
+[metrics]
+enabled = true
+host = "0.0.0.0"
+port = {{ .Metrics.port }}
+{{- end }}
+
+[backends]
+{{- range $key, $value := .Replicas }}
+[backends.{{ $key }}]
+rpc_url = "{{ $value }}"
+ws_url = "ws://dummy"
+{{- end }}
+
+[backend_groups]
+[backend_groups.replica]
+backends = [{{ range $key, $value := .Replicas }}"{{ $key }}",{{ end }}]
+
+[rpc_method_mappings]
+eth_getProof = "replica"
+eth_gasPrice = "replica"
+eth_sendRawTransaction = "replica"
+eth_chainId = "replica"
+eth_blockNumber = "replica"
+net_version = "replica"
+eth_getBlockByHash = "replica"
+eth_getBlockByNumber = "replica"
+eth_getUncleByBlockHashAndIndex = "replica"
+eth_getTransactionByHash = "replica"
+eth_getBlockTransactionCountByHash = "replica"
+eth_getTransactionByBlockHashAndIndex = "replica"
+eth_getTransactionReceipt = "replica"
+eth_maxPriorityFeePerGas = "replica"
+eth_feeHistory = "replica"
+eth_syncing = "replica"
+eth_getLogs = "replica"
+eth_getBalance = "replica"
+eth_getStorageAt = "replica"
+eth_getCode = "replica"
+eth_getTransactionCount = "replica"
+eth_getBlockTransactionCountByNumber  = "replica"
+eth_call = "replica"
+eth_estimateGas = "replica"
+debug_traceTransaction = "replica"
+debug_traceBlockByNumber = "replica"
+debug_traceBlockByHash = "replica"
+debug_storageRangeAt = "replica"
+rollup_getInfo = "replica"
+eth_getBlockRange = "replica"
+web3_clientVersion = "replica"
+eth_accounts = "replica"
+debug_dbGet = "replica"