feat: add Tempo as an additional service to collect Lighthouse tracing data

README.md

@@ -757,6 +757,7 @@ additional_services:
   - full_beaconchain_explorer
   - prometheus
   - grafana
+  - tempo
   - blobscan
   - dugtrio
   - blutgang
@@ -833,6 +834,29 @@ grafana_params:
   # Defaults to the latest image
   image: "grafana/grafana:latest"
 
+# Configuration place for tempo tracing backend
+tempo_params:
+  # How long to retain traces
+  retention_duration: "12h"
+  # Rate limiting for trace ingestion (bytes per second)
+  ingestion_rate_limit: 20971520  # 20MB
+  # Burst limit for trace ingestion (bytes)
+  ingestion_burst_limit: 52428800  # 50MB
+  # Maximum duration for trace searches
+  max_search_duration: "30s"
+  # Maximum bytes per individual trace
+  max_bytes_per_trace: 52428800  # 50MB
+  # Resource management for tempo container
+  # CPU is milicores
+  # RAM is in MB
+  min_cpu: 10
+  max_cpu: 1000
+  min_mem: 128
+  max_mem: 2048
+  # Tempo docker image to use
+  # Defaults to the latest image
+  image: "grafana/tempo:latest"
+
 # Configuration place for the assertoor testing tool - https://github.com/ethpandaops/assertoor
 assertoor_params:
   # Assertoor docker image to use

main.star

@@ -28,6 +28,7 @@ full_beaconchain_explorer = import_module(
 blockscout = import_module("./src/blockscout/blockscout_launcher.star")
 prometheus = import_module("./src/prometheus/prometheus_launcher.star")
 grafana = import_module("./src/grafana/grafana_launcher.star")
+tempo = import_module("./src/tempo/tempo_launcher.star")
 commit_boost_mev_boost = import_module(
     "./src/mev/commit-boost/mev_boost/mev_boost_launcher.star"
 )
@@ -128,6 +129,7 @@ def run(plan, args={}):
     grafana_dashboards_config_template = read_file(
         static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH
     )
+    tempo_config_template = read_file(static_files.TEMPO_CONFIG_TEMPLATE_FILEPATH)
     prometheus_additional_metrics_jobs = []
     raw_jwt_secret = read_file(static_files.JWT_PATH_FILEPATH)
     jwt_file = plan.upload_files(
@@ -175,6 +177,16 @@ def run(plan, args={}):
             )
     plan.print("Read the prometheus, grafana templates")
 
+    tempo_otlp_grpc_url = None
+    tempo_query_url = None
+    if "tempo" in args_with_right_defaults.additional_services:
+        tempo_otlp_grpc_url = "http://{}:{}".format(
+            tempo.SERVICE_NAME, tempo.OTLP_GRPC_PORT_NUMBER
+        )
+        tempo_query_url = "http://{}:{}".format(
+            tempo.SERVICE_NAME, tempo.HTTP_PORT_NUMBER
+        )
+
     if args_with_right_defaults.mev_type == constants.MEV_RS_MEV_TYPE:
         plan.print("Generating mev-rs builder config file")
         mev_rs_builder_config_file = mev_rs_mev_builder.new_builder_config(
@@ -227,6 +239,7 @@ def run(plan, args={}):
         keymanager_enabled,
         parallel_keystore_generation,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     plan.print(
@@ -712,8 +725,21 @@ def run(plan, args={}):
                 args_with_right_defaults.grafana_params,
                 args_with_right_defaults.port_publisher,
                 index,
+                tempo_query_url,
             )
             plan.print("Successfully launched grafana")
+        elif additional_service == "tempo":
+            plan.print("Launching tempo...")
+            tempo.launch_tempo(
+                plan,
+                tempo_config_template,
+                global_node_selectors,
+                global_tolerations,
+                args_with_right_defaults.tempo_params,
+                args_with_right_defaults.port_publisher,
+                index,
+            )
+            plan.print("Successfully launched tempo")
         elif additional_service == "prometheus_grafana":
             # Allow prometheus to be launched last so is able to collect metrics from other services
             launch_prometheus_grafana = True
@@ -802,6 +828,7 @@ def run(plan, args={}):
             args_with_right_defaults.grafana_params,
             args_with_right_defaults.port_publisher,
             prometheus_grafana_index,
+            tempo_query_url,
         )
         plan.print("Successfully launched grafana")
 

src/cl/cl_launcher.star

@@ -27,6 +27,7 @@ def launch(
     global_node_selectors,
     global_tolerations,
     persistent,
+    tempo_otlp_grpc_url,
     num_participants,
     validator_data,
     prysm_password_relative_filepath,
@@ -220,6 +221,7 @@ def launch(
                 index,
                 network_params,
                 extra_files_artifacts,
+                tempo_otlp_grpc_url,
             )
 
             blobber_config = get_blobber_config(
@@ -268,6 +270,7 @@ def launch(
                 index,
                 network_params,
                 extra_files_artifacts,
+                tempo_otlp_grpc_url,
             )
 
             cl_participant_info[cl_service_name] = {

src/cl/grandine/grandine_launcher.star

@@ -53,6 +53,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     config = get_beacon_config(
         plan,
@@ -74,6 +75,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, config)
@@ -111,6 +113,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS

src/cl/lighthouse/lighthouse_launcher.star

@@ -58,6 +58,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     # Launch Beacon node
     beacon_config = get_beacon_config(
@@ -80,6 +81,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, beacon_config)
@@ -117,6 +119,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS
@@ -256,6 +259,10 @@ def get_beacon_config(
     else:  # Public networks
         cmd.append("--network=" + network_params.network)
 
+    # Add tempo telemetry integration if tempo is enabled
+    if tempo_otlp_grpc_url != None:
+        cmd.append("--telemetry-collector-url={}".format(tempo_otlp_grpc_url))
+
     if len(participant.cl_extra_params) > 0:
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in participant.cl_extra_params])

src/cl/lodestar/lodestar_launcher.star

@@ -45,6 +45,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     # Launch Beacon node
     beacon_config = get_beacon_config(
@@ -67,6 +68,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, beacon_config)
@@ -104,6 +106,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_urlk,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS

src/cl/nimbus/nimbus_launcher.star

@@ -66,6 +66,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     beacon_config = get_beacon_config(
         plan,
@@ -87,6 +88,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, beacon_config)
@@ -124,6 +126,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS

src/cl/prysm/prysm_launcher.star

@@ -49,6 +49,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     beacon_config = get_beacon_config(
         plan,
@@ -70,6 +71,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, beacon_config)
@@ -107,6 +109,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS

src/cl/teku/teku_launcher.star

@@ -53,6 +53,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url=None,
 ):
     config = get_beacon_config(
         plan,
@@ -74,6 +75,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
+        tempo_otlp_grpc_url,
     )
 
     beacon_service = plan.add_service(beacon_service_name, config)
@@ -111,6 +113,7 @@ def get_beacon_config(
     participant_index,
     network_params,
     extra_files_artifacts,
+    tempo_otlp_grpc_url,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant.cl_log_level, global_log_level, VERBOSITY_LEVELS

src/grafana/grafana_launcher.star

@@ -49,6 +49,7 @@ def launch_grafana(
     grafana_params,
     port_publisher,
     index,
+    tempo_query_url=None,
 ):
     tolerations = input_parser.get_client_tolerations([], [], global_tolerations)
 
@@ -61,6 +62,7 @@ def launch_grafana(
         datasource_config_template,
         dashboard_providers_config_template,
         prometheus_private_url,
+        tempo_query_url,
         additional_dashboards=grafana_params.additional_dashboards,
     )
 
@@ -94,9 +96,12 @@ def get_grafana_config_dir_artifact_uuid(
     datasource_config_template,
     dashboard_providers_config_template,
     prometheus_private_url,
+    tempo_query_url,
     additional_dashboards=[],
 ):
-    datasource_data = new_datasource_config_template_data(prometheus_private_url)
+    datasource_data = new_datasource_config_template_data(
+        prometheus_private_url, tempo_query_url
+    )
     datasource_template_and_data = shared_utils.new_template_and_data(
         datasource_config_template, datasource_data
     )
@@ -167,8 +172,11 @@ def get_config(
     )
 
 
-def new_datasource_config_template_data(prometheus_url):
-    return {"PrometheusURL": prometheus_url}
+def new_datasource_config_template_data(prometheus_url, tempo_query_url):
+    data = {"PrometheusURL": prometheus_url}
+    if tempo_query_url != None:
+        data["TempoURL"] = tempo_query_url
+    return data
 
 
 def new_dashboard_providers_config_template_data(dashboards_dirpath):