diff --git a/README.md b/README.md index 2ecab0f6e..5ecb500b7 100644 --- a/README.md +++ b/README.md @@ -757,6 +757,7 @@ additional_services: - full_beaconchain_explorer - prometheus - grafana + - tempo - blobscan - dugtrio - blutgang @@ -833,6 +834,29 @@ grafana_params: # Defaults to the latest image image: "grafana/grafana:latest" +# Configuration place for tempo tracing backend +tempo_params: + # How long to retain traces + retention_duration: "12h" + # Rate limiting for trace ingestion (bytes per second) + ingestion_rate_limit: 20971520 # 20MB + # Burst limit for trace ingestion (bytes) + ingestion_burst_limit: 52428800 # 50MB + # Maximum duration for trace searches + max_search_duration: "30s" + # Maximum bytes per individual trace + max_bytes_per_trace: 52428800 # 50MB + # Resource management for tempo container + # CPU is milicores + # RAM is in MB + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 + # Tempo docker image to use + # Defaults to the latest image + image: "grafana/tempo:latest" + # Configuration place for the assertoor testing tool - https://github.com/ethpandaops/assertoor assertoor_params: # Assertoor docker image to use diff --git a/main.star b/main.star index 99d853658..0405a41a0 100644 --- a/main.star +++ b/main.star @@ -28,6 +28,7 @@ full_beaconchain_explorer = import_module( blockscout = import_module("./src/blockscout/blockscout_launcher.star") prometheus = import_module("./src/prometheus/prometheus_launcher.star") grafana = import_module("./src/grafana/grafana_launcher.star") +tempo = import_module("./src/tempo/tempo_launcher.star") commit_boost_mev_boost = import_module( "./src/mev/commit-boost/mev_boost/mev_boost_launcher.star" ) @@ -128,6 +129,7 @@ def run(plan, args={}): grafana_dashboards_config_template = read_file( static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH ) + tempo_config_template = read_file(static_files.TEMPO_CONFIG_TEMPLATE_FILEPATH) prometheus_additional_metrics_jobs = [] raw_jwt_secret = read_file(static_files.JWT_PATH_FILEPATH) jwt_file = plan.upload_files( @@ -175,6 +177,16 @@ def run(plan, args={}): ) plan.print("Read the prometheus, grafana templates") + tempo_otlp_grpc_url = None + tempo_query_url = None + if "tempo" in args_with_right_defaults.additional_services: + tempo_otlp_grpc_url = "http://{}:{}".format( + tempo.SERVICE_NAME, tempo.OTLP_GRPC_PORT_NUMBER + ) + tempo_query_url = "http://{}:{}".format( + tempo.SERVICE_NAME, tempo.HTTP_PORT_NUMBER + ) + if args_with_right_defaults.mev_type == constants.MEV_RS_MEV_TYPE: plan.print("Generating mev-rs builder config file") mev_rs_builder_config_file = mev_rs_mev_builder.new_builder_config( @@ -227,6 +239,7 @@ def run(plan, args={}): keymanager_enabled, parallel_keystore_generation, extra_files_artifacts, + tempo_otlp_grpc_url, ) plan.print( @@ -712,8 +725,21 @@ def run(plan, args={}): args_with_right_defaults.grafana_params, args_with_right_defaults.port_publisher, index, + tempo_query_url, ) plan.print("Successfully launched grafana") + elif additional_service == "tempo": + plan.print("Launching tempo...") + tempo.launch_tempo( + plan, + tempo_config_template, + global_node_selectors, + global_tolerations, + args_with_right_defaults.tempo_params, + args_with_right_defaults.port_publisher, + index, + ) + plan.print("Successfully launched tempo") elif additional_service == "prometheus_grafana": # Allow prometheus to be launched last so is able to collect metrics from other services launch_prometheus_grafana = True @@ -802,6 +828,7 @@ def run(plan, args={}): args_with_right_defaults.grafana_params, args_with_right_defaults.port_publisher, prometheus_grafana_index, + tempo_query_url, ) plan.print("Successfully launched grafana") diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index cef9cc548..ffdaabfc6 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -27,6 +27,7 @@ def launch( global_node_selectors, global_tolerations, persistent, + tempo_otlp_grpc_url, num_participants, validator_data, prysm_password_relative_filepath, @@ -220,6 +221,7 @@ def launch( index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) blobber_config = get_blobber_config( @@ -268,6 +270,7 @@ def launch( index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) cl_participant_info[cl_service_name] = { diff --git a/src/cl/grandine/grandine_launcher.star b/src/cl/grandine/grandine_launcher.star index 377d6dc3c..1e91f4214 100644 --- a/src/cl/grandine/grandine_launcher.star +++ b/src/cl/grandine/grandine_launcher.star @@ -53,6 +53,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): config = get_beacon_config( plan, @@ -74,6 +75,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, config) @@ -111,6 +113,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index b685ae9e9..44f6731af 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -58,6 +58,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): # Launch Beacon node beacon_config = get_beacon_config( @@ -80,6 +81,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -117,6 +119,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS @@ -256,6 +259,10 @@ def get_beacon_config( else: # Public networks cmd.append("--network=" + network_params.network) + # Add tempo telemetry integration if tempo is enabled + if tempo_otlp_grpc_url != None: + cmd.append("--telemetry-collector-url={}".format(tempo_otlp_grpc_url)) + if len(participant.cl_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.cl_extra_params]) diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index d6bd4fe6a..a95928dca 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -45,6 +45,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): # Launch Beacon node beacon_config = get_beacon_config( @@ -67,6 +68,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -104,6 +106,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_urlk, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index 943e691b5..6520b10df 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -66,6 +66,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): beacon_config = get_beacon_config( plan, @@ -87,6 +88,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -124,6 +126,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index cfda1876c..72ff5b350 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -49,6 +49,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): beacon_config = get_beacon_config( plan, @@ -70,6 +71,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, beacon_config) @@ -107,6 +109,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index e82780d0e..a8e6d7859 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -53,6 +53,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url=None, ): config = get_beacon_config( plan, @@ -74,6 +75,7 @@ def launch( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ) beacon_service = plan.add_service(beacon_service_name, config) @@ -111,6 +113,7 @@ def get_beacon_config( participant_index, network_params, extra_files_artifacts, + tempo_otlp_grpc_url, ): log_level = input_parser.get_client_log_level_or_default( participant.cl_log_level, global_log_level, VERBOSITY_LEVELS diff --git a/src/grafana/grafana_launcher.star b/src/grafana/grafana_launcher.star index 0052ee240..7a6ac6368 100644 --- a/src/grafana/grafana_launcher.star +++ b/src/grafana/grafana_launcher.star @@ -49,6 +49,7 @@ def launch_grafana( grafana_params, port_publisher, index, + tempo_query_url=None, ): tolerations = input_parser.get_client_tolerations([], [], global_tolerations) @@ -61,6 +62,7 @@ def launch_grafana( datasource_config_template, dashboard_providers_config_template, prometheus_private_url, + tempo_query_url, additional_dashboards=grafana_params.additional_dashboards, ) @@ -94,9 +96,12 @@ def get_grafana_config_dir_artifact_uuid( datasource_config_template, dashboard_providers_config_template, prometheus_private_url, + tempo_query_url, additional_dashboards=[], ): - datasource_data = new_datasource_config_template_data(prometheus_private_url) + datasource_data = new_datasource_config_template_data( + prometheus_private_url, tempo_query_url + ) datasource_template_and_data = shared_utils.new_template_and_data( datasource_config_template, datasource_data ) @@ -167,8 +172,11 @@ def get_config( ) -def new_datasource_config_template_data(prometheus_url): - return {"PrometheusURL": prometheus_url} +def new_datasource_config_template_data(prometheus_url, tempo_query_url): + data = {"PrometheusURL": prometheus_url} + if tempo_query_url != None: + data["TempoURL"] = tempo_query_url + return data def new_dashboard_providers_config_template_data(dashboards_dirpath): diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 784db2fbb..8c4714bd9 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -77,6 +77,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = ( "assertoor_params", "prometheus_params", "grafana_params", + "tempo_params", "tx_fuzz_params", "custom_flood_params", "xatu_sentry_params", @@ -108,6 +109,7 @@ def input_parser(plan, input_args): result["grafana_params"] = get_default_grafana_params() result["assertoor_params"] = get_default_assertoor_params() result["prometheus_params"] = get_default_prometheus_params() + result["tempo_params"] = get_default_tempo_params() result["xatu_sentry_params"] = get_default_xatu_sentry_params() result["persistent"] = False result["parallel_keystore_generation"] = False @@ -170,6 +172,10 @@ def input_parser(plan, input_args): for sub_attr in input_args["grafana_params"]: sub_value = input_args["grafana_params"][sub_attr] result["grafana_params"][sub_attr] = sub_value + elif attr == "tempo_params": + for sub_attr in input_args["tempo_params"]: + sub_value = input_args["tempo_params"][sub_attr] + result["tempo_params"][sub_attr] = sub_value elif attr == "xatu_sentry_params": for sub_attr in input_args["xatu_sentry_params"]: sub_value = input_args["xatu_sentry_params"][sub_attr] @@ -561,6 +567,18 @@ def input_parser(plan, input_args): max_mem=result["grafana_params"]["max_mem"], image=result["grafana_params"]["image"], ), + tempo_params=struct( + retention_duration=result["tempo_params"]["retention_duration"], + ingestion_rate_limit=result["tempo_params"]["ingestion_rate_limit"], + ingestion_burst_limit=result["tempo_params"]["ingestion_burst_limit"], + max_search_duration=result["tempo_params"]["max_search_duration"], + max_bytes_per_trace=result["tempo_params"]["max_bytes_per_trace"], + min_cpu=result["tempo_params"]["min_cpu"], + max_cpu=result["tempo_params"]["max_cpu"], + min_mem=result["tempo_params"]["min_mem"], + max_mem=result["tempo_params"]["max_mem"], + image=result["tempo_params"]["image"], + ), apache_port=result["apache_port"], nginx_port=result["nginx_port"], assertoor_params=struct( @@ -1453,6 +1471,21 @@ def get_default_grafana_params(): } +def get_default_tempo_params(): + return { + "retention_duration": "12h", + "ingestion_rate_limit": 20971520, # 20MB + "ingestion_burst_limit": 52428800, # 50MB + "max_search_duration": "30s", + "max_bytes_per_trace": 52428800, # 50MB + "min_cpu": 10, + "max_cpu": 1000, + "min_mem": 128, + "max_mem": 2048, + "image": "grafana/tempo:latest", + } + + def get_default_xatu_sentry_params(): return { "xatu_sentry_image": "ethpandaops/xatu:latest", @@ -1744,6 +1777,7 @@ def docker_cache_image_override(plan, result): "tx_fuzz_params.image", "prometheus_params.image", "grafana_params.image", + "tempo_params.image", "spamoor_params.image", "ethereum_genesis_generator_params.image", ] diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index f13f52aa5..ebabbe5c6 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -288,6 +288,18 @@ SUBCATEGORY_PARAMS = { "max_mem", "image", ], + "tempo_params": [ + "retention_duration", + "ingestion_rate_limit", + "ingestion_burst_limit", + "max_search_duration", + "max_bytes_per_trace", + "min_cpu", + "max_cpu", + "min_mem", + "max_mem", + "image", + ], "assertoor_params": [ "image", "run_stability_check", @@ -350,6 +362,7 @@ ADDITIONAL_SERVICES_PARAMS = [ "prometheus_grafana", "prometheus", "grafana", + "tempo", "blobscan", "dugtrio", "blutgang", diff --git a/src/participant_network.star b/src/participant_network.star index 175138e94..82b56e732 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -47,6 +47,7 @@ def launch_participant_network( keymanager_enabled, parallel_keystore_generation, extra_files_artifacts, + tempo_otlp_grpc_url, ): network_id = network_params.network_id num_participants = len(args_with_right_defaults.participants) @@ -182,6 +183,7 @@ def launch_participant_network( global_node_selectors, global_tolerations, persistent, + tempo_otlp_grpc_url, num_participants, validator_data, prysm_password_relative_filepath, diff --git a/src/static_files/static_files.star b/src/static_files/static_files.star index 049a1fe3b..95d3e8944 100644 --- a/src/static_files/static_files.star +++ b/src/static_files/static_files.star @@ -83,6 +83,9 @@ GRAFANA_DASHBOARDS_CONFIG_DIRPATH = ( STATIC_FILES_DIRPATH + GRAFANA_CONFIG_DIRPATH + "/dashboards" ) +# Tempo config +TEMPO_CONFIG_TEMPLATE_FILEPATH = STATIC_FILES_DIRPATH + "/tempo-config/tempo.yaml.tmpl" + # Geth + CL genesis generation GENESIS_GENERATION_CONFIG_DIRPATH = STATIC_FILES_DIRPATH + "/genesis-generation-config" diff --git a/src/tempo/tempo_launcher.star b/src/tempo/tempo_launcher.star new file mode 100644 index 000000000..c1f776ca7 --- /dev/null +++ b/src/tempo/tempo_launcher.star @@ -0,0 +1,154 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") + +SERVICE_NAME = "tempo" + +# Tempo standard ports +HTTP_PORT_ID = "http" +HTTP_PORT_NUMBER = 3200 +GRPC_PORT_ID = "grpc" +GRPC_PORT_NUMBER = 9095 +OTLP_GRPC_PORT_ID = "otlp-grpc" +OTLP_GRPC_PORT_NUMBER = 4317 +OTLP_HTTP_PORT_ID = "otlp-http" +OTLP_HTTP_PORT_NUMBER = 4318 + +TEMPO_CONFIG_FILENAME = "tempo.yaml" +TEMPO_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/etc/tempo" + +USED_PORTS = { + HTTP_PORT_ID: shared_utils.new_port_spec( + HTTP_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + GRPC_PORT_ID: shared_utils.new_port_spec( + GRPC_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + ), + OTLP_GRPC_PORT_ID: shared_utils.new_port_spec( + OTLP_GRPC_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + ), + OTLP_HTTP_PORT_ID: shared_utils.new_port_spec( + OTLP_HTTP_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), +} + + +def launch_tempo( + plan, + config_template, + global_node_selectors, + global_tolerations, + tempo_params, + port_publisher, + index, +): + tolerations = input_parser.get_client_tolerations([], [], global_tolerations) + + config_files_artifact_name = get_tempo_config_dir_artifact_uuid( + plan, + config_template, + tempo_params, + ) + + public_ports = shared_utils.get_additional_service_standard_public_port( + port_publisher, + HTTP_PORT_ID, + index, + 1, + ) + + config = get_config( + config_files_artifact_name, + global_node_selectors, + tolerations, + tempo_params, + public_ports, + ) + + service = plan.add_service(SERVICE_NAME, config) + + # Return connection info for other services + return struct( + service_name=SERVICE_NAME, + ip_addr=service.ip_address, + http_port_num=HTTP_PORT_NUMBER, + grpc_port_num=GRPC_PORT_NUMBER, + otlp_grpc_port_num=OTLP_GRPC_PORT_NUMBER, + otlp_http_port_num=OTLP_HTTP_PORT_NUMBER, + http_url="http://{}:{}".format(service.ip_address, HTTP_PORT_NUMBER), + grpc_url="{}:{}".format(service.ip_address, GRPC_PORT_NUMBER), + otlp_grpc_url="{}:{}".format(SERVICE_NAME, OTLP_GRPC_PORT_NUMBER), + otlp_http_url="http://{}:{}".format(SERVICE_NAME, OTLP_HTTP_PORT_NUMBER), + ) + + +def get_tempo_config_dir_artifact_uuid( + plan, + config_template, + tempo_params, +): + template_data = new_config_template_data(tempo_params) + + template_and_data = shared_utils.new_template_and_data( + config_template, template_data + ) + + template_and_data_by_rel_dest_filepath = {} + template_and_data_by_rel_dest_filepath[TEMPO_CONFIG_FILENAME] = template_and_data + + config_files_artifact_name = plan.render_templates( + template_and_data_by_rel_dest_filepath, "tempo-config" + ) + + return config_files_artifact_name + + +def get_config( + config_files_artifact_name, + node_selectors, + tolerations, + tempo_params, + public_ports, +): + config_file_path = shared_utils.path_join( + TEMPO_CONFIG_MOUNT_DIRPATH_ON_SERVICE, + TEMPO_CONFIG_FILENAME, + ) + + return ServiceConfig( + image=tempo_params.image, + ports=USED_PORTS, + public_ports=public_ports, + files={ + TEMPO_CONFIG_MOUNT_DIRPATH_ON_SERVICE: config_files_artifact_name, + }, + cmd=[ + "-config.file={}".format(config_file_path), + ], + min_cpu=tempo_params.min_cpu, + max_cpu=tempo_params.max_cpu, + min_memory=tempo_params.min_mem, + max_memory=tempo_params.max_mem, + node_selectors=node_selectors, + tolerations=tolerations, + ) + + +def new_config_template_data(tempo_params): + return { + "HTTPPort": HTTP_PORT_NUMBER, + "GRPCPort": GRPC_PORT_NUMBER, + "OTLPGRPCPort": OTLP_GRPC_PORT_NUMBER, + "OTLPHTTPPort": OTLP_HTTP_PORT_NUMBER, + "RetentionDuration": tempo_params.retention_duration, + "IngestionRateLimit": tempo_params.ingestion_rate_limit, + "IngestionBurstLimit": tempo_params.ingestion_burst_limit, + "MaxSearchDuration": tempo_params.max_search_duration, + "MaxBytesPerTrace": tempo_params.max_bytes_per_trace, + } diff --git a/static_files/grafana-config/templates/datasource.yml.tmpl b/static_files/grafana-config/templates/datasource.yml.tmpl index a5e27c052..3ee8f42b1 100644 --- a/static_files/grafana-config/templates/datasource.yml.tmpl +++ b/static_files/grafana-config/templates/datasource.yml.tmpl @@ -1,7 +1,5 @@ apiVersion: 1 -# TODO(old) we can improve this, adding an array that we can use to iterate -# TODO(old) an get different datasource configurations datasources: - name: Prometheus type: prometheus @@ -11,3 +9,20 @@ datasources: basicAuth: false isDefault: true editable: true +{{- if .TempoURL }} + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: {{ .TempoURL }} + basicAuth: false + isDefault: false + editable: true + jsonData: + httpMethod: GET + tracesToLogs: + datasourceUid: 'Prometheus' + spanStartTimeShift: '1h' + spanEndTimeShift: '1h' + filterByTraceID: true +{{- end }} diff --git a/static_files/tempo-config/tempo.yaml.tmpl b/static_files/tempo-config/tempo.yaml.tmpl new file mode 100644 index 000000000..aed4775f3 --- /dev/null +++ b/static_files/tempo-config/tempo.yaml.tmpl @@ -0,0 +1,29 @@ +server: + http_listen_port: {{ .HTTPPort }} + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .OTLPGRPCPort }} + http: + endpoint: 0.0.0.0:{{ .OTLPHTTPPort }} + +ingester: + trace_idle_period: 10s + max_block_duration: 5m + +compactor: + compaction: + block_retention: {{ .RetentionDuration }} + +storage: + trace: + backend: local + wal: + path: /tmp/tempo/wal + local: + path: /tmp/tempo/traces + +stream_over_http_enabled: true