From 49fb0fffb2289bd72be6a919f5f745fb2d05b953 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 26 Apr 2024 17:22:03 +0200 Subject: [PATCH 1/5] feat: add apache file server --- .github/tests/apache.yaml | 2 + .github/tests/mix-with-tools-mev.yaml | 1 + .github/tests/mix-with-tools-minimal.yaml | 1 + .github/tests/mix-with-tools.yaml | 1 + README.md | 10 +++ main.star | 11 +++ src/apache/apache_launcher.star | 92 +++++++++++++++++++++++ src/static_files/static_files.star | 2 + static_files/apache-config/index.html | 50 ++++++++++++ 9 files changed, 170 insertions(+) create mode 100644 .github/tests/apache.yaml create mode 100644 src/apache/apache_launcher.star create mode 100644 static_files/apache-config/index.html diff --git a/.github/tests/apache.yaml b/.github/tests/apache.yaml new file mode 100644 index 000000000..02155b401 --- /dev/null +++ b/.github/tests/apache.yaml @@ -0,0 +1,2 @@ +additional_services: + - apache diff --git a/.github/tests/mix-with-tools-mev.yaml b/.github/tests/mix-with-tools-mev.yaml index c8057a974..41d627594 100644 --- a/.github/tests/mix-with-tools-mev.yaml +++ b/.github/tests/mix-with-tools-mev.yaml @@ -24,6 +24,7 @@ additional_services: - blockscout - dugtrio - blutgang + - apache ethereum_metrics_exporter_enabled: true snooper_enabled: true mev_type: full diff --git a/.github/tests/mix-with-tools-minimal.yaml b/.github/tests/mix-with-tools-minimal.yaml index f8e5e0954..79314bce5 100644 --- a/.github/tests/mix-with-tools-minimal.yaml +++ b/.github/tests/mix-with-tools-minimal.yaml @@ -34,6 +34,7 @@ additional_services: - blockscout - dugtrio - blutgang + - apache ethereum_metrics_exporter_enabled: true snooper_enabled: true keymanager_enabled: true diff --git a/.github/tests/mix-with-tools.yaml b/.github/tests/mix-with-tools.yaml index 101a9cc08..82a740da8 100644 --- a/.github/tests/mix-with-tools.yaml +++ b/.github/tests/mix-with-tools.yaml @@ -26,6 +26,7 @@ additional_services: - blockscout - dugtrio - blutgang + - apache ethereum_metrics_exporter_enabled: true snooper_enabled: true keymanager_enabled: true diff --git a/README.md b/README.md index 48cbb25f9..97973e9a1 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,15 @@ For example, to retrieve the Execution Layer (EL) genesis data, run: kurtosis files download my-testnet el-genesis-data ~/Downloads ``` +# Basic file sharing + +Apache is included in the package to allow for basic file sharing. The Apache service is started when additional services are enabled. It will expose the network-configs directory, which might needed if you want to share the network config publicly. + +```yaml +additional_services: + - apache +``` + ## Configuration To configure the package behaviour, you can modify your `network_params.yaml` file. The full YAML schema that can be passed in is as follows with the defaults provided: @@ -544,6 +553,7 @@ additional_services: - blobscan - dugtrio - blutgang + - apache # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz tx_spammer_params: diff --git a/main.star b/main.star index 4d95c310e..fe93e6ebf 100644 --- a/main.star +++ b/main.star @@ -24,6 +24,7 @@ dora = import_module("./src/dora/dora_launcher.star") dugtrio = import_module("./src/dugtrio/dugtrio_launcher.star") blutgang = import_module("./src/blutgang/blutgang_launcher.star") blobscan = import_module("./src/blobscan/blobscan_launcher.star") +apache = import_module("./src/apache/apache_launcher.star") full_beaconchain_explorer = import_module( "./src/full_beaconchain/full_beaconchain_launcher.star" ) @@ -419,6 +420,16 @@ def run(plan, args={}): global_node_selectors, ) plan.print("Successfully launched blobscan") + elif additional_service == "apache": + plan.print("Launching apache") + apache_config = read_file(static_files.APACHE_CONFIG_FILEPATH) + apache.launch_apache( + plan, + apache_config, + el_cl_data_files_artifact_uuid, + global_node_selectors, + ) + plan.print("Successfully launched apache") elif additional_service == "full_beaconchain_explorer": plan.print("Launching full-beaconchain-explorer") full_beaconchain_explorer_config_template = read_file( diff --git a/src/apache/apache_launcher.star b/src/apache/apache_launcher.star new file mode 100644 index 000000000..b1620a3ea --- /dev/null +++ b/src/apache/apache_launcher.star @@ -0,0 +1,92 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +static_files = import_module("../static_files/static_files.star") +constants = import_module("../package_io/constants.star") +SERVICE_NAME = "apache" + +HTTP_PORT_ID = "http" +HTTP_PORT_NUMBER = 80 + +APACHE_CONFIG_FILENAME = "index.html" + +APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/usr/local/apache2/htdocs" + + +# The min/max CPU/memory that assertoor can use +MIN_CPU = 100 +MAX_CPU = 300 +MIN_MEMORY = 128 +MAX_MEMORY = 256 + +USED_PORTS = { + HTTP_PORT_ID: shared_utils.new_port_spec( + HTTP_PORT_NUMBER, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ) +} + + +def launch_apache( + plan, + apache_config, + el_cl_genesis_data, + global_node_selectors, +): + template_and_data_by_rel_dest_filepath = {} + template_and_data_by_rel_dest_filepath[ + APACHE_CONFIG_FILENAME + ] = shared_utils.new_template_and_data(apache_config, {}) + + config_files_artifact_name = plan.render_templates( + template_and_data_by_rel_dest_filepath, "apache-config" + ) + + config = get_config( + config_files_artifact_name, + el_cl_genesis_data, + global_node_selectors, + ) + + plan.add_service(SERVICE_NAME, config) + + +def get_config( + config_files_artifact_name, + el_cl_genesis_data, + node_selectors, +): + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data, + APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE: config_files_artifact_name, + } + + cmd = [ + "echo", + "AddType application/octet-stream .tar", + ">>", + "/usr/local/apache2/conf/httpd.conf", + "&&", + "tar", + "-czvf", + "/usr/local/apache2/htdocs/network-config.tar", + "-C", + "/network-configs/", + ".", + "&&", + "httpd-foreground" + ] + + cmd_str = " ".join(cmd) + + return ServiceConfig( + image="httpd:latest", + ports=USED_PORTS, + cmd = [cmd_str], + entrypoint=["sh", "-c"], + files=files, + min_cpu=MIN_CPU, + max_cpu=MAX_CPU, + min_memory=MIN_MEMORY, + max_memory=MAX_MEMORY, + node_selectors=node_selectors, + ) diff --git a/src/static_files/static_files.star b/src/static_files/static_files.star index dfbdd412c..022f5b300 100644 --- a/src/static_files/static_files.star +++ b/src/static_files/static_files.star @@ -16,6 +16,8 @@ VALIDATOR_RANGES_CONFIG_TEMPLATE_FILEPATH = ( STATIC_FILES_DIRPATH + "/validator-ranges/config.yaml.tmpl" ) +APACHE_CONFIG_FILEPATH = STATIC_FILES_DIRPATH + "/apache-config/index.html" + DORA_CONFIG_TEMPLATE_FILEPATH = STATIC_FILES_DIRPATH + "/dora-config/config.yaml.tmpl" DUGTRIO_CONFIG_TEMPLATE_FILEPATH = ( STATIC_FILES_DIRPATH + "/dugtrio-config/config.yaml.tmpl" diff --git a/static_files/apache-config/index.html b/static_files/apache-config/index.html new file mode 100644 index 000000000..8bf2833e0 --- /dev/null +++ b/static_files/apache-config/index.html @@ -0,0 +1,50 @@ + + + + + + Welcome to My Website + + + +
+

Welcome to Kurtosis File sharing site

+ Download network configs +
+ + From 0cdcc9aa813ad6e819a2013deb44bde6fae08d77 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 26 Apr 2024 17:25:16 +0200 Subject: [PATCH 2/5] fix lint --- src/apache/apache_launcher.star | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apache/apache_launcher.star b/src/apache/apache_launcher.star index b1620a3ea..380fa7eb6 100644 --- a/src/apache/apache_launcher.star +++ b/src/apache/apache_launcher.star @@ -73,7 +73,7 @@ def get_config( "/network-configs/", ".", "&&", - "httpd-foreground" + "httpd-foreground", ] cmd_str = " ".join(cmd) @@ -81,7 +81,7 @@ def get_config( return ServiceConfig( image="httpd:latest", ports=USED_PORTS, - cmd = [cmd_str], + cmd=[cmd_str], entrypoint=["sh", "-c"], files=files, min_cpu=MIN_CPU, From f3800e4b68420344de2721e2e896224671ea19db Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 26 Apr 2024 17:34:10 +0200 Subject: [PATCH 3/5] use file upload instead of template --- src/apache/apache_launcher.star | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/apache/apache_launcher.star b/src/apache/apache_launcher.star index 380fa7eb6..119781c3e 100644 --- a/src/apache/apache_launcher.star +++ b/src/apache/apache_launcher.star @@ -8,7 +8,7 @@ HTTP_PORT_NUMBER = 80 APACHE_CONFIG_FILENAME = "index.html" -APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/usr/local/apache2/htdocs" +APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/usr/local/apache2/htdocs/" # The min/max CPU/memory that assertoor can use @@ -32,13 +32,8 @@ def launch_apache( el_cl_genesis_data, global_node_selectors, ): - template_and_data_by_rel_dest_filepath = {} - template_and_data_by_rel_dest_filepath[ - APACHE_CONFIG_FILENAME - ] = shared_utils.new_template_and_data(apache_config, {}) - - config_files_artifact_name = plan.render_templates( - template_and_data_by_rel_dest_filepath, "apache-config" + config_files_artifact_name = plan.upload_files( + src=static_files.APACHE_CONFIG_FILEPATH, name="apache-config" ) config = get_config( From 1e14bdcd2228315b2985d5990cdf5f8e0d9ecc9f Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 26 Apr 2024 17:40:22 +0200 Subject: [PATCH 4/5] yeet apache_config --- main.star | 2 -- src/apache/apache_launcher.star | 1 - 2 files changed, 3 deletions(-) diff --git a/main.star b/main.star index fe93e6ebf..db802750f 100644 --- a/main.star +++ b/main.star @@ -422,10 +422,8 @@ def run(plan, args={}): plan.print("Successfully launched blobscan") elif additional_service == "apache": plan.print("Launching apache") - apache_config = read_file(static_files.APACHE_CONFIG_FILEPATH) apache.launch_apache( plan, - apache_config, el_cl_data_files_artifact_uuid, global_node_selectors, ) diff --git a/src/apache/apache_launcher.star b/src/apache/apache_launcher.star index 119781c3e..0178e7077 100644 --- a/src/apache/apache_launcher.star +++ b/src/apache/apache_launcher.star @@ -28,7 +28,6 @@ USED_PORTS = { def launch_apache( plan, - apache_config, el_cl_genesis_data, global_node_selectors, ): From 6521b07821927d492db2c3f8afda737dbedbbc92 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 26 Apr 2024 17:55:23 +0200 Subject: [PATCH 5/5] fix newline --- src/apache/apache_launcher.star | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/apache/apache_launcher.star b/src/apache/apache_launcher.star index 0178e7077..7e4ec01f6 100644 --- a/src/apache/apache_launcher.star +++ b/src/apache/apache_launcher.star @@ -2,7 +2,6 @@ shared_utils = import_module("../shared_utils/shared_utils.star") static_files = import_module("../static_files/static_files.star") constants = import_module("../package_io/constants.star") SERVICE_NAME = "apache" - HTTP_PORT_ID = "http" HTTP_PORT_NUMBER = 80 @@ -10,7 +9,6 @@ APACHE_CONFIG_FILENAME = "index.html" APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/usr/local/apache2/htdocs/" - # The min/max CPU/memory that assertoor can use MIN_CPU = 100 MAX_CPU = 300