diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 536d76f2492d..e130c6cc5e18 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -36,6 +36,7 @@ env: CXX: sccache clang++ FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters SHELL_IMAGE: busybox + FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH: /var/tmp/forest_snapshot_calibnet_latest.forest.car.zst jobs: build-macos: name: Build MacOS diff --git a/docs/docs/users/reference/env_variables.md b/docs/docs/users/reference/env_variables.md index cf28d14de39c..3c399c85c3f5 100644 --- a/docs/docs/users/reference/env_variables.md +++ b/docs/docs/users/reference/env_variables.md @@ -56,6 +56,7 @@ process. | `FOREST_ETH_BLOCK_CACHE_SIZE` | positive integer | 500 | 1 | The size of Eth block cache | | `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` | 1 or true | false | 1 | Whether or not to backfill full tipsets from the p2p network | | `FOREST_STRICT_JSON` | 1 or true | false | 1 | Enable strict JSON validation to detect duplicate keys in RPC requests | +| `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH` | URL or file path | empty | `/var/tmp/forest_snapshot_calibnet.forest.car.zst` | Override snapshot path for `--auto-download-snapshot` | ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT` diff --git a/scripts/tests/calibnet_kademlia_check.sh b/scripts/tests/calibnet_kademlia_check.sh index db916312d6c1..59629301a88e 100755 --- a/scripts/tests/calibnet_kademlia_check.sh +++ b/scripts/tests/calibnet_kademlia_check.sh @@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH kademlia = true EOF +handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --save-token ./admin_token --rpc-address 127.0.0.1:12345 --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 & FOREST_NODE_PID=$! # Verify that more peers are connected via kademlia diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 8d4c14f13ffb..9b5ff73ddc83 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -11,6 +11,7 @@ function shutdown { trap shutdown EXIT +handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --exit-after-init $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" & FOREST_NODE_PID=$! diff --git a/scripts/tests/calibnet_stateless_mode_check.sh b/scripts/tests/calibnet_stateless_mode_check.sh index 0d4b602e89f1..f4fa5d44e735 100755 --- a/scripts/tests/calibnet_stateless_mode_check.sh +++ b/scripts/tests/calibnet_stateless_mode_check.sh @@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH kademlia = false EOF +handle_auto_download_snapshot_env # Disable discovery to not connect to more nodes $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --rpc false --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 & FOREST_NODE_PID=$! diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index f949b7700ab6..1a1eb5489f9a 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -16,6 +16,19 @@ LOG_DIRECTORY=$TMP_DIR/logs export TMP_DIR export LOG_DIRECTORY +# Use `aria2` to fetch the latest calibnet snapshot +# when `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH` is set +# This optimization is used on CI to speed up snapshot downloading +function handle_auto_download_snapshot_env { + if [[ -n "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH:-}" ]]; then + echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" + mkdir -p "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" + aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ \ + -d "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" \ + -o "$(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" + fi +} + function forest_import_non_calibnet_snapshot { echo "Importing a non calibnet snapshot" $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --import-snapshot ./test-snapshots/chain4.car @@ -23,6 +36,7 @@ function forest_import_non_calibnet_snapshot { function forest_download_and_import_snapshot { echo "Downloading and importing snapshot" + handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --auto-download-snapshot } diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index c753a239aed9..a836def9f13c 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -709,8 +709,19 @@ async fn maybe_set_snapshot_path( (false, _, _) => {} // noop - don't need a snapshot (true, true, _) => {} // noop - we need a snapshot, and we have one (true, false, true) => { - let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?; - config.client.snapshot_path = Some(url.to_string().into()); + const AUTO_SNAPSHOT_PATH_ENV_KEY: &str = "FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH"; + match std::env::var(AUTO_SNAPSHOT_PATH_ENV_KEY) { + Ok(path) if !path.is_empty() => { + tracing::info!( + "importing snapshot from {path} set by `{AUTO_SNAPSHOT_PATH_ENV_KEY}`" + ); + config.client.snapshot_path = Some(path.into()); + } + _ => { + let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?; + config.client.snapshot_path = Some(url.to_string().into()); + } + } } (true, false, false) => { // we need a snapshot, don't have one, and don't have permission to download one, so ask the user