diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f58093b59e..809665f417 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: rust: [ '1.60.0' ] - os: [ ubuntu-latest, macos-latest ] + os: [ ubuntu-latest, macos-latest, windows-latest ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/prepare-dfx-assets.yml b/.github/workflows/prepare-dfx-assets.yml index 807b577795..b6c7ef0d00 100644 --- a/.github/workflows/prepare-dfx-assets.yml +++ b/.github/workflows/prepare-dfx-assets.yml @@ -34,6 +34,7 @@ jobs: ~/.cargo/registry ~/.cargo/git target + !target/*/build/dfx-*/out/dfx-assets key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Install Rust run: | @@ -41,8 +42,9 @@ jobs: rustup component add clippy rustup default ${{ matrix.rust }} - - name: Prepare dfx assets - run: scripts/prepare-dfx-assets.sh "$(mktemp -d)" + - name: Run build script + run: | + cargo check aggregate: name: prepare-dfx-assets:required diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 4d7c53aa91..2a9581ad9a 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -36,5 +36,4 @@ jobs: run: $HOME/bin/shellcheck-v0.7.1/shellcheck e2e/**/*.*sh - name: Check release script run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/release.sh - - name: Check asset prep script - run: $HOME/bin/shellcheck-v0.7.1/shellcheck scripts/prepare-dfx-assets.sh + diff --git a/Cargo.lock b/Cargo.lock index 8f685a8685..945170aa8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,7 @@ dependencies = [ "time", "tiny-bip39", "tokio", + "toml", "url", "walkdir", "wasmparser 0.87.0", @@ -3522,9 +3523,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ "autocfg", "bytes", @@ -3532,7 +3533,6 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", diff --git a/Cargo.toml b/Cargo.toml index 18c2f830bc..4e27f41615 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,9 @@ lto = true [profile.dev.package.argon2] opt-level = 3 +[profile.dev.package.miniz_oxide] +opt-level = 3 + [profile.release.package.ic-frontend-canister] opt-level = 'z' diff --git a/nix/sources.json b/nix/sources.json index 32ce604365..eb3c08a849 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -12,18 +12,18 @@ "url": "https://github.com/RustSec/advisory-db/archive/fe7b79e8ebf9ad03660e573e0c2a1ed4e80dee5d.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "canister-sandbox-x86_64-darwin": { + "canister_sandbox-x86_64-darwin": { "builtin": false, - "description": "The canister-sandbox binary. It must be updated together with the replica binary.", + "description": "The canister_sandbox binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "07d5jf6y6a7kl3s4pgmjs7xsr93cvn6lp56gyr7zxziab81wp1i3", "type": "file", "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-darwin/canister_sandbox.gz" }, - "canister-sandbox-x86_64-linux": { + "canister_sandbox-x86_64-linux": { "builtin": false, - "description": "The canister-sandbox binary. It must be updated together with the replica binary.", + "description": "The canister_sandbox binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "1vyxwlvh59irx0j4rbjdyrj2j7hhhnc1c5552yc0pbmip9xalzkr", "type": "file", @@ -178,7 +178,7 @@ }, "replica-x86_64-darwin": { "builtin": false, - "description": "The replica binary. It must be updated together with the canister-sandbox binary.", + "description": "The replica binary. It must be updated together with the canister_sandbox binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "06mvsa9ibj3waj600digpgnipj74lc020lgvvamzzvpqpvwh6hag", "type": "file", @@ -187,14 +187,14 @@ }, "replica-x86_64-linux": { "builtin": false, - "description": "The replica binary. It must be updated together with the canister-sandbox binary.", + "description": "The replica binary. It must be updated together with the canister_sandbox binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", "sha256": "1c71c3zzljdjgbgx85qgijf75ccqv9vm083sidycnrh8m1lrzh27", "type": "file", "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-linux/replica.gz" }, - "sandbox-launcher-x86_64-darwin": { + "sandbox_launcher-x86_64-darwin": { "builtin": false, "description": "The sandbox_launcher binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", @@ -203,7 +203,7 @@ "url": "https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz", "url_template": "https://download.dfinity.systems/blessed/ic//sdk-release/x86_64-darwin/sandbox_launcher.gz" }, - "sandbox-launcher-x86_64-linux": { + "sandbox_launcher-x86_64-linux": { "builtin": false, "description": "The sandbox_launcher binary. It must be updated together with the replica binary.", "rev": "3e1be1316341811db5c9300935c4236bfab8fa2a", diff --git a/scripts/dfx-asset-sources.sh b/scripts/dfx-asset-sources.sh deleted file mode 100644 index 8ab7e51053..0000000000 --- a/scripts/dfx-asset-sources.sh +++ /dev/null @@ -1,54 +0,0 @@ -# generated by write-dfx-asset-sources.sh -IC_REF_X86_64_DARWIN_SHA256=07fb2cf2a570d6d90259ce165e6d5b9bbc408508743291be7890d8cb01c8a178 -IC_REF_X86_64_DARWIN_URL=https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-darwin.tar.gz -IC_REF_X86_64_LINUX_SHA256=d6f66f45b3fc904bea7810d3795b7761115b4635c3846991c4516ecaa42e2d36 -IC_REF_X86_64_LINUX_URL=https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-linux.tar.gz -ICX_PROXY_X86_64_DARWIN_SHA256=5783bba5021cf43149bc118789cea29f6462fd97dd78bdb776f8782ea7813d27 -ICX_PROXY_X86_64_DARWIN_URL=https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz -ICX_PROXY_X86_64_LINUX_SHA256=7a5612a1fb7512d22dcd37627a9d626fbc282b172665a832fe2cc2b243789fa1 -ICX_PROXY_X86_64_LINUX_URL=https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz -IC_ADMIN_X86_64_DARWIN_SHA256=303fe743a47180d1a196f7c4cf6acc9d17530768fd1417af30f615a2f92ccd3e -IC_ADMIN_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-admin.gz -IC_ADMIN_X86_64_LINUX_SHA256=c201463f2498dbf980fb3b1fb5ec75f46e7e8464d7530c0423d889ead09de44e -IC_ADMIN_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-admin.gz -IC_BTC_ADAPTER_X86_64_DARWIN_SHA256=b1e26a1b242b29ab61346d8a5856199d735d75ef6aa3ab5af94de250a4797164 -IC_BTC_ADAPTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-btc-adapter.gz -IC_BTC_ADAPTER_X86_64_LINUX_SHA256=26941f1fb7897faa72239871a27a18c1bccdd538fef13911b028f7568ef28c71 -IC_BTC_ADAPTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-btc-adapter.gz -IC_CANISTER_HTTP_ADAPTER_X86_64_DARWIN_SHA256=d9dc8a228b3507f0e1df73781d9738eecd216cbfda7a1f5cb063cbf2a2a7b54b -IC_CANISTER_HTTP_ADAPTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-canister-http-adapter.gz -IC_CANISTER_HTTP_ADAPTER_X86_64_LINUX_SHA256=610a565ec150be9cb29f69b99cecf56f25e131ca740bf41c5b309be140fa0da8 -IC_CANISTER_HTTP_ADAPTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-canister-http-adapter.gz -IC_NNS_INIT_X86_64_DARWIN_SHA256=4086229540cced5742acb8b9885a841e7c47a8e3d7b59a393034d79f7cb6d2a4 -IC_NNS_INIT_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-nns-init.gz -IC_NNS_INIT_X86_64_LINUX_SHA256=71cacac1053c07e4b607948662674c564e6f134b68f5febe49a8e421b7ab2768 -IC_NNS_INIT_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-nns-init.gz -IC_STARTER_X86_64_DARWIN_SHA256=b3b5dcc554b9419f5929cce069e84d8d5070661d8a1a60b278c39272f5916c4e -IC_STARTER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-starter.gz -IC_STARTER_X86_64_LINUX_SHA256=0bee932157078e868649365485755e9055391a6dcb05e369f617a95c05c461e6 -IC_STARTER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-starter.gz -MOTOKO_X86_64_DARWIN_SHA256=0fd98250ce7f2c6dbbaa9ccae141ad60d9542bd4e915e0fb4f82a235db666c88 -MOTOKO_X86_64_DARWIN_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-macos-0.7.1.tar.gz -MOTOKO_X86_64_LINUX_SHA256=132b13907425d3a7bccf10f48a420140244e663f0b48b200908c295a71a3a3c7 -MOTOKO_X86_64_LINUX_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-linux64-0.7.1.tar.gz -# The replica and canister_sandbox binaries must have the same revision. -REPLICA_X86_64_DARWIN_SHA256=4f4103f9bef8eeffabdafb512000a3e4c81bedbb2f36008c547cc81593d2bb1a -REPLICA_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/replica.gz -REPLICA_X86_64_LINUX_SHA256=47c09f69a80866cb7c8b7a205077da98b1729c8c0f17d4df7ab249faff60e1b0 -REPLICA_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz -# The replica and canister_sandbox binaries must have the same revision. -CANISTER_SANDBOX_X86_64_DARWIN_SHA256=2386cb035a2afefe4ff6cf944b8ddd6ca4acfbd1b2be4bf4a0f328e38d93a51d -CANISTER_SANDBOX_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz -CANISTER_SANDBOX_X86_64_LINUX_SHA256=797eaa7abab1ae0b9817a514169885101e2964f64dae4c24e839a60237e5ddef -CANISTER_SANDBOX_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/canister_sandbox.gz -SANDBOX_LAUNCHER_X86_64_DARWIN_SHA256=a5f1cb3dc57be908fe5bd880a42160f4b13f62b9c3acc5baef9e7a2518b53ad1 -SANDBOX_LAUNCHER_X86_64_DARWIN_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz -SANDBOX_LAUNCHER_X86_64_LINUX_SHA256=9fb47f54a2690b0a5de3c5908f09282323548181c36d20f55e2fd816f72994ba -SANDBOX_LAUNCHER_X86_64_LINUX_URL=https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/sandbox_launcher.gz -SNS_X86_64_DARWIN_SHA256=0402538652ba6296ac60e4fef255691226ae20cbd5d20297f585369565d1f035 -SNS_X86_64_DARWIN_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/sns.gz -SNS_X86_64_LINUX_SHA256=2f04439436171ba96912ac7967fcd3a275d68b71267f07f221c7381f54fb0ef2 -SNS_X86_64_LINUX_URL=https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/sns.gz -MOTOKO_BASE_URL=https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz -MOTOKO_BASE_SHA256=efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1 -REPLICA_REV=3e1be1316341811db5c9300935c4236bfab8fa2a diff --git a/scripts/prepare-dfx-assets.sh b/scripts/prepare-dfx-assets.sh deleted file mode 100755 index b2f748d41f..0000000000 --- a/scripts/prepare-dfx-assets.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -SDK_ROOT_DIR="$( cd -- "$(dirname -- "$( dirname -- "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )" - -# shellcheck disable=SC1090 -source "$SDK_ROOT_DIR/scripts/dfx-asset-sources.sh" - -DFX_ASSETS_FINAL_DIR=${1?'Must specify a destination directory.'} - -DFX_ASSETS_TEMP_DIR=$(mktemp -d) -BINARY_CACHE_TEMP_DIR=$(mktemp -d) -DOWNLOAD_TEMP_DIR=$(mktemp -d) - -function cleanup { - rm -rf "$DFX_ASSETS_TEMP_DIR" "$BINARY_CACHE_TEMP_DIR" "$DOWNLOAD_TEMP_DIR" -} -trap cleanup EXIT - -# We use x86_64 even on Apple M1 (arm64), through rosetta -MACHINE=x86_64 -case "$OSTYPE" in - darwin*) PLATFORM="darwin" ;; - linux*) PLATFORM="linux" ;; - *) echo "Unsupported OS type: $OSTYPE" ; exit 1;; -esac - -add_canisters() { - tar -czf "$DFX_ASSETS_TEMP_DIR"/assetstorage_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./assetstorage.did ./assetstorage.wasm.gz - tar -czf "$DFX_ASSETS_TEMP_DIR"/wallet_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./wallet.did ./wallet.wasm - tar -czf "$DFX_ASSETS_TEMP_DIR"/ui_canister.tgz -C "$SDK_ROOT_DIR"/src/distributed ./ui.did ./ui.wasm -} - -download_url_and_check_sha() { - URL="$1" - EXPECTED_SHA256="$2" - LOCAL_PATH="$3" - - echo "Downloading $URL with expected sha256=$EXPECTED_SHA256 to $LOCAL_PATH" - - curl --fail --location --output "$LOCAL_PATH" "$URL" - - ACTUAL_SHA256=$(shasum -a 256 "$LOCAL_PATH" | cut -f 1 -d ' ') - - if [ "$EXPECTED_SHA256" != "$ACTUAL_SHA256" ]; then - echo "SHA256 mismatch for $URL: expected $EXPECTED_SHA256, got $ACTUAL_SHA256" - exit 1 - fi -} - -get_variable() { - NAME="$1" - PART="$2" - - VAR_NAME=$(echo "${NAME}_${MACHINE}_${PLATFORM}_${PART}" | tr '[:lower:]-' '[:upper:]_') - VAR_VALUE=${!VAR_NAME} - - echo "$VAR_VALUE" -} - -download_binary() { - NAME="$1" - SHA256=$(get_variable "$NAME" "SHA256") - URL=$(get_variable "$NAME" "URL") - - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/$NAME.gz" - BINARY_CACHE_PATH="$BINARY_CACHE_TEMP_DIR/$NAME" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - gunzip -c "$DOWNLOAD_PATH" >"$BINARY_CACHE_PATH" - chmod 0500 "$BINARY_CACHE_PATH" -} - -download_tarball() { - NAME="$1" - - SHA256=$(get_variable "$NAME" "SHA256") - URL=$(get_variable "$NAME" "URL") - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/$NAME.tar.gz" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - # -k: some archives contain r-x ".", and on linux the default behavior is to overwrite the - # metadata. We only want to extract new files anyway. - tar -xkvf "$DOWNLOAD_PATH" -C "$BINARY_CACHE_TEMP_DIR" -} - -download_ic_ref() { - download_tarball "ic-ref" - chmod 0500 "$BINARY_CACHE_TEMP_DIR/ic-ref" -} - -download_icx_proxy() { - download_tarball "icx-proxy" - - chmod 0500 "$BINARY_CACHE_TEMP_DIR/icx-proxy" -} - -download_motoko_binaries() { - download_tarball "motoko" - - for a in mo-doc mo-ide moc; - do - chmod 0500 "$BINARY_CACHE_TEMP_DIR/$a" - done -} - -download_motoko_base() { - URL="$MOTOKO_BASE_URL" - SHA256="$MOTOKO_BASE_SHA256" - DOWNLOAD_PATH="$DOWNLOAD_TEMP_DIR/motoko-base-tarball.tar.gz" - - download_url_and_check_sha "$URL" "$SHA256" "$DOWNLOAD_PATH" - - mkdir "$DOWNLOAD_TEMP_DIR/motoko-base" - tar -xkvf "$DOWNLOAD_PATH" -C "$DOWNLOAD_TEMP_DIR/motoko-base" - - cp -R "$DOWNLOAD_TEMP_DIR/motoko-base/src/" "$BINARY_CACHE_TEMP_DIR/base" - chmod 0755 "$BINARY_CACHE_TEMP_DIR/base" - find "$BINARY_CACHE_TEMP_DIR/base" -type f -exec touch {} \; -exec chmod 0644 {} \; - - chmod -R 0744 "$DOWNLOAD_TEMP_DIR/motoko-base" - rm -rf "$DOWNLOAD_TEMP_DIR/motoko-base" -} - -add_binary_cache() { - download_binary "ic-admin" - download_binary "ic-btc-adapter" - download_binary "ic-canister-http-adapter" - download_binary "ic-nns-init" - download_binary "replica" - download_binary "canister_sandbox" - download_binary "sandbox_launcher" - download_binary "ic-starter" - download_binary "sns" - download_ic_ref - download_icx_proxy - download_motoko_binaries - download_motoko_base - - tar -czf "$DFX_ASSETS_TEMP_DIR"/binary_cache.tgz -C "$BINARY_CACHE_TEMP_DIR" . -} - -echo "Building $DFX_ASSETS_FINAL_DIR" - -add_canisters -add_binary_cache - -if [ -d "$DFX_ASSETS_FINAL_DIR" ] -then - ( - cd "$DFX_ASSETS_FINAL_DIR" - rm -f binary_cache.tgz assetstorage_canister.tgz wallet_canister.tgz ui_canister.tgz - ) - rmdir "$DFX_ASSETS_FINAL_DIR" -fi -mv "$DFX_ASSETS_TEMP_DIR" "$DFX_ASSETS_FINAL_DIR" - -echo "Built $DFX_ASSETS_FINAL_DIR" diff --git a/scripts/update-replica.sh b/scripts/update-replica.sh index 5367094ddd..d13d2f87a5 100755 --- a/scripts/update-replica.sh +++ b/scripts/update-replica.sh @@ -22,10 +22,10 @@ niv update ic-starter-x86_64-darwin -a rev=$SHA niv update ic-starter-x86_64-linux -a rev=$SHA niv update replica-x86_64-darwin -a rev=$SHA niv update replica-x86_64-linux -a rev=$SHA -niv update canister-sandbox-x86_64-darwin -a rev=$SHA -niv update canister-sandbox-x86_64-linux -a rev=$SHA -niv update sandbox-launcher-x86_64-darwin -a rev=$SHA -niv update sandbox-launcher-x86_64-linux -a rev=$SHA +niv update canister_sandbox-x86_64-darwin -a rev=$SHA +niv update canister_sandbox-x86_64-linux -a rev=$SHA +niv update sandbox_launcher-x86_64-darwin -a rev=$SHA +niv update sandbox_launcher-x86_64-linux -a rev=$SHA niv update sns-x86_64-darwin -a rev=$SHA niv update sns-x86_64-linux -a rev=$SHA diff --git a/scripts/write-dfx-asset-sources.sh b/scripts/write-dfx-asset-sources.sh index abd897d144..0bfd8434ae 100755 --- a/scripts/write-dfx-asset-sources.sh +++ b/scripts/write-dfx-asset-sources.sh @@ -16,7 +16,7 @@ which curl >/dev/null || ( echo "Please install curl in order to run this script SDK_ROOT_DIR="$( cd -- "$(dirname -- "$( dirname -- "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )" -DFX_ASSET_SOURCES="$SDK_ROOT_DIR/scripts/dfx-asset-sources.sh" +DFX_ASSET_SOURCES="$SDK_ROOT_DIR/src/dfx/assets/dfx-asset-sources.toml" NIX_SOURCES_JSON="$SDK_ROOT_DIR/nix/sources.json" read_sha256_from_nix_sources() { @@ -39,17 +39,16 @@ read_rev_from_nix_sources() { jq -r .'"'"$KEY"'".rev' "$NIX_SOURCES_JSON" } -normalize_varname() { - echo "$1" | tr '[:lower:]-' '[:upper:]_' -} - -write_sha256() { - KEY="$1" - SHA256=$(read_sha256_from_nix_sources "$KEY") - - NAME=$(normalize_varname "${KEY}_SHA256") - - echo "$NAME=$SHA256" >>"$DFX_ASSET_SOURCES" +write_entry() { + NAME="$1" + PLATFORM="$2" + KEY="${3:-"${NAME}-${PLATFORM}"}" + URL=$(read_url_from_nix_sources "$KEY") + SHA256="${4:-"$(read_sha256_from_nix_sources "$KEY")"}" + cat >>"$DFX_ASSET_SOURCES" <<<" +[${PLATFORM}.${NAME}] +url = '${URL}' +sha256 = '${SHA256}'" } calculate_sha256() { @@ -73,50 +72,27 @@ calculate_sha256() { echo "SHA256 mismatch for $URL: expected $EXPECTED_BASE32_SHA256, got $ACTUAL_BASE32_SHA256" exit 1 fi - - NAME=$(normalize_varname "${KEY}_SHA256") - - echo "$NAME=$SHA256" >>"$DFX_ASSET_SOURCES" - -} - -write_url() { - KEY="$1" - URL=$(read_url_from_nix_sources "$KEY") - - NAME=$(normalize_varname "${KEY}_URL") - - echo "$NAME=$URL" >>"$DFX_ASSET_SOURCES" + echo "$SHA256" } write_replica_rev() { REV=$(read_rev_from_nix_sources "replica-x86_64-darwin") - echo "REPLICA_REV=$REV" >>"$DFX_ASSET_SOURCES" -} - -write_var() { - VALUE=$(jq -r .'"'"$1"'"."'"$2"'"' "$NIX_SOURCES_JSON") - NAME=$(normalize_varname "${1}_${2}") - echo "$NAME=$VALUE" >>"$DFX_ASSET_SOURCES" + echo "replica-rev = '$REV'" >>"$DFX_ASSET_SOURCES" } echo "# generated by write-dfx-asset-sources.sh" >"$DFX_ASSET_SOURCES" -for name in "ic-ref" "icx-proxy" "ic-admin" "ic-btc-adapter" "ic-canister-http-adapter" "ic-nns-init" "ic-starter" "motoko" "replica" "canister-sandbox" "sandbox-launcher" "sns"; +write_replica_rev +motoko_base_sha="$(calculate_sha256 "motoko-base")" +for platform in "darwin" "linux"; do - if [[ "$name" == "replica" || "$name" == "canister-sandbox" ]]; then - echo "# The replica and canister_sandbox binaries must have the same revision." >>"$DFX_ASSET_SOURCES" - fi - for platform in "darwin" "linux"; + for name in "ic-ref" "icx-proxy" "ic-admin" "ic-btc-adapter" "ic-canister-http-adapter" "ic-nns-init" "ic-starter" "motoko" "replica" "canister_sandbox" "sandbox_launcher" "sns"; do - write_sha256 "${name}-x86_64-${platform}" - write_url "${name}-x86_64-${platform}" + if [[ "$name" == "replica" || "$name" == "canister_sandbox" ]]; then + echo "# The replica and canister_sandbox binaries must have the same revision." >>"$DFX_ASSET_SOURCES" + fi + write_entry "$name" "x86_64-${platform}" done + write_entry "motoko-base" "x86_64-${platform}" "motoko-base" "$motoko_base_sha" done - -write_url "motoko-base" -calculate_sha256 "motoko-base" - -write_replica_rev - diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index abdd4d2f4f..5422371c53 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -10,10 +10,15 @@ name = "dfx" path = "src/main.rs" [build-dependencies] +bytes = "1" flate2 = "1.0.11" -hex = "0.4.2" +hex = "0.4.3" +reqwest = "0.11.9" +serde = { version = "1.0", features = ["derive"] } sha2 = "0.10.6" tar = "0.4.26" +tokio = { version = "1.17.0", features = ["full"] } +toml = "0.5.9" walkdir = "2.3.2" [dependencies] diff --git a/src/dfx/assets/README.adoc b/src/dfx/assets/README.adoc index 6336d14a96..ddc45435a9 100644 --- a/src/dfx/assets/README.adoc +++ b/src/dfx/assets/README.adoc @@ -1,11 +1,11 @@ = Assets -The `files/` directory contains all the files of a new project. This is tar gzipped at build time +The `new_project_*/` directories contain all the files of a new project. These are tar gzipped at build time and injected into the binary. The following strings are replaced: -- `{project_name}` => the project name. +- `{project_name}` / `+__project_name__+` => the project name. - `{dfx_version}` => the DFX version used to create the project. Also, files that start with `+++__dot__+++` will be replaced with `.`. diff --git a/src/dfx/assets/build.rs b/src/dfx/assets/build.rs index 0702dbb611..d06c31c4e1 100644 --- a/src/dfx/assets/build.rs +++ b/src/dfx/assets/build.rs @@ -1,17 +1,21 @@ use flate2::write::GzEncoder; use flate2::Compression; +use serde::Deserialize; use sha2::{Digest, Sha256}; +use std::collections::HashMap; use std::fs::{read_to_string, File}; -use std::io::{BufRead, Read, Write}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs, io}; +use std::{env, fs}; use walkdir::WalkDir; +mod prepare_assets; + const INPUTS: &[&str] = &[ "nix/sources.json", - "scripts/dfx-asset-sources.sh", - "scripts/prepare-dfx-assets.sh", + "src/dfx/assets/prepare_assets.rs", + "src/dfx/assets/build.rs", "src/distributed/assetstorage.did", "src/distributed/assetstorage.wasm.gz", "src/distributed/ui.did", @@ -42,13 +46,34 @@ fn get_project_root_path() -> PathBuf { .expect("Unable to determine project root") } -fn find_assets() -> PathBuf { +#[derive(Deserialize, Clone)] +struct Source { + url: String, + sha256: String, +} + +impl Source { + fn sha256(&self) -> Vec { + hex::decode(&self.sha256).expect("Invalid SHA-256") + } +} + +#[derive(Deserialize)] +struct Sources { + #[serde(rename = "x86_64-linux")] + x86_64_linux: HashMap, + #[serde(rename = "x86_64-darwin")] + x86_64_darwin: HashMap, + #[serde(rename = "replica-rev")] + replica_rev: String, +} + +fn find_assets(sources: Sources) -> PathBuf { println!("cargo:rerun-if-env-changed=DFX_ASSETS"); if let Ok(a) = env::var("DFX_ASSETS") { PathBuf::from(a) } else { let project_root_path = get_project_root_path(); - let prepare_script_path = project_root_path.join("scripts/prepare-dfx-assets.sh"); for input in INPUTS { println!( "cargo:rerun-if-changed={}", @@ -69,18 +94,15 @@ fn find_assets() -> PathBuf { } } - let result = Command::new(&prepare_script_path) - .arg(&dfx_assets_path) - .output() - .expect("unable to run prepare script"); - if !result.status.success() { - println!( - "cargo:error=unable to run {}:", - prepare_script_path.to_string_lossy() - ); - println!("cargo:error={}", String::from_utf8_lossy(&result.stderr)); - std::process::exit(1) - } + let source_set = match ( + &*env::var("CARGO_CFG_TARGET_ARCH").unwrap(), + &*env::var("CARGO_CFG_TARGET_OS").unwrap(), + ) { + ("x86_64" | "aarch64", "macos") => sources.x86_64_darwin, // rosetta + ("x86_64", "linux" | "windows") => sources.x86_64_linux, + (arch, os) => panic!("Unsupported OS type {arch}-{os}"), + }; + prepare_assets::prepare(&dfx_assets_path, source_set); fs::write(last_hash_of_inputs_path, hash_of_inputs) .expect("unable to write last hash of inputs"); @@ -128,11 +150,8 @@ fn write_archive_accessor(fn_name: &str, f: &mut File) { f.write_all( format!( " - pub fn {fn_name}() -> Result>>>> {{ - let mut v = Vec::new(); - v.extend_from_slice(include_bytes!(\"{fn_name}.tgz\")); - - let tar = GzDecoder::new(std::io::Cursor::new(v)); + pub fn {fn_name}() -> Result>> {{ + let tar = GzDecoder::new(&include_bytes!(\"{fn_name}.tgz\")[..]); let archive = Archive::new(tar); Ok(archive) }} @@ -197,7 +216,7 @@ fn get_git_hash() -> Result { )) } -fn add_assets() { +fn add_assets(sources: Sources) { let out_dir = env::var("OUT_DIR").unwrap(); let loader_path = Path::new(&out_dir).join("load_assets.rs"); let mut f = File::create(&loader_path).unwrap(); @@ -205,7 +224,7 @@ fn add_assets() { f.write_all( b" use flate2::read::GzDecoder; - use std::io::{Cursor, Result}; + use std::io::Result; use std::vec::Vec; use tar::Archive; @@ -213,7 +232,7 @@ fn add_assets() { ) .unwrap(); - let dfx_assets = find_assets(); + let dfx_assets = find_assets(sources); add_asset_archive("binary_cache", &mut f, &dfx_assets); add_asset_archive("assetstorage_canister", &mut f, &dfx_assets); add_asset_archive("wallet_canister", &mut f, &dfx_assets); @@ -253,25 +272,16 @@ fn define_dfx_version() { } } -fn define_replica_rev() { - let pathname = get_project_root_path().join("scripts/dfx-asset-sources.sh"); - let file = File::open(pathname).expect("Unable to read scripts/dfx-asset-sources.sh"); - let reader = io::BufReader::new(file); - - let prefix = "REPLICA_REV="; - - let replica_rev_line = reader - .lines() - .map(|line| line.expect("Could not parse line")) - .find(|line| line.starts_with(prefix)) - .expect("No REPLICA_REV in scripts/dfx-asset-sources.sh"); - let replica_rev = &replica_rev_line[prefix.len()..]; - +fn define_replica_rev(replica_rev: &str) { println!("cargo:rustc-env=DFX_ASSET_REPLICA_REV={}", replica_rev); } fn main() { - add_assets(); + let sources: Sources = toml::from_slice( + &fs::read("assets/dfx-asset-sources.toml").expect("unable to read dfx-asset-sources.toml"), + ) + .expect("unable to parse dfx-asset-sources.toml"); + define_replica_rev(&sources.replica_rev); + add_assets(sources); define_dfx_version(); - define_replica_rev(); } diff --git a/src/dfx/assets/dfx-asset-sources.toml b/src/dfx/assets/dfx-asset-sources.toml new file mode 100644 index 0000000000..7294342489 --- /dev/null +++ b/src/dfx/assets/dfx-asset-sources.toml @@ -0,0 +1,110 @@ +# generated by write-dfx-asset-sources.sh +replica-rev = '3e1be1316341811db5c9300935c4236bfab8fa2a' + +[x86_64-darwin.ic-ref] +url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-darwin.tar.gz' +sha256 = '07fb2cf2a570d6d90259ce165e6d5b9bbc408508743291be7890d8cb01c8a178' + +[x86_64-darwin.icx-proxy] +url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz' +sha256 = '5783bba5021cf43149bc118789cea29f6462fd97dd78bdb776f8782ea7813d27' + +[x86_64-darwin.ic-admin] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-admin.gz' +sha256 = '303fe743a47180d1a196f7c4cf6acc9d17530768fd1417af30f615a2f92ccd3e' + +[x86_64-darwin.ic-btc-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-btc-adapter.gz' +sha256 = 'b1e26a1b242b29ab61346d8a5856199d735d75ef6aa3ab5af94de250a4797164' + +[x86_64-darwin.ic-canister-http-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-canister-http-adapter.gz' +sha256 = 'd9dc8a228b3507f0e1df73781d9738eecd216cbfda7a1f5cb063cbf2a2a7b54b' + +[x86_64-darwin.ic-nns-init] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/ic-nns-init.gz' +sha256 = '4086229540cced5742acb8b9885a841e7c47a8e3d7b59a393034d79f7cb6d2a4' + +[x86_64-darwin.ic-starter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/ic-starter.gz' +sha256 = 'b3b5dcc554b9419f5929cce069e84d8d5070661d8a1a60b278c39272f5916c4e' + +[x86_64-darwin.motoko] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-macos-0.7.1.tar.gz' +sha256 = '0fd98250ce7f2c6dbbaa9ccae141ad60d9542bd4e915e0fb4f82a235db666c88' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-darwin.replica] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/replica.gz' +sha256 = '4f4103f9bef8eeffabdafb512000a3e4c81bedbb2f36008c547cc81593d2bb1a' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-darwin.canister_sandbox] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/canister_sandbox.gz' +sha256 = '2386cb035a2afefe4ff6cf944b8ddd6ca4acfbd1b2be4bf4a0f328e38d93a51d' + +[x86_64-darwin.sandbox_launcher] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-darwin/sandbox_launcher.gz' +sha256 = 'a5f1cb3dc57be908fe5bd880a42160f4b13f62b9c3acc5baef9e7a2518b53ad1' + +[x86_64-darwin.sns] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/nix-release/x86_64-darwin/sns.gz' +sha256 = '0402538652ba6296ac60e4fef255691226ae20cbd5d20297f585369565d1f035' + +[x86_64-darwin.motoko-base] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz' +sha256 = 'efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1' + +[x86_64-linux.ic-ref] +url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-1fba03ee-x86_64-linux.tar.gz' +sha256 = 'd6f66f45b3fc904bea7810d3795b7761115b4635c3846991c4516ecaa42e2d36' + +[x86_64-linux.icx-proxy] +url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz' +sha256 = '7a5612a1fb7512d22dcd37627a9d626fbc282b172665a832fe2cc2b243789fa1' + +[x86_64-linux.ic-admin] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-admin.gz' +sha256 = 'c201463f2498dbf980fb3b1fb5ec75f46e7e8464d7530c0423d889ead09de44e' + +[x86_64-linux.ic-btc-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-btc-adapter.gz' +sha256 = '26941f1fb7897faa72239871a27a18c1bccdd538fef13911b028f7568ef28c71' + +[x86_64-linux.ic-canister-http-adapter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-canister-http-adapter.gz' +sha256 = '610a565ec150be9cb29f69b99cecf56f25e131ca740bf41c5b309be140fa0da8' + +[x86_64-linux.ic-nns-init] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/ic-nns-init.gz' +sha256 = '71cacac1053c07e4b607948662674c564e6f134b68f5febe49a8e421b7ab2768' + +[x86_64-linux.ic-starter] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/ic-starter.gz' +sha256 = '0bee932157078e868649365485755e9055391a6dcb05e369f617a95c05c461e6' + +[x86_64-linux.motoko] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-linux64-0.7.1.tar.gz' +sha256 = '132b13907425d3a7bccf10f48a420140244e663f0b48b200908c295a71a3a3c7' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-linux.replica] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/replica.gz' +sha256 = '47c09f69a80866cb7c8b7a205077da98b1729c8c0f17d4df7ab249faff60e1b0' +# The replica and canister_sandbox binaries must have the same revision. + +[x86_64-linux.canister_sandbox] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/canister_sandbox.gz' +sha256 = '797eaa7abab1ae0b9817a514169885101e2964f64dae4c24e839a60237e5ddef' + +[x86_64-linux.sandbox_launcher] +url = 'https://download.dfinity.systems/blessed/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/sdk-release/x86_64-linux/sandbox_launcher.gz' +sha256 = '9fb47f54a2690b0a5de3c5908f09282323548181c36d20f55e2fd816f72994ba' + +[x86_64-linux.sns] +url = 'https://download.dfinity.systems/ic/3e1be1316341811db5c9300935c4236bfab8fa2a/release/sns.gz' +sha256 = '2f04439436171ba96912ac7967fcd3a275d68b71267f07f221c7381f54fb0ef2' + +[x86_64-linux.motoko-base] +url = 'https://github.com/dfinity/motoko/releases/download/0.7.1/motoko-base-library.tar.gz' +sha256 = 'efa535dfbf49119f98c3b7fd1c464a7f9f602e7e1d00ce15a7ee2872316c0af1' diff --git a/src/dfx/assets/prepare_assets.rs b/src/dfx/assets/prepare_assets.rs new file mode 100644 index 0000000000..bd33f844f7 --- /dev/null +++ b/src/dfx/assets/prepare_assets.rs @@ -0,0 +1,221 @@ +use std::{ + collections::HashMap, + fs::File, + io::{self, BufWriter}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use flate2::{bufread::GzDecoder, write::GzEncoder, Compression}; +use reqwest::Client; +use sha2::{Digest, Sha256}; +use tar::{Archive, Builder, EntryType, Header}; +use tokio::task::{spawn, spawn_blocking, JoinSet}; + +use crate::Source; + +#[tokio::main] +pub(crate) async fn prepare(out_dir: &Path, source_set: HashMap) { + std::fs::create_dir_all(out_dir).expect("error creating output directory"); + let out_dir_ = out_dir.to_owned(); + let copy_join = spawn_blocking(|| copy_canisters(out_dir_)); + let out_dir = out_dir.to_owned(); + make_binary_cache(out_dir, source_set).await; + copy_join.await.unwrap(); +} + +fn copy_canisters(out_dir: PathBuf) { + let distributed = Path::new("../distributed"); + for can in ["assetstorage", "wallet", "ui"] { + let mut tar = Builder::new(GzEncoder::new( + BufWriter::new(File::create(out_dir.join(format!("{can}_canister.tgz"))).unwrap()), + Compression::new(6), + )); + for ext in [ + ".did", + if can == "assetstorage" { + ".wasm.gz" + } else { + ".wasm" + }, + ] { + let filename = format!("{can}{ext}"); + let input_file = File::open(distributed.join(&filename)).unwrap(); + let metadata = input_file.metadata().unwrap(); + let mut header = Header::new_gnu(); + header.set_mode(0o644); + header.set_size(metadata.len()); + tar.append_data(&mut header, &filename, input_file).unwrap(); + } + tar.finish().unwrap(); + } +} + +async fn make_binary_cache(out_dir: PathBuf, sources: HashMap) { + let sources = Arc::new(sources); + let client = Client::builder() + .timeout(Duration::from_secs(300)) + .build() + .unwrap(); + let mo_base = spawn(download_mo_base(client.clone(), sources.clone())); + let bins = spawn(download_binaries(client.clone(), sources.clone())); + let bin_tars = spawn(download_bin_tarballs(client.clone(), sources.clone())); + let (mo_base, bins, bin_tars) = tokio::try_join!(mo_base, bins, bin_tars).unwrap(); + spawn_blocking(|| write_binary_cache(out_dir, mo_base, bins, bin_tars)) + .await + .unwrap(); +} + +fn write_binary_cache( + out_dir: PathBuf, + mo_base: HashMap, + bins: HashMap, + mut bin_tars: HashMap, +) { + let mut tar = Builder::new(GzEncoder::new( + BufWriter::new(File::create(out_dir.join("binary_cache.tgz")).unwrap()), + Compression::new(6), + )); + for (path, bin) in bins.into_iter().chain( + ["icx-proxy", "ic-ref", "moc", "mo-doc", "mo-ide"] + .map(|bin| (bin.into(), bin_tars.remove(Path::new(bin)).unwrap())), + ) { + let mut header = Header::new_gnu(); + header.set_size(bin.len() as u64); + header.set_mode(0o500); + tar.append_data(&mut header, path, bin.reader()).unwrap(); + } + + for (path, file) in bin_tars { + let mut header = Header::new_gnu(); + header.set_size(file.len() as u64); + header.set_mode(0o644); + tar.append_data(&mut header, path, file.reader()).unwrap(); + } + let mut base_hdr = Header::new_gnu(); + base_hdr.set_entry_type(EntryType::dir()); + base_hdr.set_mode(0o755); + base_hdr.set_size(0); + tar.append_data(&mut base_hdr, "base", io::empty()).unwrap(); + for (path, file) in mo_base { + let mut header = Header::new_gnu(); + header.set_mode(0o644); + header.set_size(file.len() as u64); + tar.append_data(&mut header, Path::new("base").join(path), file.reader()) + .unwrap(); + } + tar.finish().unwrap(); +} + +async fn download_and_check_sha(client: Client, source: Source) -> Bytes { + let response = client.get(&source.url).send().await.unwrap(); + response.error_for_status_ref().unwrap(); + let content = response.bytes().await.unwrap(); + let sha = Sha256::digest(&content); + assert_eq!( + sha[..], + source.sha256()[..], + "sha256 hash for {} did not match", + source.url + ); + content +} + +async fn download_binaries( + client: Client, + sources: Arc>, +) -> HashMap { + let mut joinset = JoinSet::new(); + for bin in [ + "ic-admin", + "ic-btc-adapter", + "ic-canister-http-adapter", + "ic-nns-init", + "replica", + "canister_sandbox", + "sandbox_launcher", + "ic-starter", + "sns", + ] { + let source = sources + .get(bin) + .unwrap_or_else(|| panic!("Cannot find source for {bin}")) + .clone(); + let client_ = client.clone(); + joinset.spawn(async move { (bin, download_and_check_sha(client_, source).await) }); + } + let mut map = HashMap::new(); + while let Some(res) = joinset.join_next().await { + let (bin, content) = res.unwrap(); + let decompressed = spawn_blocking(|| { + let mut buf = BytesMut::new(); + io::copy( + &mut GzDecoder::new(content.reader()), + &mut (&mut buf).writer(), + ) + .unwrap(); + buf.freeze() + }) + .await + .unwrap(); + map.insert(bin.into(), decompressed); + } + map +} + +async fn download_bin_tarballs( + client: Client, + sources: Arc>, +) -> HashMap { + let mut map = HashMap::new(); + let [motoko, icx_proxy, ic_ref] = ["motoko", "icx-proxy", "ic-ref"].map(|pkg| { + let client = client.clone(); + let source = sources[pkg].clone(); + spawn(download_and_check_sha(client, source)) + }); + let (motoko, icx_proxy, ic_ref) = tokio::try_join!(motoko, icx_proxy, ic_ref).unwrap(); + for tar in [motoko, icx_proxy, ic_ref] { + tar_xzf(&tar, |path, content| { + map.insert(path, content); + }); + } + map +} + +async fn download_mo_base( + client: Client, + sources: Arc>, +) -> HashMap { + let source = sources["motoko-base"].clone(); + let mo_base = download_and_check_sha(client, source).await; + let mut map = HashMap::new(); + tar_xzf(&mo_base, |path, content| { + let path = path.strip_prefix(".").unwrap_or(&path); // normalize ./x to x + if let Ok(file) = path.strip_prefix("src") { + map.insert(file.to_owned(), content); + } + }); + map +} + +fn tar_xzf(gz: &[u8], mut each: impl FnMut(PathBuf, Bytes)) { + let mut tar = Archive::new(GzDecoder::new(gz)); + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry.path().unwrap_or_else(|e| { + panic!( + "Malformed file path {}: {e}", + String::from_utf8_lossy(&entry.path_bytes()) + ) + }); + let path = path.strip_prefix(".").unwrap_or(&path).to_owned(); + let mut content = BytesMut::with_capacity(entry.header().size().unwrap() as usize); + io::copy(&mut entry, &mut (&mut content).writer()).unwrap(); + each(path, content.freeze()); + } +}