mina-verify-packaged-fork-config checks the hashes against a block from the archive

buildkite/src/Command/MinaArtifact.dhall

@@ -136,7 +136,21 @@ let hardforkPipeline : DebianVersions.DebVersion -> Pipeline.Config.Type =
                 Cmd.runInDocker Cmd.Docker::{ 
                   image = "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network}"
                 , extraEnv = [ "CONFIG_JSON_GZ_URL=\$CONFIG_JSON_GZ_URL",  "NETWORK_NAME=\$NETWORK_NAME" ]
-                } "curl \$CONFIG_JSON_GZ_URL > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config config.json /workdir/verification"
+                -- an account with this balance seems present in many ledgers?
+                } "curl \$CONFIG_JSON_GZ_URL > config.json.gz && gunzip config.json.gz && sed -e '0,/20.000001/{s/20.000001/20.01/}' -i config.json && ! (mina-verify-packaged-fork-config \$NETWORK_NAME config.json /workdir/verification)"
+            ]
+            , label = "Assert corrupted packaged artifacts are unverifiable"
+            , key = "assert-unverify-corrupted-packaged-artifacts"
+            , target = Size.XLarge
+            , depends_on = [{ name = pipelineName, key = "daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" }]
+            , `if` = None B/If
+            }
+        , Command.build Command.Config::{
+            commands = [
+                Cmd.runInDocker Cmd.Docker::{
+                  image = "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network}"
+                , extraEnv = [ "CONFIG_JSON_GZ_URL=\$CONFIG_JSON_GZ_URL",  "NETWORK_NAME=\$NETWORK_NAME" ]
+                } "curl \$CONFIG_JSON_GZ_URL > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config \$NETWORK_NAME config.json /workdir/verification"
             ]
             , label = "Verify packaged artifacts"
             , key = "verify-packaged-artifacts"

dockerfiles/Dockerfile-mina-daemon

@@ -69,6 +69,7 @@ RUN echo "Building image with version $deb_version from repo $deb_release $deb_c
   && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \
   && apt-get update --quiet --yes \
   && apt-get install --quiet --yes --allow-downgrades "${MINA_DEB}=$deb_version" \
+  && apt-get install --quiet --yes --allow-downgrades "mina-create-legacy-genesis=1.4.0beta2-compatible-97f7d8c" \
   && rm -rf /var/lib/apt/lists/*
 
 

docs/upgrading-to-berkeley.md

@@ -66,7 +66,7 @@ done
 A script is installed (from `./scripts/mina-verify-packaged-fork-config`) that automates this process. If you want to verify that an installed Mina package was generated from the same configuration as the one exported earlier, it is as easy as:
 
 ```
-mina-verify-packaged-fork-config fork_config.json /tmp/mina-verification
+mina-verify-packaged-fork-config (mainnet|devnet) fork_config.json /tmp/mina-verification
 ```
 
 Many of the script inputs are environment variables that default to the locations used by the debs. If you are building from source, inspect the script, determine what you need to change, and then run it.

scripts/mina-verify-packaged-fork-config

@@ -3,7 +3,7 @@
 set -eo pipefail
 
 if [ $# -lt 2 ]; then
-    echo "Usage: $0 <mainnet-fork-config.json> <working-dir>"
+    echo "Usage: $0 <network-name> <fork-config.json> <working-dir>"
     cat <<EOF
 This script is used to validate that an installed package is correct
 according to an exported fork_config.json file.
@@ -12,6 +12,7 @@ Inputs:
 - The exported mainnet full config.json fork config, with all accounts
 - A working directory where ledgers/configs will be created
 - Installed MINA_EXE (default: mina) and MINA_GENESIS_EXE (default: mina-create-genesis) programs
+- Installed MINA_LEGACY_GENESIS_EXE program (default: mina-legacy-create-genesis)
 - PACKAGED_DAEMON_CONFIG (default: /var/lib/coda/config_*.json)
   the runtime config generated by the HF packaging
 - CREATE_RUNTIME_CONFIG (default: mina-hf-create-runtime-config)
@@ -21,6 +22,9 @@ Inputs:
 - FORKING_FROM_CONFIG_JSON (default: /var/lib/coda/mainnet.json)
   the pre-fork genesis ledger
 - SECONDS_PER_SLOT (default: 180)
+- PRECOMPUTED_FORK_BLOCK (default: fetches with gsutil)
+- GSUTIL (default: gsutil)
+  the Google Cloud Storage utility if the PRECOMPUTED_FORK_BLOCK isn't a file
 
 Ensures:
 - The accounts listed in config.json are the ones in the PACKAGED_DAEMON_CONFIG
@@ -30,7 +34,7 @@ Ensures:
 Outputs:
 - Exit code 0 if validated, 1 otherwise.
 EOF
-    
+
     exit 1
 fi
 
@@ -49,7 +53,14 @@ source_build_fallback() {
 
 MINA_EXE=${MINA_EXE:-$(source_build_fallback "$(command -v mina)" ./_build/default/src/app/cli/src/mina.exe)}
 MINA_GENESIS_EXE=${MINA_GENESIS_EXE:-$(source_build_fallback "$(command -v mina-create-genesis)" ./_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe)}
+MINA_LEGACY_GENESIS_EXE=${MINA_LEGACY_GENESIS_EXE:-$(source_build_fallback "$(command -v mina-create-legacy-genesis)" ./runtime_genesis_ledger_of_mainnet.exe)}
 CREATE_RUNTIME_CONFIG=${CREATE_RUNTIME_CONFIG:-$(source_build_fallback "$(command -v mina-hf-create-runtime-config)" ./scripts/hardfork/create_runtime_config.sh)}
+GSUTIL=${GSUTIL:-$(source_build_fallback "$(command -v gsutil)" false)}
+
+if [[ -e "$PRECOMPUTED_FORK_BLOCK"  && ! -x "$GSUTIL" ]]; then
+    echo "Error: gsutil is required when PRECOMPUTED_FORK_BLOCK is nonexistent path"
+    exit 1
+fi
 
 installed_config=$(echo /var/lib/coda/config_*.json)
 PACKAGED_DAEMON_CONFIG=${PACKAGED_DAEMON_CONFIG:-$installed_config}
@@ -67,24 +78,52 @@ export FORKING_FROM_CONFIG_JSON=${FORKING_FROM_CONFIG_JSON:-$(source_build_fallb
 
 export MINA_LIBP2P_PASS=''
 
-workdir=$2
-mkdir -p "$workdir"
-mkdir -p "$workdir/ledgers"
-mkdir -p "$workdir/ledgers-backup"
-mkdir -p "$workdir/keys"
+workdir=$3
+mkdir -p "$workdir"/{ledgers{,-backup},keys}
 chmod 700 "$workdir/keys"
 
+fork_block_state_hash=$(jq -r '.proof.fork.state_hash' "$2")
+fork_block_length=$(jq -r '.proof.fork.blockchain_length' "$2")
+
+# Put the fork block where we want it, fetch it from gcloud if necessary
+if [ ! -e "$PRECOMPUTED_FORK_BLOCK" ]; then
+    if [ "$PRECOMPUTED_FORK_BLOCK" = "" ]; then
+        PRECOMPUTED_FORK_BLOCK="gs://mina_network_block_data/$1-$fork_block_length-$fork_block_state_hash.json"
+    fi
+    "$GSUTIL" cp "$PRECOMPUTED_FORK_BLOCK" "$workdir/precomputed_fork_block.json"
+else
+    cp "$PRECOMPUTED_FORK_BLOCK" "$workdir/precomputed_fork_block.json"
+fi
+
+
 if [ ! -e "$workdir/keys/p2p" ]; then
     "$MINA_EXE" libp2p generate-keypair --privkey-path "$workdir/keys/p2p"
 fi
 
 echo "generating genesis ledgers ... (this may take a while)" >&2
 
-cp "$1" "$workdir/config.json"
-sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/' "$workdir/config.json"
-"$MINA_GENESIS_EXE" --config-file "$workdir/config.json"  --genesis-dir "$workdir/ledgers"  --hash-output-file "$workdir/hashes.json"
+cp "$2" "$workdir/config_orig.json"
+
+# Patch against a bug in 1.4 which is fixed by PR #15462
+jq 'del(.ledger.num_accounts) | del(.ledger.name)' "$workdir/config_orig.json" > "$workdir/config.json" 
+
+"$MINA_LEGACY_GENESIS_EXE" --config-file "$workdir/config.json" --genesis-dir "$workdir/legacy_ledgers" --hash-output-file "$workdir/legacy_hashes.json"
 
-FORK_CONFIG_JSON="$1" \
+result=$(jq --slurpfile block "$workdir/precomputed_fork_block.json" \
+  --slurpfile legacy_hashes "$workdir/legacy_hashes.json" -n '
+  ($legacy_hashes[0].epoch_data.staking.hash == $block[0].protocol_state.body.consensus_state.staking_epoch_data.ledger.hash and
+   $legacy_hashes[0].epoch_data.next.hash == $block[0].protocol_state.body.consensus_state.next_epoch_data.ledger.hash and
+   $legacy_hashes[0].ledger.hash == $block[0].protocol_state.body.blockchain_state.staged_ledger_hash.non_snark.ledger_hash)')
+
+if [ "$result" != "true" ]; then
+    echo "Hashes in config $2 don't match hashes from the precomputed block $PRECOMPUTED_FORK_BLOCK" >&2
+    exit 1
+fi
+
+sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/g' "$workdir/config.json"
+"$MINA_GENESIS_EXE" --config-file "$workdir/config.json" --genesis-dir "$workdir/ledgers" --hash-output-file "$workdir/hashes.json"
+
+FORK_CONFIG_JSON="$2" \
 LEDGER_HASHES_JSON="$workdir/hashes.json" \
 GENESIS_TIMESTAMP=$(jq -r '.genesis.genesis_state_timestamp' "$PACKAGED_DAEMON_CONFIG") \
 "$CREATE_RUNTIME_CONFIG" > "$workdir/config-substituted.json"
@@ -97,18 +136,18 @@ function extract_ledgers() {
     ledger_dir=$2
     json_prefix=$3
     "$MINA_EXE" daemon --libp2p-keypair "$workdir/keys/p2p" --config-file "$config_file" --seed --genesis-ledger-dir "$ledger_dir" &
-    
-    while ! "$MINA_EXE" ledger export staged-ledger  | jq . >"$json_prefix-staged.json"; do
+
+    while ! "$MINA_EXE" ledger export staged-ledger  | jq . 2>/dev/null >"$json_prefix-staged.json"; do
         sleep 1m
         if ! grep -qFx "$(cat ~/.mina-config/.mina-lock)" <(jobs -rp); then
             echo "daemon died before exporting ledgers" >&2
             exit 1
         fi
     done
-    
+
     "$MINA_EXE" ledger export staking-epoch-ledger | jq . > "$json_prefix-staking.json"
     "$MINA_EXE" ledger export next-epoch-ledger | jq . > "$json_prefix-next.json"
-    
+
     "$MINA_EXE" client stop
 }
 
@@ -134,7 +173,7 @@ for file in "$workdir"/packaged-*.json; do
     name=$(basename "$file")
     name=${name%.json}
     name=${name#packaged-}
-    
+
     if ! cmp "$file" "$workdir/reference-$name.json"; then
         echo "Error: $file does not match reference" >&2
         error=1
@@ -167,4 +206,4 @@ if [ $error -ne 0 ]; then
 else
     echo "Validation successful" >&2
     exit 0
-fi
+fi