From fa690f78686b063c9612883a29bf54e4b19d2783 Mon Sep 17 00:00:00 2001 From: georgeee Date: Wed, 3 Apr 2024 19:34:11 +0200 Subject: [PATCH 1/4] Remove --fork-state-hash parameter from final step Problem: parameter is clearly redundant Solution: take the value from the fork config --- scripts/archive/migration/README.md | 2 +- .../migration/mina-berkeley-migration-script | 24 ++++--------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/scripts/archive/migration/README.md b/scripts/archive/migration/README.md index d7736879c716..62878180993a 100644 --- a/scripts/archive/migration/README.md +++ b/scripts/archive/migration/README.md @@ -118,7 +118,7 @@ migration-checkpoint-597.json - is a last checkpoint from initial run #### Final migration ``` -./scripts/archive/migration/berkeley_migration.sh final -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_final -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-2381.json -fc ../../umt_testing/fork-umt-02-29-2024.json -f 3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7 +./scripts/archive/migration/berkeley_migration.sh final -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_final -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-2381.json -fc ../../umt_testing/fork-umt-02-29-2024.json ``` where `3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7` was extracted from fork config file: diff --git a/scripts/archive/migration/mina-berkeley-migration-script b/scripts/archive/migration/mina-berkeley-migration-script index a066dedbb936..f6f27d93a4e1 100755 --- a/scripts/archive/migration/mina-berkeley-migration-script +++ b/scripts/archive/migration/mina-berkeley-migration-script @@ -537,7 +537,6 @@ function final_help(){ echo "" printf " %-25s %s\n" "-h | --help" "show help"; printf " %-25s %s\n" "-r | --replayer-checkpoint" "[file] path to genesis ledger file"; - printf " %-25s %s\n" "-f | --fork-state-hash" "[hash] fork state hash"; printf " %-25s %s\n" "-s | --source-db" "[connection_str] connection string to database to be migrated"; printf " %-25s %s\n" "-t | --target-db" "[connection_str] connection string to database which will hold migrated data"; printf " %-25s %s\n" "-b | --blocks-bucket" "[string] name of precomputed blocks bucket. NOTICE: there is an assumption that precomputed blocks are named with format: {network}-{height}-{state_hash}.json"; @@ -549,7 +548,7 @@ function final_help(){ echo "" echo "Example:" echo "" - echo " " $CLI_NAME final --replayer-checkpoint migration-replayer-checkpoint-1233.json --fork-state-hash 3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7 --genesis-ledger "genesis_ledgers/mainnet.json" --source-db "postgres://postgres:pass@localhost:5432/archive_balances_migrated" --target-db "postgres://postgres:pass@localhost:5432/migrated" --blocks-batch-size 10 --blocks-bucket "mina_network_block_data" --network "mainnet" --fork-config fork_config.json + echo " " $CLI_NAME final --replayer-checkpoint migration-replayer-checkpoint-1233.json --genesis-ledger "genesis_ledgers/mainnet.json" --source-db "postgres://postgres:pass@localhost:5432/archive_balances_migrated" --target-db "postgres://postgres:pass@localhost:5432/migrated" --blocks-batch-size 10 --blocks-bucket "mina_network_block_data" --network "mainnet" --fork-config fork_config.json echo "" echo "Notes:" echo " 1. After run migrated data will be filled with migrated blocks till last block in source db" @@ -574,7 +573,6 @@ function final(){ local __stream_blocks=true local __network='' local __checkpoint_interval=1000 - local __fork_state_hash='' local __fork_config='' while [ ${#} -gt 0 ]; do @@ -587,10 +585,6 @@ function final(){ __genesis_ledger=${2:?$error_message} shift 2; ;; - -f | --fork-state-hash ) - __fork_state_hash=${2:?$error_message} - shift 2; - ;; -fc | --fork-config ) __fork_config=${2:?$error_message} shift 2; @@ -662,12 +656,6 @@ function final(){ echo "Genesis ledger not defined" exit 1 fi - if [ -z "$__fork_state_hash" ]; then - echo "" - echo "Fork state hash not defined. Please refer to mina or o(1) Labs team announcements regarding fork block state hash" - echo "which is required to run final migration" - exit 1 - fi if [ -z "$__fork_config" ]; then echo "" echo "Fork config file is not defined. Please refer to mina or o(1) Labs team announcements regarding fork block state hash" @@ -694,7 +682,6 @@ function final(){ "$__keep_precomputed_blocks" \ "$__stream_blocks" \ "$__network" \ - "$__fork_state_hash" \ "$__checkpoint_interval" \ "$__replayer_checkpoint" \ "$__fork_config" @@ -709,12 +696,11 @@ function run_final_migration() { local __keep_precomputed_blocks=$6 local __stream_blocks=$7 local __network=$8 - local __fork_state_hash=$9 - local __checkpoint_interval=${10} - local __replayer_checkpoint=${11} - local __fork_config=${12} - + local __checkpoint_interval=$9 + local __replayer_checkpoint=${10} + local __fork_config=${11} + local __fork_state_hash="$(jq -r .proof.fork.state_hash "$__fork_config")" local __date=$(date '+%Y-%m-%d_%H%M') local __berkely_migration_log="berkeley_migration_$__date.log" local __replayer_log="replayer_$__date.log" From 4369ab49090b2480d38f72740f5db76a4ad5109e Mon Sep 17 00:00:00 2001 From: georgeee Date: Wed, 3 Apr 2024 19:39:22 +0200 Subject: [PATCH 2/4] Make error-log-grepping expression more strict --- scripts/archive/migration/mina-berkeley-migration-script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/archive/migration/mina-berkeley-migration-script b/scripts/archive/migration/mina-berkeley-migration-script index f6f27d93a4e1..5444bc6509be 100755 --- a/scripts/archive/migration/mina-berkeley-migration-script +++ b/scripts/archive/migration/mina-berkeley-migration-script @@ -209,7 +209,7 @@ function initial(){ function check_log_for_error() { local __log=$1; - grep Error "$__log"; + grep '"level":"Error"' "$__log"; local __have_errors=$?; if [ $__have_errors -eq 0 ]; then From 298b1a4bc070c316bc107b21212892ea3b10b61c Mon Sep 17 00:00:00 2001 From: emberian Date: Tue, 2 Apr 2024 20:34:03 -0400 Subject: [PATCH 3/4] mina-berkeley-migration-script name --- scripts/archive/migration/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/archive/migration/README.md b/scripts/archive/migration/README.md index 62878180993a..b76ad777b40d 100644 --- a/scripts/archive/migration/README.md +++ b/scripts/archive/migration/README.md @@ -100,7 +100,7 @@ Since berkeley migration script supports all berkeley migration phases. We need #### Initial migration ``` -./scripts/archive/migration/berkeley_migration.sh initial -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_initial -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 1000 -n o1labs-umt-pre-fork-run-1 +mina-berkeley-migration-script initial -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_initial -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 1000 -n o1labs-umt-pre-fork-run-1 ``` this command should output migration-replayer-XXX.json which should be used in next run @@ -108,7 +108,7 @@ this command should output migration-replayer-XXX.json which should be used in n #### Incremental migration ``` -./scripts/archive/migration/berkeley_migration.sh incremental -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_increment -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-597.json +mina-berkeley-migration-script incremental -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_increment -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-597.json ``` where: @@ -118,7 +118,7 @@ migration-checkpoint-597.json - is a last checkpoint from initial run #### Final migration ``` -./scripts/archive/migration/berkeley_migration.sh final -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_final -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-2381.json -fc ../../umt_testing/fork-umt-02-29-2024.json +mina-berkeley-migration-script final -g ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_final -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-2381.json -fc ../../umt_testing/fork-umt-02-29-2024.json ``` where `3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7` was extracted from fork config file: From b22841263360bb250e93ba2eb172bae7dd01314b Mon Sep 17 00:00:00 2001 From: georgeee Date: Thu, 4 Apr 2024 15:45:44 +0200 Subject: [PATCH 4/4] Rename incorrect references to fork config Problem: migration verifier tool and migration script incorrectly named genesis config of the fork network as the fork config which introduced certain confusion. Solution: rename all such occurences of fork config to fork genesis config. --- .../migration/mina-berkeley-migration-script | 22 ++++++------ src/app/berkeley_migration_verifier/Readme.md | 4 +-- .../berkeley_migration_verifier.ml | 35 ++++++++++--------- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/scripts/archive/migration/mina-berkeley-migration-script b/scripts/archive/migration/mina-berkeley-migration-script index 5444bc6509be..4f8e1be58f71 100755 --- a/scripts/archive/migration/mina-berkeley-migration-script +++ b/scripts/archive/migration/mina-berkeley-migration-script @@ -542,13 +542,13 @@ function final_help(){ printf " %-25s %s\n" "-b | --blocks-bucket" "[string] name of precomputed blocks bucket. NOTICE: there is an assumption that precomputed blocks are named with format: {network}-{height}-{state_hash}.json"; printf " %-25s %s\n" "-bs | --blocks-batch-size" "[int] number of precomputed blocks to be fetch at once from Gcloud. Bigger number like 1000 can help speed up migration process"; printf " %-25s %s\n" "-n | --network" "[string] network name when determining precomputed blocks. NOTICE: there is an assumption that precomputed blocks are named with format: {network}-{height}-{state_hash}.json"; - printf " %-25s %s\n" "-fc | --fork-config" "[file] Fork-state config file is dump file in form of json containing fork block and ledger file. It should be provied by MF or O(1)Labs team after fork block is announced"; + printf " %-25s %s\n" "-fc | --fork-genesis-config" "[file] Genesis config file for the fork network. It should be provied by MF or O(1)Labs team after fork block is announced"; printf " %-25s %s\n" "-d | --delete-blocks" "delete blocks after they are processed (saves space with -sb)" printf " %-25s %s\n" "-p | --prefetch-blocks" "downloads all blocks at once instead of incrementally" echo "" echo "Example:" echo "" - echo " " $CLI_NAME final --replayer-checkpoint migration-replayer-checkpoint-1233.json --genesis-ledger "genesis_ledgers/mainnet.json" --source-db "postgres://postgres:pass@localhost:5432/archive_balances_migrated" --target-db "postgres://postgres:pass@localhost:5432/migrated" --blocks-batch-size 10 --blocks-bucket "mina_network_block_data" --network "mainnet" --fork-config fork_config.json + echo " " $CLI_NAME final --replayer-checkpoint migration-replayer-checkpoint-1233.json --genesis-ledger "genesis_ledgers/mainnet.json" --source-db "postgres://postgres:pass@localhost:5432/archive_balances_migrated" --target-db "postgres://postgres:pass@localhost:5432/migrated" --blocks-batch-size 10 --blocks-bucket "mina_network_block_data" --network "mainnet" --fork-genesis-config fork_genesis_config.json echo "" echo "Notes:" echo " 1. After run migrated data will be filled with migrated blocks till last block in source db" @@ -573,7 +573,7 @@ function final(){ local __stream_blocks=true local __network='' local __checkpoint_interval=1000 - local __fork_config='' + local __fork_genesis_config='' while [ ${#} -gt 0 ]; do error_message="Error: a value is needed for '$1'"; @@ -585,8 +585,8 @@ function final(){ __genesis_ledger=${2:?$error_message} shift 2; ;; - -fc | --fork-config ) - __fork_config=${2:?$error_message} + -fc | --fork-genesis-config ) + __fork_genesis_config=${2:?$error_message} shift 2; ;; -r | --replayer-checkpoint ) @@ -656,9 +656,9 @@ function final(){ echo "Genesis ledger not defined" exit 1 fi - if [ -z "$__fork_config" ]; then + if [ -z "$__fork_genesis_config" ]; then echo "" - echo "Fork config file is not defined. Please refer to mina or o(1) Labs team announcements regarding fork block state hash" + echo "Fork genesis config file is not defined. Please refer to mina or o(1) Labs team announcements regarding fork block state hash" echo "which is required to run final migration" exit 1 fi @@ -684,7 +684,7 @@ function final(){ "$__network" \ "$__checkpoint_interval" \ "$__replayer_checkpoint" \ - "$__fork_config" + "$__fork_genesis_config" } function run_final_migration() { @@ -698,9 +698,9 @@ function run_final_migration() { local __network=$8 local __checkpoint_interval=$9 local __replayer_checkpoint=${10} - local __fork_config=${11} + local __fork_genesis_config=${11} - local __fork_state_hash="$(jq -r .proof.fork.state_hash "$__fork_config")" + local __fork_state_hash="$(jq -r .proof.fork.state_hash "$__fork_genesis_config")" local __date=$(date '+%Y-%m-%d_%H%M') local __berkely_migration_log="berkeley_migration_$__date.log" local __replayer_log="replayer_$__date.log" @@ -744,7 +744,7 @@ function run_final_migration() { mina-berkeley-migration-verifier post-fork \ --mainnet-archive-uri "$__mainnet_archive_uri" \ --migrated-archive-uri "$__migrated_archive_uri" \ - --fork-config-file "$__fork_config" \ + --fork-genesis-config "$__fork_genesis_config" \ --migrated-replayer-output "$migrated_replayer_output" } diff --git a/src/app/berkeley_migration_verifier/Readme.md b/src/app/berkeley_migration_verifier/Readme.md index 550f44c80024..c6928a5ae12f 100644 --- a/src/app/berkeley_migration_verifier/Readme.md +++ b/src/app/berkeley_migration_verifier/Readme.md @@ -12,7 +12,7 @@ Application for validating migrated archive schema content. Performed checks rel Basic usage : ``` -mina-berkeley-migration-verifier --mainnet-archive-uri postgres://postgres:postgres@localhost:5432/source_archive --migrated-archive-uri postgres://postgres:postgres@localhost:5432/archive_migrated --migrated-replayer-output migrated_replayer.json --fork-config-file fork_config_fixed.json +mina-berkeley-migration-verifier --mainnet-archive-uri postgres://postgres:postgres@localhost:5432/source_archive --migrated-archive-uri postgres://postgres:postgres@localhost:5432/archive_migrated --migrated-replayer-output migrated_replayer.json --fork-genesis-config fork_config_fixed.json ``` where: @@ -20,7 +20,7 @@ where: - **mainnet-archive-uri** is a connection string to original schema - **migrated-archive-uri** is a connection string to already migrated schema - **migrated-replayer-output** is an output for replayer on migrated schema -- **fork-config-file** is a state dump with forked ledger +- **fork-genesis-config** is a genesis config for the fork network ### Dependencies diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml index f377d693fc6b..1e14a04088b1 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml @@ -255,14 +255,14 @@ let compare_internal_commands migrated_pool mainnet_pool ~work_dir = in diff_files internal_commands_berk internal_commands_main -let compare_ledger_hash ~migrated_replayer_output ~fork_config_file = +let compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file = let checkpoint_ledger_hash = Yojson.Basic.from_file migrated_replayer_output |> member "genesis_ledger" |> member "hash" |> to_string |> Ledger_hash.of_base58_check_exn in let fork_ledger_hash = - Yojson.Basic.from_file fork_config_file + Yojson.Basic.from_file fork_genesis_config_file |> member "ledger" |> member "hash" |> to_string |> Ledger_hash.of_base58_check_exn in @@ -353,24 +353,26 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = Deferred.Or_error.errorf "Some tests failed. Please refer to above output for details" -let fork_config_exn ~fork_config_file = - Yojson.Basic.from_file fork_config_file |> member "proof" |> member "fork" +let fork_config_exn ~fork_genesis_config_file = + Yojson.Basic.from_file fork_genesis_config_file + |> member "proof" |> member "fork" -let fork_block_state_hash_exn ~fork_config_file = - fork_config_exn ~fork_config_file |> member "state_hash" |> to_string +let fork_block_state_hash_exn ~fork_genesis_config_file = + fork_config_exn ~fork_genesis_config_file |> member "state_hash" |> to_string -let fork_block_height_exn ~fork_config_file = - fork_config_exn ~fork_config_file |> member "blockchain_length" |> to_int +let fork_block_height_exn ~fork_genesis_config_file = + fork_config_exn ~fork_genesis_config_file + |> member "blockchain_length" |> to_int let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri - ~migrated_replayer_output ~fork_config_file () = + ~migrated_replayer_output ~fork_genesis_config_file () = Async.printf "Running verifications for incremental migration between '%s' and '%s' \ schemas. It may take a couple of minutes... \n" mainnet_archive_uri migrated_archive_uri ; - let fork_height = fork_block_height_exn ~fork_config_file in - let fork_state_hash = fork_block_state_hash_exn ~fork_config_file in + let fork_height = fork_block_height_exn ~fork_genesis_config_file in + let fork_state_hash = fork_block_state_hash_exn ~fork_genesis_config_file in let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in @@ -451,7 +453,7 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri |> Test.eval ; let check = - compare_ledger_hash ~migrated_replayer_output ~fork_config_file + compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file in Test.of_check check ~name:"Verify fork config vs migrated replayer output" ~idx:8 ~prefix:"A10.3" test_count @@ -494,14 +496,15 @@ let post_fork_migration_command = ~aliases:[ "-migrated-replayer-output" ] Command.Param.(required string) ~doc:"Path Path to migrated replayer output" - and fork_config_file = - Command.Param.flag "--fork-config-file" ~aliases:[ "-fork-config-file" ] + and fork_genesis_config_file = + Command.Param.flag "--fork-genesis-config" + ~aliases:[ "-fork-genesis-config" ] Command.Param.(required string) - ~doc:"String Path to fork config file" + ~doc:"String Path to config file of the fork network" in post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri - ~migrated_replayer_output ~fork_config_file) + ~migrated_replayer_output ~fork_genesis_config_file) let commands = [ ("pre-fork", incremental_migration_command)