Merge berkeley into main

.gitattributes

@@ -13,3 +13,4 @@ frontend/website/static/presskit.zip filter=lfs diff=lfs merge=lfs -text
 frontend/website/static/*.bc.js filter=lfs diff=lfs merge=lfs -text
 src/app/archive/archive_graphql_schema.json linguist-generated=true
 docs/res/block_production_fsm.dot.png filter=lfs diff=lfs merge=lfs -text
+rfcs/res/hard-fork-package-generation-buildkite-pipeline.dot.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -86,7 +86,7 @@ automation/services/watchdog/check_libp2p/check_libp2p
 _opam
 
 target
-release
+./release
 .vscode
 
 ### IntelliJ IDEA ###

CODEOWNERS

@@ -11,7 +11,6 @@
 /LICENSE @aneesharaines
 /README.md @MinaProtocol/product-eng-reviewers
 /README-dev.md @MinaProtocol/protocol-eng-reviewers
-/graphql_schema.json @MinaProtocol/product-eng-reviewers
 
 /src/external/ @bkase @psteckler @aneesharaines @mrmr1993
 
@@ -30,7 +29,6 @@
 /src/lib/ @MinaProtocol/protocol-eng-reviewers
 
 /src/lib/blake2/ @MinaProtocol/crypto-eng-reviewers
-/src/lib/mina_graphql @MinaProtocol/product-eng-reviewers
 /src/lib/mina_numbers/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/crs/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/crypto_params/ @MinaProtocol/crypto-eng-reviewers

buildkite/scripts/build-artifact.sh

@@ -2,9 +2,9 @@
 
 set -eo pipefail
 
-eval $(opam config env)
-export PATH=/home/opam/.cargo/bin:/usr/lib/go/bin:$PATH
-export GO=/usr/lib/go/bin/go
+([ -z ${DUNE_PROFILE+x} ]) && echo "required env vars were not provided" && exit 1
+
+source ~/.profile
 
 MINA_COMMIT_SHA1=$(git rev-parse HEAD)
 
@@ -19,6 +19,7 @@ echo "--- Build all major tagets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
 echo "Rust Version: $(rustc --version)"
 dune build "--profile=${DUNE_PROFILE}" \
+  ${MAINNET_TARGETS} \
   src/app/logproc/logproc.exe \
   src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \
   src/app/generate_keypair/generate_keypair.exe \
@@ -29,6 +30,7 @@ dune build "--profile=${DUNE_PROFILE}" \
   src/app/extract_blocks/extract_blocks.exe \
   src/app/archive_blocks/archive_blocks.exe \
   src/app/berkeley_migration/berkeley_migration.exe \
+  src/app/berkeley_migration_verifier/berkeley_migration_verifier.exe \
   src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.exe \
   src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe \
   src/app/batch_txn_tool/batch_txn_tool.exe \
@@ -39,14 +41,3 @@ dune build "--profile=${DUNE_PROFILE}" \
   src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe  \
   src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
-
-echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
-echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
-[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
-make deb
-
-echo "--- Upload debs to amazon s3 repo"
-make publish_debs
-
-echo "--- Git diff after build is complete:"
-git diff --exit-code -- .

buildkite/scripts/build-hardfork-package.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -eo pipefail
+
+([ -z ${CONFIG_JSON_GZ_URL+x} ] || [ -z ${NETWORK_NAME+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
+
+# Set the DUNE_PROFILE from the NETWORK_NAME. For now, these are 1-1, but in the future, this may need to be a case statement
+case "${NETWORK_NAME}" in
+  mainnet)
+    DUNE_PROFILE=mainnet
+    ;;
+  devnet|berkeley)
+    DUNE_PROFILE=devnet
+    ;;
+  *)
+    echo "unrecognized network name: ${NETWORK_NAME}"
+    exit 1
+    ;;
+esac
+export DUNE_PROFILE
+
+# Set the base network config for ./scripts/hardfork/create_runtime_config.sh
+export FORKING_FROM_CONFIG_JSON="genesis_ledgers/${NETWORK_NAME}.json"
+[ ! -f "${FORKING_FROM_CONFIG_JSON}" ] && echo "${NETWORK_NAME} is not a known network name; check for existing network configs in 'genesis_ledgers/'" && exit 1
+
+source ~/.profile
+
+MINA_COMMIT_SHA1=$(git rev-parse HEAD)
+
+echo "--- Download and extract previous network config"
+curl -o config.json.gz $CONFIG_JSON_GZ_URL
+gunzip config.json.gz
+
+echo "--- Migrate accounts to new network format"
+# TODO: At this stage, we need to migrate the json accounts into the new network's format.
+#       For now, this is hard-coded to the mainnet -> berkeley migration, but we need to select
+#       a migration to perform in the future.
+# NB: we use sed here instead of jq, because jq is extremely slow at processing this file
+sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/' config.json
+
+case "${NETWORK_NAME}" in
+  mainnet)
+    MINA_BUILD_MAINNET=true ./buildkite/scripts/build-artifact.sh
+    ;;
+  *)
+    ./buildkite/scripts/build-artifact.sh
+    ;;
+esac
+
+echo "--- Generate hardfork ledger tarballs"
+mkdir hardfork_ledgers
+_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | _build/default/src/app/logproc/logproc.exe
+
+echo "--- Create hardfork config"
+FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json
+
+existing_files=$(aws s3 ls s3://snark-keys.o1test.net/ | awk '{print $4}')
+for file in hardfork_ledgers/*; do
+  filename=$(basename "$file")
+
+  if echo "$existing_files" | grep -q "$filename"; then
+    echo "Info: $filename already exists in the bucket, packaging it instead."
+    oldhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}')
+    aws s3 cp "s3://snark-keys.o1test.net/$filename" "$file"
+    newhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}')
+    sed -i 's/$oldhash/$newhash/g' new_config.json 
+  else
+    aws s3 cp --acl public-read "$file" s3://snark-keys.o1test.net/
+  fi
+done
+
+echo "--- Build hardfork package for Debian ${MINA_DEB_CODENAME}"
+RUNTIME_CONFIG_JSON=new_config.json LEDGER_TARBALLS="$(echo hardfork_ledgers/*.tar.gz)" ./scripts/create_hardfork_deb.sh
+mkdir -p /tmp/artifacts
+cp _build/mina*.deb /tmp/artifacts/.
+
+echo "--- Upload debs to amazon s3 repo"
+make publish_debs
+
+echo "--- Git diff after build is complete:"
+git diff --exit-code -- .

buildkite/scripts/build-release.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -eo pipefail
+
+([ -z ${DUNE_PROFILE+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
+
+if [[ $1 = "archive_migration" ]] ; then
+    git apply ./buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch
+fi
+
+source ~/.profile
+source ./buildkite/scripts/export-git-env-vars.sh
+./buildkite/scripts/build-artifact.sh
+
+echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
+echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
+[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
+
+
+echo "--- Prepare debian packages"
+./scripts/rebuild-deb.sh $@
+
+echo "--- Upload debs to amazon s3 repo"
+./buildkite/scripts/publish-deb.sh
+
+if [[ $1 = "archive_migration" ]] ; then 
+    git apply -R buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch
+fi
+
+echo "--- Git diff after build is complete:"
+git diff --exit-code -- .