Skip to content

Commit 03e10fc

Browse files
authored
Merge pull request #15466 from MinaProtocol/merge/berkeley-main
Merge berkeley with o1js-main
2 parents c56bde5 + 8b1fedd commit 03e10fc

File tree

288 files changed

+58405
-1283480
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

288 files changed

+58405
-1283480
lines changed

.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ frontend/website/static/presskit.zip filter=lfs diff=lfs merge=lfs -text
1313
frontend/website/static/*.bc.js filter=lfs diff=lfs merge=lfs -text
1414
src/app/archive/archive_graphql_schema.json linguist-generated=true
1515
docs/res/block_production_fsm.dot.png filter=lfs diff=lfs merge=lfs -text
16+
rfcs/res/hard-fork-package-generation-buildkite-pipeline.dot.png filter=lfs diff=lfs merge=lfs -text

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ automation/services/watchdog/check_libp2p/check_libp2p
8686
_opam
8787

8888
target
89-
release
89+
./release
9090
.vscode
9191

9292
### IntelliJ IDEA ###

CODEOWNERS

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
/LICENSE @aneesharaines
1212
/README.md @MinaProtocol/product-eng-reviewers
1313
/README-dev.md @MinaProtocol/protocol-eng-reviewers
14-
/graphql_schema.json @MinaProtocol/product-eng-reviewers
1514

1615
/src/external/ @bkase @psteckler @aneesharaines @mrmr1993
1716

@@ -30,7 +29,6 @@
3029
/src/lib/ @MinaProtocol/protocol-eng-reviewers
3130

3231
/src/lib/blake2/ @MinaProtocol/crypto-eng-reviewers
33-
/src/lib/mina_graphql @MinaProtocol/product-eng-reviewers
3432
/src/lib/mina_numbers/ @MinaProtocol/crypto-eng-reviewers
3533
/src/lib/crs/ @MinaProtocol/crypto-eng-reviewers
3634
/src/lib/crypto_params/ @MinaProtocol/crypto-eng-reviewers

buildkite/scripts/build-artifact.sh

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
set -eo pipefail
44

5-
eval $(opam config env)
6-
export PATH=/home/opam/.cargo/bin:/usr/lib/go/bin:$PATH
7-
export GO=/usr/lib/go/bin/go
5+
([ -z ${DUNE_PROFILE+x} ]) && echo "required env vars were not provided" && exit 1
6+
7+
source ~/.profile
88

99
MINA_COMMIT_SHA1=$(git rev-parse HEAD)
1010

@@ -19,6 +19,7 @@ echo "--- Build all major tagets required for packaging"
1919
echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
2020
echo "Rust Version: $(rustc --version)"
2121
dune build "--profile=${DUNE_PROFILE}" \
22+
${MAINNET_TARGETS} \
2223
src/app/logproc/logproc.exe \
2324
src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \
2425
src/app/generate_keypair/generate_keypair.exe \
@@ -29,6 +30,7 @@ dune build "--profile=${DUNE_PROFILE}" \
2930
src/app/extract_blocks/extract_blocks.exe \
3031
src/app/archive_blocks/archive_blocks.exe \
3132
src/app/berkeley_migration/berkeley_migration.exe \
33+
src/app/berkeley_migration_verifier/berkeley_migration_verifier.exe \
3234
src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.exe \
3335
src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe \
3436
src/app/batch_txn_tool/batch_txn_tool.exe \
@@ -39,14 +41,3 @@ dune build "--profile=${DUNE_PROFILE}" \
3941
src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
4042
src/app/test_executive/test_executive.exe \
4143
src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
42-
43-
echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
44-
echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
45-
[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
46-
make deb
47-
48-
echo "--- Upload debs to amazon s3 repo"
49-
make publish_debs
50-
51-
echo "--- Git diff after build is complete:"
52-
#git diff --exit-code -- .
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
([ -z ${CONFIG_JSON_GZ_URL+x} ] || [ -z ${NETWORK_NAME+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
6+
7+
# Set the DUNE_PROFILE from the NETWORK_NAME. For now, these are 1-1, but in the future, this may need to be a case statement
8+
case "${NETWORK_NAME}" in
9+
mainnet)
10+
DUNE_PROFILE=mainnet
11+
;;
12+
devnet|berkeley)
13+
DUNE_PROFILE=devnet
14+
;;
15+
*)
16+
echo "unrecognized network name: ${NETWORK_NAME}"
17+
exit 1
18+
;;
19+
esac
20+
export DUNE_PROFILE
21+
22+
# Set the base network config for ./scripts/hardfork/create_runtime_config.sh
23+
export FORKING_FROM_CONFIG_JSON="genesis_ledgers/${NETWORK_NAME}.json"
24+
[ ! -f "${FORKING_FROM_CONFIG_JSON}" ] && echo "${NETWORK_NAME} is not a known network name; check for existing network configs in 'genesis_ledgers/'" && exit 1
25+
26+
source ~/.profile
27+
28+
MINA_COMMIT_SHA1=$(git rev-parse HEAD)
29+
30+
echo "--- Download and extract previous network config"
31+
curl -o config.json.gz $CONFIG_JSON_GZ_URL
32+
gunzip config.json.gz
33+
34+
echo "--- Migrate accounts to new network format"
35+
# TODO: At this stage, we need to migrate the json accounts into the new network's format.
36+
# For now, this is hard-coded to the mainnet -> berkeley migration, but we need to select
37+
# a migration to perform in the future.
38+
# NB: we use sed here instead of jq, because jq is extremely slow at processing this file
39+
sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/' config.json
40+
41+
case "${NETWORK_NAME}" in
42+
mainnet)
43+
MINA_BUILD_MAINNET=true ./buildkite/scripts/build-artifact.sh
44+
;;
45+
*)
46+
./buildkite/scripts/build-artifact.sh
47+
;;
48+
esac
49+
50+
echo "--- Generate hardfork ledger tarballs"
51+
mkdir hardfork_ledgers
52+
_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | _build/default/src/app/logproc/logproc.exe
53+
54+
echo "--- Create hardfork config"
55+
FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json
56+
57+
existing_files=$(aws s3 ls s3://snark-keys.o1test.net/ | awk '{print $4}')
58+
for file in hardfork_ledgers/*; do
59+
filename=$(basename "$file")
60+
61+
if echo "$existing_files" | grep -q "$filename"; then
62+
echo "Info: $filename already exists in the bucket, packaging it instead."
63+
oldhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}')
64+
aws s3 cp "s3://snark-keys.o1test.net/$filename" "$file"
65+
newhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}')
66+
sed -i 's/$oldhash/$newhash/g' new_config.json
67+
else
68+
aws s3 cp --acl public-read "$file" s3://snark-keys.o1test.net/
69+
fi
70+
done
71+
72+
echo "--- Build hardfork package for Debian ${MINA_DEB_CODENAME}"
73+
RUNTIME_CONFIG_JSON=new_config.json LEDGER_TARBALLS="$(echo hardfork_ledgers/*.tar.gz)" ./scripts/create_hardfork_deb.sh
74+
mkdir -p /tmp/artifacts
75+
cp _build/mina*.deb /tmp/artifacts/.
76+
77+
echo "--- Upload debs to amazon s3 repo"
78+
make publish_debs
79+
80+
echo "--- Git diff after build is complete:"
81+
git diff --exit-code -- .

buildkite/scripts/build-release.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
([ -z ${DUNE_PROFILE+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
6+
7+
if [[ $1 = "archive_migration" ]] ; then
8+
git apply ./buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch
9+
fi
10+
11+
source ~/.profile
12+
source ./buildkite/scripts/export-git-env-vars.sh
13+
./buildkite/scripts/build-artifact.sh
14+
15+
echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
16+
echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
17+
[[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
18+
19+
20+
echo "--- Prepare debian packages"
21+
./scripts/rebuild-deb.sh $@
22+
23+
echo "--- Upload debs to amazon s3 repo"
24+
./buildkite/scripts/publish-deb.sh
25+
26+
if [[ $1 = "archive_migration" ]] ; then
27+
git apply -R buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch
28+
fi
29+
30+
echo "--- Git diff after build is complete:"
31+
git diff --exit-code -- .

0 commit comments

Comments
 (0)