Merge branch '0LNetworkCommunity:main' into change-authkey

Author: ea-open-source

.gitattributes

@@ -11,6 +11,5 @@
 *.txt text
 *.yml text
 
-# Use Rust syntax highlighter for Move and Move IR code
-*.move linguist-language=Rust
-*.mvir linguist-language=Rust
+# Use Move syntax highlighter for Move IR code
+*.mvir linguist-language=Move

.github/workflows/integration.yaml

@@ -35,6 +35,9 @@ jobs:
       - name: MINING
         run: RUSTC_WRAPPER=sccache SOURCE_PATH=./ make -f ol/integration-tests/test-mining.mk test
 
+      # - name: MINING ACROSS EPOCHS
+      #   run: RUSTC_WRAPPER=sccache SOURCE_PATH=./ make -f ol/integration-tests/test-mining-epochs.mk test
+
       - name: ONBOARD
         run: RUSTC_WRAPPER=sccache SOURCE_PATH=./ make -f ol/integration-tests/test-onboard.mk test
       - name: File permissions

.gitignore

@@ -64,4 +64,5 @@ key_store.json
 *.swp
 
 # prevent mv files from being committed
-*.mv
+*.mv
+language/diem-framework/releases/

Cargo.lock

@@ -28,16 +28,15 @@ CARGO_ARGS = --release
 # testnet automation settings
 ifeq (${TEST}, y)
 REPO_NAME = dev-genesis
-MNEM = $(shell cat ol/
-fixtures/mnemonic/${NS}.mnem)
+MNEM = $(shell cat ol/fixtures/mnemonic/${NS}.mnem)
 CARGO_ARGS = --locked # just keeping this from doing --release mode, while in testnet mode.
 GITHUB_USER = OLSF
 endif
 
 # Registration params
 REMOTE = 'backend=github;repository_owner=${GITHUB_USER};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC}'
 
-GENESIS_REMOTE = 'backend=github;repository_owner=${REPO_ORG};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC}'
+GENESIS_REMOTE = 'backend=github;repository_owner=${REPO_ORG};repository=${REPO_NAME};token=${DATA_PATH}/github_token.txt;namespace=${ACC};branch=master'
 
 LOCAL = 'backend=disk;path=${DATA_PATH}/key_store.json;namespace=${ACC}'
 
@@ -93,7 +92,8 @@ bins: stdlib
 stdlib:
 # cargo run ${CARGO_ARGS} -p diem-framework
 	cargo run ${CARGO_ARGS} -p diem-framework -- --create-upgrade-payload
-	sha256sum language/diem-framework/staged/stdlib.mv
+# linux uses sha265sum, but not available by default on OS X
+	bash -c "sha256sum language/diem-framework/staged/stdlib.mv || shasum -a 256 language/diem-framework/staged/stdlib.mv"
 
 
 install: mv-bin bin-path
@@ -109,7 +109,7 @@ install: mv-bin bin-path
 	cp -f ${SOURCE}/target/release/onboard ${USER_BIN_PATH}/onboard
 
 bin-path:
-	@if (cat ~/.bashrc | grep '~/bin:') ; then \
+	@if (cat ~/.bashrc | grep ${USER_BIN_PATH}) ; then \
 		echo "OK .bashrc correctly configured with PATH=~/bin" ; \
 	else \
 		echo -n "WARN Your .bashrc doesn't seem to have ~/bin as a search path. Append .bashrc with PATH=~/bin:$$PATH ? (y/n) " ; \
@@ -230,8 +230,9 @@ gen-register:
 	@echo OPER send signed transaction with configurations for *OWNER* account
 	ACC=${ACC}-oper OWNER=${ACC} IP=${IP} make reg
 
-	@echo Making pull request to genesis coordination repo
-	make gen-make-pull
+# TODO: implement the forking workflow for dev genesis?
+# @echo Making pull request to genesis coordination repo
+# make gen-make-pull
 
 init-test:
 	echo ${MNEM} | head -c -1 | cargo run -p diem-genesis-tool --  init --path=${DATA_PATH} --namespace=${ACC}
@@ -301,18 +302,14 @@ genesis:
   --layout-path ${DATA_PATH}/set_layout.toml \
 	--val-ip-address ${IP}
 
-
-	sha256sum ${DATA_PATH}/genesis.blob
+# linux uses sha265sum, but not available by default on OS X
+	bash -c "sha256sum ${DATA_PATH}/genesis.blob || shasum -a 256 ${DATA_PATH}/genesis.blob"
 
 #### NODE MANAGEMENT ####
 start:
 # run in foreground. Only for testing, use a daemon for net.
 	RUST_LOG=error cargo run -p diem-node -- --config ${DATA_PATH}/validator.node.yaml
 
-# Start a fullnode instead of a validator node
-start-full:
-	cargo run -p diem-node -- --config ${DATA_PATH}/fullnode.node.yaml
-
 daemon:
 	mkdir -p ~/.config/systemd/user/
 	cp ./ol/util/diem-node.service ~/.config/systemd/user/
@@ -401,18 +398,16 @@ ifdef TEST
 	cp ./ol/devnet/set_layout_test.toml ${DATA_PATH}/set_layout.toml
 endif
 
-fix-genesis:
-	cp ./ol/devnet/genesis/${V}/genesis.blob ${DATA_PATH}/
-	cp ./ol/devnet/genesis/${V}/genesis_waypoint ${DATA_PATH}/
-
 
 #### HELPERS ####
 set-waypoint:
 	@if test -f ${DATA_PATH}/key_store.json; then \
 		jq -r '. | with_entries(select(.key|match("-oper/waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/client_waypoint; \
-		jq -r '. | with_entries(select(.key|match("-oper/genesis-waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/genesis_waypoint; \
+		jq -r '. | with_entries(select(.key|match("-oper/genesis-waypoint";"i")))[].value' ${DATA_PATH}/key_store.json > ${DATA_PATH}/genesis_waypoint.txt; \
 	fi
 
+	cargo r -p ol -- init --update-waypoint --waypoint $(shell cat ${DATA_PATH}/client_waypoint)
+
 	@echo client_waypoint:
 	@cat ${DATA_PATH}/client_waypoint
 
@@ -450,71 +445,65 @@ debug:
 	make smoke-onboard <<< $$'${MNEM}'
 
 
-##### DEVNET TESTS #####
+#### TESTNET #####
+# The testnet is started using the same tools as genesis to have a faithful reproduction of a network from a clean slate.
 
-devnet: clear fix dev-wizard dev-genesis start
-# runs a smoke test from fixtures. 
-# Uses genesis blob from fixtures, assumes 3 validators, and test settings.
-# This will work for validator nodes alice, bob, carol, and any fullnodes; 'eve'
+# 1. The first thing necessary is initializing testnet genesis validators. All genesis nodes need to set up environment variables for their namespace/personas e.g. NS=alice. Also the TEST=y mode must be set, as well as a chain environment e.g. NODE_ENV=test. These settings must be done manually, preferably in .bashrc
 
-dev-join: clear fix fix-genesis dev-wizard
-# REQUIRES MOCK GIT INFRASTRUCTURE: OLSF/dev-genesis OLSF/dev-epoch-archive
-# see `devnet-archive` below 
-# We want to simulate the onboarding/new validator fetching genesis files from the mock archive: dev-genesis-archive
+# 2. Next those validators will register config data to a github repo OLSD/dev-genesis. Note: there could be github http errors, if validators attempt to write the same resource simultaneously
 
-# mock restore backups from dev-epoch-archive
-	rm -rf ~/.0L/restore
-# restore from MOCK archive OLSF/dev-epoch-archive
-	cargo r -p ol -- restore
-# start a node with fullnode.node.yaml configs
-	make start-full
+# THESE STEPS ARE ACHIEVED WITH `make testnet-register`
 
-dev-wizard:
-#  REQUIRES there is a genesis.blob in the fixtures/genesis/<version> you are testing
-	MNEM='${MNEM}' cargo run -p onboard -- val --prebuilt-genesis ${DATA_PATH}/genesis.blob --skip-mining --chain-id 1 --genesis-ceremony
+# 3. Wait. All genesis nodes need to complete registration. Otherwise buidling a genesis.blob (the first block), will fail.
+# 4. Each genesis node builds the genesis file locally, and submits to the github repo. (this remote genesis file is what subsequent non-genesis validators will use to bootstrap their db).
+# 5. Genesis validators can start their nodes.
 
-#### DEVNET RESTART ####
-# usually do this on Alice, which has the dev-epoch-archive repo, and dev-genesis
+# THESE STEPS ARE ACHIEVED WITH  `make testnet`
 
-# Do the ceremony: and also save the genesis fixtures, needs to happen before fix.
-dev-register: clear fix dev-wizard gen-register
-# Do a dev genesis on each node after EVERY NODE COMPLETED registration.
-dev-genesis: genesis dev-save-genesis fix-genesis
 
-#### DEVNET INFRA ####
-# To make reproducible devnet files.
+# 6. Assuming there is progress in the block production, subsequent validators can join.
 
-# Save the files to mock infrastructure i.e. devnet github
-dev-infra: dev-backup-archive dev-commit
+# THIS IS ACHIEVED WITH: testnet-onboard
 
-dev-save-genesis: set-waypoint
-	rsync -a ${DATA_PATH}/genesis* ${SOURCE}/ol/devnet/genesis/${V}/
-	git add ${SOURCE}/ol/devnet/genesis/${V}/
 
-dev-backup-archive:
-	cd ${HOME}/dev-epoch-archive && make devnet-backup
+#### 1. TESTNET SETUP ####
 
-dev-commit:
-	git commit -a -m "save genesis fixtures to ${V}" | true
-	git push | true
+testnet-init: clear fix
+#  REQUIRES there is a genesis.blob in the fixtures/genesis/<version> you are testing
+	MNEM='${MNEM}' cargo run -p onboard -- val --skip-mining --chain-id ${CHAIN_ID} --genesis-ceremony
 
+# Do the genesis ceremony registration, this includes the step testnet-validator-init-wizard
+testnet-register:  testnet-init gen-register
+# Do a dev genesis on each node after EVERY NODE COMPLETED registration.
+
+# Makes the gensis file on each genesis validator, AND SAVES TO GITHUB so that other validators can be onboarded after genesis.
+testnet-genesis: genesis set-waypoint
+	cargo run -p diem-genesis-tool ${CARGO_ARGS} -- create-repo \
+	--publish-genesis ${DATA_PATH}/genesis.blob \
+	--shared-backend ${GENESIS_REMOTE}
+
+	cargo run -p diem-genesis-tool ${CARGO_ARGS} -- create-repo \
+	--publish-genesis ${DATA_PATH}/genesis_waypoint.txt \
+	--shared-backend ${GENESIS_REMOTE}
+
+#### 2. TESTNET START ####
+
+# Do this to restart the network with new code. Assumes a registration has been completed, and the genesis validators are unchanged. If new IP addresses or number of genesis nodes changed, you must RERUN SETUP below.
+# - builds stdlib from source
+# - clears many of the home files
+# - adds fixtures
+# - initializes node configs
+# - rebuids genesis files and shares to github genesis repo
+# - starts node in validator mode
+testnet: clear fix testnet-init testnet-genesis start
+
+# For subsequent validators joining the testnet. This will fetch the genesis information saved
+testnet-onboard: clear
+	MNEM='${MNEM}' cargo run -p onboard -- val --github-org OLSF --repo dev-genesis --chain-id ${CHAIN_ID}
+# start a node with fullnode.node.yaml configs
+	cargo r -p diem-node -- -f ~/.0L/fullnode.node.yaml
 
-TAG=$(shell git tag -l "previous")
-clean-tags:
-	git push origin --delete ${TAG}
-	git tag -d ${TAG}
-	
-nuke-testnet:
-	@echo WIPING EVERYTHING but keeping: github_token.txt, autopay_batch.json, set_layout.toml, /vdf_proofs/proof_0.json
 
-	@if test -d ${DATA_PATH}; then \
-		cd ${DATA_PATH} && cp github_token.txt autopay_batch.json set_layout.toml vdf_proofs/proof_0.json ~/; \
-		cd ${DATA_PATH} && rm -rf *; \
-		cd ~ && cp github_token.txt autopay_batch.json set_layout.toml ${DATA_PATH}; \
-		cd ${DATA_PATH} && mkdir vdf_proofs;\
-		cd ~ && cp proof_0.json ${DATA_PATH}/vdf_proofs/; \
-	fi
-	
 
 ####### SWARM ########
 
@@ -566,6 +555,13 @@ fork-config:
 	cargo run -p onboard -- fork -u http://167.172.248.37 --prebuilt-genesis ${DATA_PATH}/genesis_from_snapshot.blob
 
 # start node from files
-fork-start: 
+fork-start:
 	rm -rf ~/.0L/db
 	cargo run -p libra-node -- --config ~/.0L/validator.node.yaml
+
+##### UTIL #####
+TAG=$(shell git tag -l "previous")
+clean-tags:
+	git push origin --delete ${TAG}
+	git tag -d ${TAG}
+	

Makefile

@@ -19,7 +19,7 @@ We live 24/7 on Discord [join us there](https://discord.gg/AzCp63pggW).
 
 Read our documentation [here](ol/documentation/README.md).
 
-Skip straight to [toubleshooting onboarding a node to network.](ol/documentation/node-ops/validators/troubleshoting_onboarding.md)
+Skip straight to [troubleshooting onboarding a node to network.](ol/documentation/node-ops/validators/troubleshoting_onboarding.md)
 
 Contribute to [issues](https://github.com/OLSF/libra/issues).
 
@@ -43,7 +43,7 @@ This is a non-exhaustive summary of the key features of the Diem architecture th
 The smart-contract language of the platform is called Move. It is the most unique breakthrough of the team. This is a language that is designed to be extremely safe in adversarial environments, and for hurried, less experienced developers. It's a very ergonomic language, it's easy to approach it if you have even entry-level coding experience. In terms of safety it incorporates much from the Rust language concepts of "borrowing" memory. The compiler is pretty obnoxious, which is something you want when designing autonomous financial systems. One standout feature of the Move language is built-in Formal Verification. Adjacent to the code you can write specs for invariants which your code must preserve (i.e. this function should never be called by this type of account), and it can be checked during the development and build process. This is unique and powerful.
 
 ### Programming model
-The execution of the smart contract and scripts has some subtle but important safety features. By design what are referred to in other platforms as smart-contracts are in fact "modules" here. Users can publish modules, which any other module or transaction can import. This is important. The transactions are scripts. So compared to Ethereum, much of what happens in a smart contract, can actually be split into long lived code in a module, and transaction-scripts which can import from the module (and other modules). This decoupling allows for powerful composability and reliability. The developer can evolve the application without necessarily needing to upgrade modules every time a new transaction use-case emerges.
+The execution of the smart contract and scripts has some subtle but important safety features. By design what are referred to in other platforms as smart contracts are in fact "modules" here. Users can publish modules, which any other module or transaction can import. This is important. The transactions are scripts. So compared to Ethereum, much of what happens in a smart contract, can actually be split into long-lived code in a module, and transaction-scripts which can import from the module (and other modules). This decoupling allows for powerful composability and reliability. The developer can evolve the application without necessarily needing to upgrade modules every time a new transaction use case emerges.
 
 Modules can have "resources" bound to them. A resource can be thought of as an object in memory, but with restrictions: they can only be modified by the module that instantiated them, and are restricted in how they get created and transferred. Writing a non-fungible token is basically just instantiating one such structure, and something like a fungible token, can be done in a handful of lines of code.
 
@@ -75,7 +75,7 @@ The technology we are inheriting is a spaceship. It is also purely infrastructur
 
 But most importantly, the architecture is designed as a private, consortium chain. 
 
-For system administration, there is on omnipresent Diem Association account. Yes, a private key that controls many functions including: Freezing accounts (!), selecting validators for inclusion, paying transaction fees to validators, upgrading the system code. This is obviously a non-starter. So a lot of work had to go into making system policies execute in a permissionless environment.
+For system administration, there is an omnipresent Diem Association account. Yes, a private key that controls many functions including: Freezing accounts (!), selecting validators for inclusion, paying transaction fees to validators, upgrading the system code. This is obviously a non-starter. So a lot of work had to go into making system policies execute in a permissionless environment.
 
 We also had to add Sybil resistance mechanisms. Typically communities have been choosing Proof-of-Stake as the Sybil resistance method for BFT networks. This is not the route we chose given community growth considerations (as well as regulatory). Elsewhere we've talked in detail about our Delay Towers complement to consensus.
 
@@ -134,10 +134,10 @@ We've made some additions to the MoveVM that were necessary for us to implement
 
 - Decimal - we needed a number type that could be used for financial math that could lead into polynomial curves etc. So we added the Rust Decimal library and some initial APIs and their corresponding native instructions.
 
-- VDF verification - to verify the Delay Towers proofs we added the ChiaVDF verifier to the VM. The prover is not needed in the VM. The VDF prover can accept a number of parameters (not hardcoded for 0L's use-case). So application builders could leverage it in their own games.
+- VDF verification - to verify the Delay Towers proofs we added the ChiaVDF verifier to the VM. The prover is not needed in the VM. The VDF prover can accept a number of parameters (not hardcoded for 0L's use case). So application builders could leverage it in their own games.
 
 ### Auto Pay
-The ability to create payments in the future, and regular payments as a percent of account balance or of new income. This was a feature requested early by the community. It powers a number of use-cases important at the start of a network.
+The ability to create payments in the future, and regular payments as a percent of account balance or of new income. This was a feature requested early by the community. It powers a number of use cases important at the start of a network.
 Namely it is useful for anyone that wants to run a community program. In this case an entity or a person is seeking to accomplish a goal or a project, and is asking for donations. Autopay allows for set-it-and-forget-it donations to programs. Most people in 0L use this to send a portion of their mining rewards to programs automatically.
 
 Autopay can be programmed for:

README.md

@@ -1,6 +1,6 @@
 [package]
 name = "diem-config"
-version = "5.1.1"
+version = "5.2.0"
 authors = ["Diem Association <[email protected]>"]
 description = "Diem diem-config"
 repository = "https://github.com/diem/diem"