diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 029c486365088..7b16c349ad92e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -186,300 +186,6 @@ default: - sccache -s -#### stage: .pre - -skip-if-draft: - image: paritytech/tools:latest - <<: *kubernetes-env - stage: .pre - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - echo "Commit message is ${CI_COMMIT_MESSAGE}" - - echo "Ref is ${CI_COMMIT_REF_NAME}" - - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - - ./scripts/ci/gitlab/skip_if_draft.sh - -#### stage: check - -check-runtime: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - variables: - <<: *default-vars - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./scripts/ci/gitlab/check_runtime.sh - allow_failure: true - -check-signed-tag: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./scripts/ci/gitlab/check_signed.sh - -test-dependency-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./scripts/ci/gitlab/ensure-deps.sh - -test-prometheus-alerting-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_BRANCH - changes: - - .gitlab-ci.yml - - ./scripts/ci/monitoring/**/* - script: - - promtool check rules ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml - - cat ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml | - promtool test rules ./scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml - -#### stage: test - -cargo-deny: - stage: test - <<: *docker-env - <<: *nightly-pipeline - script: - - cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml - after_script: - - echo "___The complete log is in the artifacts___" - - cargo deny check -c ./scripts/ci/deny.toml 2> deny.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - when: always - paths: - - deny.log - # FIXME: Temporarily allow to fail. - allow_failure: true - -cargo-fmt: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly fmt --all -- --check - -cargo-clippy: - stage: test - <<: *docker-env - <<: *test-refs - script: - - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets - -cargo-check-benches: - stage: test - <<: *docker-env - <<: *test-refs - <<: *collect-artifacts - before_script: - # merges in the master branch on PRs - - *merge-ref-into-master-script - - *rust-info-script - script: - - *cargo-check-benches-script - tags: - - linux-docker-benches - -node-bench-regression-guard: - # it's not belong to `build` semantically, but dag jobs can't depend on each other - # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632 - # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402 - stage: build - <<: *docker-env - <<: *test-refs-no-trigger-prs-only - needs: - # this is a DAG - - job: cargo-check-benches - artifacts: true - # this does not like a DAG, just polls the artifact - - project: $CI_PROJECT_PATH - job: cargo-check-benches - ref: master - artifacts: true - variables: - CI_IMAGE: "paritytech/node-bench-regression-guard:latest" - before_script: [""] - script: - - echo "------- IMPORTANT -------" - - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job" - - echo "In case of this job failure, check your pipeline's cargo-check-benches" - - 'node-bench-regression-guard --reference artifacts/benches/master-* - --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' - -cargo-check-subkey: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - -cargo-check-try-runtime: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features try-runtime - - sccache -s - -cargo-check-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features wasmer-sandbox - - sccache -s - -test-deterministic-wasm: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - WASM_BUILD_NO_COLOR: 1 - script: - # build runtime - - cargo build --verbose --release -p node-runtime - # make checksum - - sha256sum ./target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 - # clean up – FIXME: can we reuse some of the artifacts? - - cargo clean - # build again - - cargo build --verbose --release -p node-runtime - # confirm checksum - - sha256sum -c ./checksum.sha256 - - sccache -s - -test-linux-stable: &test-linux - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - RUST_BACKTRACE: 1 - WASM_BUILD_NO_COLOR: 1 - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests - - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml - - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec - - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - - sccache -s - -test-frame-examples-compile-to-wasm: - # into one job - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd ./frame/examples/offchain-worker/ - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../basic - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - sccache -s - -test-linux-stable-int: - <<: *test-linux - stage: test - script: - - echo "___Logs will be partly shown at the end in case of failure.___" - - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 - RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace - time cargo test -p node-cli --release --verbose --locked -- --ignored - &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - - sccache -s - after_script: - - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - when: on_failure - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_int_failure.log - -check-tracing: - stage: test - <<: *docker-env - <<: *test-refs - script: - # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features - - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - - sccache -s - -test-full-crypto-feature: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd primitives/core/ - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - cd ../application-crypto - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - sccache -s - -test-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs-wasmer-sandbox - variables: - <<: *default-vars - script: - - time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests - - sccache -s - -cargo-check-macos: - stage: test - # shell runner on mac ignores the image set in *docker-env - <<: *docker-env - <<: *test-refs-no-trigger - script: - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - tags: - - osx - -#### stage: build - -# PIPELINE_SCRIPTS_TAG can be found in the project variables - .check-dependent-project: &check-dependent-project stage: build <<: *docker-env @@ -487,8 +193,8 @@ cargo-check-macos: script: - git clone --depth=1 - "--branch=$PIPELINE_SCRIPTS_TAG" - https://github.com/paritytech/pipeline-scripts + --branch=crates + https://github.com/joao-paulo-parity/pipeline-scripts - ./pipeline-scripts/check_dependent_project.sh paritytech substrate @@ -512,254 +218,3 @@ check-dependent-cumulus: variables: DEPENDENT_REPO: cumulus CARGO_UPDATE_CRATES: "sp-io polkadot-runtime-common" - - -build-linux-substrate: - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: test-linux-stable - artifacts: false - before_script: - - mkdir -p ./artifacts/substrate/ - script: - - *build-linux-substrate-script - - printf '\n# building node-template\n\n' - - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - -build-linux-subkey: &build-subkey - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: cargo-check-subkey - artifacts: false - before_script: - - mkdir -p ./artifacts/subkey - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo build --release --verbose - - cd - - - mv ./target/release/subkey ./artifacts/subkey/. - - echo -n "Subkey version = " - - ./artifacts/subkey/subkey --version | - sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | - tee ./artifacts/subkey/VERSION; - - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/ - - sccache -s - -build-macos-subkey: - <<: *build-subkey - tags: - - osx - -check-rustdoc: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - SKIP_WASM_BUILD: 1 - RUSTDOCFLAGS: "-Dwarnings" - script: - - time cargo +nightly doc --workspace --all-features --verbose --no-deps - - sccache -s - -build-rustdoc: - stage: build - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - SKIP_WASM_BUILD: 1 - DOC_INDEX_PAGE: "sc_service/index.html" # default redirected page - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 7 days - paths: - - ./crate-docs/ - script: - - time cargo +nightly doc --workspace --all-features --verbose - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - - sccache -s - -#### stage: publish - -.build-push-docker-image: &build-push-docker-image - <<: *build-refs - <<: *kubernetes-env - image: quay.io/buildah/stable - variables: &docker-build-vars - <<: *default-vars - GIT_STRATEGY: none - DOCKERFILE: $PRODUCT.Dockerfile - IMAGE_NAME: docker.io/parity/$PRODUCT - before_script: - - cd ./artifacts/$PRODUCT/ - - VERSION="$(cat ./VERSION)" - - echo "${PRODUCT} version = ${VERSION}" - - test -z "${VERSION}" && exit 1 - script: - - test "$DOCKER_HUB_USER" -a "$DOCKER_HUB_PASS" || - ( echo "no docker credentials provided"; exit 1 ) - - buildah bud - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:latest" - --file "$DOCKERFILE" . - - echo "$DOCKER_HUB_PASS" | - buildah login --username "$DOCKER_HUB_USER" --password-stdin docker.io - - buildah info - - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - - buildah push --format=v2s2 "$IMAGE_NAME:latest" - after_script: - - buildah logout --all - - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env - - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)" - - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env - - cat ./artifacts/$PRODUCT/build.env - -publish-docker-substrate: - stage: publish - <<: *build-push-docker-image - <<: *build-refs - needs: - - job: build-linux-substrate - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: substrate - -publish-docker-subkey: - stage: publish - <<: *build-push-docker-image - needs: - - job: build-linux-subkey - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: subkey - -publish-s3-release: - stage: publish - <<: *build-refs - <<: *kubernetes-env - needs: - - job: build-linux-substrate - artifacts: true - - job: build-linux-subkey - artifacts: true - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate/${ARCH}-${DOCKER_OS}" - script: - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - - echo "update objects in latest path" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - -publish-rustdoc: - stage: publish - <<: *kubernetes-env - image: node:16 - variables: - GIT_DEPTH: 100 - RUSTDOCS_DEPLOY_REFS: "master" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/ # to support: monthly-2021-09+1 - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other - # artifacts from being dowloaded by this job. - needs: - - job: build-rustdoc - artifacts: true - script: - # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we - # exit immediately. - # Putting spaces at the front and back to ensure we are not matching just any substring, but the - # whole space-separated value. - - '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0' - # setup ssh - - eval $(ssh-agent) - - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} - - mkdir ~/.ssh && touch ~/.ssh/known_hosts - - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - # Set git config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp README.md /tmp/doc/ - # we don't need to commit changes because we copy docs to /tmp - - git checkout gh-pages --force - # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS - - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud - - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME} - # Ensure the destination dir doesn't exist. - - rm -rf ${CI_COMMIT_REF_NAME} - - mv -f /tmp/doc ${CI_COMMIT_REF_NAME} - # Upload files - - git add --all - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - after_script: - - rm -rf .git/ ./* - -publish-draft-release: - stage: publish - image: paritytech/tools:latest - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./scripts/ci/gitlab/publish_draft_release.sh - allow_failure: true - -#### stage: deploy - -deploy-prometheus-alerting-rules: - stage: deploy - needs: - - job: test-prometheus-alerting-rules - artifacts: false - allow_failure: true - trigger: - project: parity/infrastructure/cloud-infra - variables: - SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" - UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_REF_NAME == "master" - changes: - - .gitlab-ci.yml - - ./scripts/ci/monitoring/**/* diff --git a/Cargo.lock b/Cargo.lock index 609ccb0baeef9..f46fa5fadc6b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "sp-finality-grandpa", "sp-keyring", "sp-keystore", + "sp-mmr-primitives", "sp-runtime", "sp-tracing", "strum", @@ -5664,7 +5665,6 @@ dependencies = [ "log 0.4.14", "pallet-beefy", "pallet-mmr", - "pallet-mmr-primitives", "pallet-session", "parity-scale-codec", "scale-info", @@ -6093,27 +6093,11 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "pallet-mmr-primitives", "parity-scale-codec", "scale-info", "sp-core", "sp-io", - "sp-runtime", - "sp-std", -] - -[[package]] -name = "pallet-mmr-primitives" -version = "4.0.0-dev" -dependencies = [ - "frame-support", - "frame-system", - "hex-literal", - "log 0.4.14", - "parity-scale-codec", - "serde", - "sp-api", - "sp-core", + "sp-mmr-primitives", "sp-runtime", "sp-std", ] @@ -6125,13 +6109,13 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "pallet-mmr-primitives", "parity-scale-codec", "serde", "serde_json", "sp-api", "sp-blockchain", "sp-core", + "sp-mmr-primitives", "sp-runtime", ] @@ -9106,7 +9090,7 @@ dependencies = [ name = "sc-sysinfo" version = "6.0.0-dev" dependencies = [ - "futures 0.3.19", + "futures 0.3.21", "libc", "log 0.4.14", "rand 0.7.3", @@ -10167,6 +10151,21 @@ dependencies = [ "zstd", ] +[[package]] +name = "sp-mmr-primitives" +version = "4.0.0-dev" +dependencies = [ + "hex-literal", + "log 0.4.14", + "parity-scale-codec", + "serde", + "sp-api", + "sp-core", + "sp-debug-derive", + "sp-runtime", + "sp-std", +] + [[package]] name = "sp-npos-elections" version = "4.0.0-dev" diff --git a/Cargo.toml b/Cargo.toml index 576bae6b574b2..45df7258ef253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,6 @@ members = [ "frame/lottery", "frame/membership", "frame/merkle-mountain-range", - "frame/merkle-mountain-range/primitives", "frame/merkle-mountain-range/rpc", "frame/multisig", "frame/nicks", @@ -172,6 +171,7 @@ members = [ "primitives/keyring", "primitives/keystore", "primitives/maybe-compressed-blob", + "primitives/merkle-mountain-range", "primitives/npos-elections", "primitives/npos-elections/fuzzer", "primitives/offchain", diff --git a/README.md b/README.md index b716794428a00..99f34b6f91536 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,4 @@ The reason for the split-licensing is to ensure that for the vast majority of te In the interests of the community, we require any deeper improvements made to Substrate's core logic (e.g. Substrate's internal consensus, crypto or database code) to be contributed back so everyone can benefit. + diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 57584bed39b2c..f422d1ffc45b8 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1261,7 +1261,7 @@ impl pallet_mmr::Config for Runtime { const INDEXING_PREFIX: &'static [u8] = b"mmr"; type Hashing = ::Hashing; type Hash = ::Hash; - type LeafData = frame_system::Pallet; + type LeafData = pallet_mmr::ParentNumberAndHash; type OnNewRoot = (); type WeightInfo = (); } @@ -1804,6 +1804,10 @@ impl_runtime_apis! { let node = mmr::DataOrHash::Data(leaf.into_opaque_leaf()); pallet_mmr::verify_leaf_proof::(root, node, proof) } + + fn mmr_root() -> Result { + Ok(Mmr::mmr_root()) + } } impl sp_session::SessionKeys for Runtime { diff --git a/client/beefy/Cargo.toml b/client/beefy/Cargo.toml index 02be645b3fc08..96248249200af 100644 --- a/client/beefy/Cargo.toml +++ b/client/beefy/Cargo.toml @@ -27,6 +27,7 @@ sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } +sp-mmr-primitives = { version = "4.0.0-dev", path = "../../primitives/merkle-mountain-range" } sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" } diff --git a/client/beefy/src/lib.rs b/client/beefy/src/lib.rs index 8a6e175f58321..5d8aa5c866c4a 100644 --- a/client/beefy/src/lib.rs +++ b/client/beefy/src/lib.rs @@ -27,9 +27,10 @@ use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; use sp_consensus::SyncOracle; use sp_keystore::SyncCryptoStorePtr; +use sp_mmr_primitives::MmrApi; use sp_runtime::traits::Block; -use beefy_primitives::BeefyApi; +use beefy_primitives::{BeefyApi, MmrRootHash}; use crate::notification::{BeefyBestBlockSender, BeefySignedCommitmentSender}; @@ -87,7 +88,7 @@ pub fn beefy_peers_set_config( /// of today, Rust does not allow a type alias to be used as a trait bound. Tracking /// issue is . pub trait Client: - BlockchainEvents + HeaderBackend + Finalizer + ProvideRuntimeApi + Send + Sync + BlockchainEvents + HeaderBackend + Finalizer + Send + Sync where B: Block, BE: Backend, @@ -110,18 +111,21 @@ where } /// BEEFY gadget initialization parameters. -pub struct BeefyParams +pub struct BeefyParams where B: Block, BE: Backend, C: Client, - C::Api: BeefyApi, + R: ProvideRuntimeApi, + R::Api: BeefyApi + MmrApi, N: GossipNetwork + Clone + SyncOracle + Send + Sync + 'static, { /// BEEFY client pub client: Arc, /// Client Backend pub backend: Arc, + /// Runtime Api Provider + pub runtime: Arc, /// Local key store pub key_store: Option, /// Gossip network @@ -138,21 +142,22 @@ where pub protocol_name: std::borrow::Cow<'static, str>, } -#[cfg(not(test))] /// Start the BEEFY gadget. /// /// This is a thin shim around running and awaiting a BEEFY worker. -pub async fn start_beefy_gadget(beefy_params: BeefyParams) +pub async fn start_beefy_gadget(beefy_params: BeefyParams) where B: Block, BE: Backend, C: Client, - C::Api: BeefyApi, + R: ProvideRuntimeApi, + R::Api: BeefyApi + MmrApi, N: GossipNetwork + Clone + SyncOracle + Send + Sync + 'static, { let BeefyParams { client, backend, + runtime, key_store, network, signed_commitment_sender, @@ -188,6 +193,7 @@ where let worker_params = worker::WorkerParams { client, backend, + runtime, key_store: key_store.into(), signed_commitment_sender, beefy_best_block_sender, @@ -198,7 +204,7 @@ where sync_oracle, }; - let worker = worker::BeefyWorker::<_, _, _, _>::new(worker_params); + let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params); worker.run().await } diff --git a/client/beefy/src/metrics.rs b/client/beefy/src/metrics.rs index 20fa98e52fdd5..a6d29dbf88abb 100644 --- a/client/beefy/src/metrics.rs +++ b/client/beefy/src/metrics.rs @@ -18,9 +18,7 @@ //! BEEFY Prometheus metrics definition -#[cfg(not(test))] -use prometheus::{register, PrometheusError, Registry}; -use prometheus::{Counter, Gauge, U64}; +use prometheus::{register, Counter, Gauge, PrometheusError, Registry, U64}; /// BEEFY metrics exposed through Prometheus pub(crate) struct Metrics { @@ -39,7 +37,6 @@ pub(crate) struct Metrics { } impl Metrics { - #[cfg(not(test))] pub(crate) fn register(registry: &Registry) -> Result { Ok(Self { beefy_validator_set_id: register( diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs index 92b5ad91c11e1..e568daba8e112 100644 --- a/client/beefy/src/tests.rs +++ b/client/beefy/src/tests.rs @@ -28,18 +28,20 @@ use sc_chain_spec::{ChainSpec, GenericChainSpec}; use sc_client_api::HeaderBackend; use sc_consensus::BoxJustificationImport; use sc_keystore::LocalKeystore; -use sc_network::{config::ProtocolConfig, NetworkService}; -use sc_network_gossip::GossipEngine; +use sc_network::config::ProtocolConfig; use sc_network_test::{ Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient, - PeersFullClient, TestNetFactory, + TestNetFactory, }; use sc_utils::notification::NotificationReceiver; use beefy_primitives::{ - crypto::AuthorityId, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID, + crypto::AuthorityId, BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID, KEY_TYPE as BeefyKeyType, }; +use sp_mmr_primitives::{EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof}; + +use sp_api::{ApiRef, ProvideRuntimeApi}; use sp_consensus::BlockOrigin; use sp_core::H256; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; @@ -47,19 +49,16 @@ use sp_runtime::{ codec::Encode, generic::BlockId, traits::Header as HeaderT, BuildStorage, DigestItem, Storage, }; -use substrate_test_runtime_client::{runtime::Header, Backend, ClientExt}; +use substrate_test_runtime_client::{runtime::Header, ClientExt}; -use crate::{ - beefy_protocol_name, - keystore::tests::Keyring as BeefyKeyring, - notification::*, - worker::{tests::TestModifiers, BeefyWorker}, -}; +use crate::{beefy_protocol_name, keystore::tests::Keyring as BeefyKeyring, notification::*}; -const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1"; +pub(crate) const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1"; +const GOOD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0xbf); +const BAD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0x42); -type BeefyValidatorSet = ValidatorSet; -type BeefyPeer = Peer; +pub(crate) type BeefyValidatorSet = ValidatorSet; +pub(crate) type BeefyPeer = Peer; #[derive(Debug, Serialize, Deserialize)] struct Genesis(std::collections::BTreeMap); @@ -101,27 +100,13 @@ fn beefy_protocol_name() { #[allow(dead_code)] #[derive(Clone)] pub(crate) struct BeefyLinkHalf { - signed_commitment_stream: BeefySignedCommitmentStream, - beefy_best_block_stream: BeefyBestBlockStream, + pub signed_commitment_stream: BeefySignedCommitmentStream, + pub beefy_best_block_stream: BeefyBestBlockStream, } #[derive(Default)] pub(crate) struct PeerData { pub(crate) beefy_link_half: Mutex>, - pub(crate) test_modifiers: Option, -} - -impl PeerData { - pub(crate) fn use_validator_set(&mut self, validator_set: &ValidatorSet) { - if let Some(tm) = self.test_modifiers.as_mut() { - tm.active_validators = validator_set.clone(); - } else { - self.test_modifiers = Some(TestModifiers { - active_validators: validator_set.clone(), - corrupt_mmr_roots: false, - }); - } - } } pub(crate) struct BeefyTestNet { @@ -153,17 +138,19 @@ impl BeefyTestNet { count: usize, session_length: u64, validator_set: &BeefyValidatorSet, + include_mmr_digest: bool, ) { self.peer(0).generate_blocks(count, BlockOrigin::File, |builder| { let mut block = builder.build().unwrap().block; - let block_num = *block.header.number(); - let num_byte = block_num.to_le_bytes().into_iter().next().unwrap(); - let mmr_root = MmrRootHash::repeat_byte(num_byte); - - add_mmr_digest(&mut block.header, mmr_root); + if include_mmr_digest { + let block_num = *block.header.number(); + let num_byte = block_num.to_le_bytes().into_iter().next().unwrap(); + let mmr_root = MmrRootHash::repeat_byte(num_byte); + add_mmr_digest(&mut block.header, mmr_root); + } - if block_num % session_length == 0 { + if *block.header.number() % session_length == 0 { add_auth_change_digest(&mut block.header, validator_set.clone()); } @@ -223,6 +210,79 @@ impl TestNetFactory for BeefyTestNet { } } +macro_rules! create_test_api { + ( $api_name:ident, mmr_root: $mmr_root:expr, $($inits:expr),+ ) => { + pub(crate) mod $api_name { + use super::*; + + #[derive(Clone, Default)] + pub(crate) struct TestApi {} + + // compiler gets confused and warns us about unused inner + #[allow(dead_code)] + pub(crate) struct RuntimeApi { + inner: TestApi, + } + + impl ProvideRuntimeApi for TestApi { + type Api = RuntimeApi; + fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> { + RuntimeApi { inner: self.clone() }.into() + } + } + sp_api::mock_impl_runtime_apis! { + impl BeefyApi for RuntimeApi { + fn validator_set() -> Option { + BeefyValidatorSet::new(make_beefy_ids(&[$($inits),+]), 0) + } + } + + impl MmrApi for RuntimeApi { + fn generate_proof(_leaf_index: LeafIndex) + -> Result<(EncodableOpaqueLeaf, Proof), MmrError> { + unimplemented!() + } + + fn verify_proof(_leaf: EncodableOpaqueLeaf, _proof: Proof) + -> Result<(), MmrError> { + unimplemented!() + } + + fn verify_proof_stateless( + _root: MmrRootHash, + _leaf: EncodableOpaqueLeaf, + _proof: Proof + ) -> Result<(), MmrError> { + unimplemented!() + } + + fn mmr_root() -> Result { + Ok($mmr_root) + } + } + } + } + }; +} + +create_test_api!(two_validators, mmr_root: GOOD_MMR_ROOT, BeefyKeyring::Alice, BeefyKeyring::Bob); +create_test_api!( + four_validators, + mmr_root: GOOD_MMR_ROOT, + BeefyKeyring::Alice, + BeefyKeyring::Bob, + BeefyKeyring::Charlie, + BeefyKeyring::Dave +); +create_test_api!( + bad_four_validators, + mmr_root: BAD_MMR_ROOT, + BeefyKeyring::Alice, + BeefyKeyring::Bob, + BeefyKeyring::Charlie, + BeefyKeyring::Dave +); + fn add_mmr_digest(header: &mut Header, mmr_hash: MmrRootHash) { header.digest_mut().push(DigestItem::Consensus( BEEFY_ENGINE_ID, @@ -248,54 +308,43 @@ pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> SyncCryptoStoreP keystore } -pub(crate) fn create_beefy_worker( - peer: &BeefyPeer, - key: &BeefyKeyring, - min_block_delta: u32, -) -> BeefyWorker>> { - let keystore = create_beefy_keystore(*key); - - let (signed_commitment_sender, signed_commitment_stream) = - BeefySignedCommitmentStream::::channel(); - let (beefy_best_block_sender, beefy_best_block_stream) = - BeefyBestBlockStream::::channel(); - - let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream }; - *peer.data.beefy_link_half.lock() = Some(beefy_link_half); - let test_modifiers = peer.data.test_modifiers.clone().unwrap(); - - let network = peer.network_service().clone(); - let sync_oracle = network.clone(); - let gossip_validator = Arc::new(crate::gossip::GossipValidator::new()); - let gossip_engine = - GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None); - let worker_params = crate::worker::WorkerParams { - client: peer.client().as_client(), - backend: peer.client().as_backend(), - key_store: Some(keystore).into(), - signed_commitment_sender, - beefy_best_block_sender, - gossip_engine, - gossip_validator, - min_block_delta, - metrics: None, - sync_oracle, - }; - - BeefyWorker::<_, _, _, _>::new(worker_params, test_modifiers) -} - // Spawns beefy voters. Returns a future to spawn on the runtime. -fn initialize_beefy( +fn initialize_beefy( net: &mut BeefyTestNet, - peers: &[BeefyKeyring], + peers: Vec<(usize, &BeefyKeyring, Arc)>, min_block_delta: u32, -) -> impl Future { +) -> impl Future +where + API: ProvideRuntimeApi + Default + Sync + Send, + API::Api: BeefyApi + MmrApi, +{ let voters = FuturesUnordered::new(); - for (peer_id, key) in peers.iter().enumerate() { - let worker = create_beefy_worker(&net.peers[peer_id], key, min_block_delta); - let gadget = worker.run(); + for (peer_id, key, api) in peers.into_iter() { + let peer = &net.peers[peer_id]; + + let keystore = create_beefy_keystore(*key); + + let (signed_commitment_sender, signed_commitment_stream) = + BeefySignedCommitmentStream::::channel(); + let (beefy_best_block_sender, beefy_best_block_stream) = + BeefyBestBlockStream::::channel(); + let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream }; + *peer.data.beefy_link_half.lock() = Some(beefy_link_half); + + let beefy_params = crate::BeefyParams { + client: peer.client().as_client(), + backend: peer.client().as_backend(), + runtime: api.clone(), + key_store: Some(keystore), + network: peer.network_service().clone(), + signed_commitment_sender, + beefy_best_block_sender, + min_block_delta, + prometheus_registry: None, + protocol_name: BEEFY_PROTOCOL_NAME.into(), + }; + let gadget = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params); fn assert_send(_: &T) {} assert_send(&gadget); @@ -443,13 +492,12 @@ fn beefy_finalizing_blocks() { let mut net = BeefyTestNet::new(2, 0); - for i in 0..peers.len() { - net.peer(i).data.use_validator_set(&validator_set); - } - runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + let api = Arc::new(two_validators::TestApi {}); + let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect(); + runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta)); // push 42 blocks including `AuthorityChange` digests every 10 blocks. - net.generate_blocks(42, session_len, &validator_set); + net.generate_blocks(42, session_len, &validator_set, true); net.block_until_sync(); let net = Arc::new(Mutex::new(net)); @@ -477,19 +525,18 @@ fn lagging_validators() { sp_tracing::try_init_simple(); let mut runtime = Runtime::new().unwrap(); - let peers = &[BeefyKeyring::Charlie, BeefyKeyring::Dave]; + let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob]; let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap(); let session_len = 30; let min_block_delta = 1; let mut net = BeefyTestNet::new(2, 0); - for i in 0..peers.len() { - net.peer(i).data.use_validator_set(&validator_set); - } - runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + let api = Arc::new(two_validators::TestApi {}); + let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect(); + runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta)); // push 42 blocks including `AuthorityChange` digests every 30 blocks. - net.generate_blocks(42, session_len, &validator_set); + net.generate_blocks(42, session_len, &validator_set, true); net.block_until_sync(); let net = Arc::new(Mutex::new(net)); @@ -498,7 +545,7 @@ fn lagging_validators() { // diff-power-of-two rule. finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[15], &[1, 9, 13, 14, 15]); - // Charlie finalizes #25, Dave lags behind + // Alice finalizes #25, Bob lags behind let finalize = BlockId::number(25); let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap(); @@ -507,7 +554,7 @@ fn lagging_validators() { streams_empty_after_timeout(best_blocks, &net, &mut runtime, timeout); streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None); - // Dave catches up and also finalizes #25 + // Bob catches up and also finalizes #25 let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap(); // expected beefy finalizes block #17 from diff-power-of-two @@ -530,16 +577,23 @@ fn correct_beefy_payload() { let min_block_delta = 2; let mut net = BeefyTestNet::new(4, 0); - for i in 0..peers.len() { - net.peer(i).data.use_validator_set(&validator_set); - } + + // Alice, Bob, Charlie will vote on good payloads + let good_api = Arc::new(four_validators::TestApi {}); + let good_peers = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie] + .iter() + .enumerate() + .map(|(id, key)| (id, key, good_api.clone())) + .collect(); + runtime.spawn(initialize_beefy(&mut net, good_peers, min_block_delta)); // Dave will vote on bad mmr roots - net.peer(3).data.test_modifiers.as_mut().map(|tm| tm.corrupt_mmr_roots = true); - runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + let bad_api = Arc::new(bad_four_validators::TestApi {}); + let bad_peers = vec![(3, &BeefyKeyring::Dave, bad_api)]; + runtime.spawn(initialize_beefy(&mut net, bad_peers, min_block_delta)); // push 10 blocks - net.generate_blocks(12, session_len, &validator_set); + net.generate_blocks(12, session_len, &validator_set, false); net.block_until_sync(); let net = Arc::new(Mutex::new(net)); diff --git a/client/beefy/src/worker.rs b/client/beefy/src/worker.rs index 85674c09a278b..128850a5f172f 100644 --- a/client/beefy/src/worker.rs +++ b/client/beefy/src/worker.rs @@ -26,9 +26,10 @@ use parking_lot::Mutex; use sc_client_api::{Backend, FinalityNotification, FinalityNotifications}; use sc_network_gossip::GossipEngine; -use sp_api::BlockId; +use sp_api::{BlockId, ProvideRuntimeApi}; use sp_arithmetic::traits::AtLeast32Bit; use sp_consensus::SyncOracle; +use sp_mmr_primitives::MmrApi; use sp_runtime::{ generic::OpaqueDigestItemId, traits::{Block, Header, NumberFor}, @@ -52,12 +53,10 @@ use crate::{ Client, }; -pub(crate) struct WorkerParams -where - B: Block, -{ +pub(crate) struct WorkerParams { pub client: Arc, pub backend: Arc, + pub runtime: Arc, pub key_store: BeefyKeystore, pub signed_commitment_sender: BeefySignedCommitmentSender, pub beefy_best_block_sender: BeefyBestBlockSender, @@ -69,15 +68,10 @@ where } /// A BEEFY worker plays the BEEFY protocol -pub(crate) struct BeefyWorker -where - B: Block, - BE: Backend, - C: Client, - SO: SyncOracle + Send + Sync + Clone + 'static, -{ +pub(crate) struct BeefyWorker { client: Arc, backend: Arc, + runtime: Arc, key_store: BeefyKeystore, signed_commitment_sender: BeefySignedCommitmentSender, gossip_engine: Arc>>, @@ -99,17 +93,15 @@ where sync_oracle: SO, // keep rustc happy _backend: PhantomData, - #[cfg(test)] - // behavior modifiers used in tests - test_res: tests::TestModifiers, } -impl BeefyWorker +impl BeefyWorker where B: Block + Codec, BE: Backend, C: Client, - C::Api: BeefyApi, + R: ProvideRuntimeApi, + R::Api: BeefyApi + MmrApi, SO: SyncOracle + Send + Sync + Clone + 'static, { /// Return a new BEEFY worker instance. @@ -118,15 +110,11 @@ where /// BEEFY pallet has been deployed on-chain. /// /// The BEEFY pallet is needed in order to keep track of the BEEFY authority set. - pub(crate) fn new( - worker_params: WorkerParams, - #[cfg(test)] - // behavior modifiers used in tests - test_res: tests::TestModifiers, - ) -> Self { + pub(crate) fn new(worker_params: WorkerParams) -> Self { let WorkerParams { client, backend, + runtime, key_store, signed_commitment_sender, beefy_best_block_sender, @@ -144,6 +132,7 @@ where BeefyWorker { client: client.clone(), backend, + runtime, key_store, signed_commitment_sender, gossip_engine: Arc::new(Mutex::new(gossip_engine)), @@ -159,20 +148,9 @@ where beefy_best_block_sender, sync_oracle, _backend: PhantomData, - #[cfg(test)] - test_res, } } -} -impl BeefyWorker -where - B: Block, - BE: Backend, - C: Client, - C::Api: BeefyApi, - SO: SyncOracle + Send + Sync + Clone + 'static, -{ /// Return `Some(number)` if we should be voting on block `number` now, /// return `None` if there is no block we should vote on now. fn current_vote_target(&self) -> Option> { @@ -396,7 +374,7 @@ where }; let target_hash = target_header.hash(); - let mmr_root = if let Some(hash) = self.extract_mmr_root_digest(&target_header) { + let mmr_root = if let Some(hash) = self.get_mmr_root_digest(&target_header) { hash } else { warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash); @@ -458,13 +436,12 @@ where } /// Wait for BEEFY runtime pallet to be available. - #[cfg(not(test))] async fn wait_for_runtime_pallet(&mut self) { self.client .finality_notification_stream() .take_while(|notif| { let at = BlockId::hash(notif.header.hash()); - if let Some(active) = self.client.runtime_api().validator_set(&at).ok().flatten() { + if let Some(active) = self.runtime.runtime_api().validator_set(&at).ok().flatten() { if active.id() == GENESIS_AUTHORITY_SET_ID { // When starting from genesis, there is no session boundary digest. // Just initialize `rounds` to Block #1 as BEEFY mandatory block. @@ -488,13 +465,6 @@ where self.finality_notifications = self.client.finality_notification_stream(); } - /// For tests don't use runtime pallet. Start rounds from block #1. - #[cfg(test)] - async fn wait_for_runtime_pallet(&mut self) { - let active = self.test_res.active_validators.clone(); - self.init_session_at(active, 1u32.into()); - } - /// Main loop for BEEFY worker. /// /// Wait for BEEFY runtime pallet to be available, then start the main async loop @@ -550,20 +520,15 @@ where } } - /// Simple wrapper over mmr root extraction. - #[cfg(not(test))] - fn extract_mmr_root_digest(&self, header: &B::Header) -> Option { - find_mmr_root_digest::(header) - } - - /// For tests, have the option to modify mmr root. - #[cfg(test)] - fn extract_mmr_root_digest(&self, header: &B::Header) -> Option { - let mut mmr_root = find_mmr_root_digest::(header); - if self.test_res.corrupt_mmr_roots { - mmr_root.as_mut().map(|hash| *hash ^= MmrRootHash::random()); - } - mmr_root + /// Simple wrapper that gets MMR root from header digests or from client state. + fn get_mmr_root_digest(&self, header: &B::Header) -> Option { + find_mmr_root_digest::(header).or_else(|| { + self.runtime + .runtime_api() + .mmr_root(&BlockId::hash(header.hash())) + .ok() + .and_then(|r| r.ok()) + }) } } @@ -658,11 +623,16 @@ pub(crate) mod tests { use super::*; use crate::{ keystore::tests::Keyring, - tests::{create_beefy_worker, get_beefy_streams, make_beefy_ids, BeefyTestNet}, + notification::{BeefyBestBlockStream, BeefySignedCommitmentStream}, + tests::{ + create_beefy_keystore, get_beefy_streams, make_beefy_ids, two_validators::TestApi, + BeefyPeer, BeefyTestNet, BEEFY_PROTOCOL_NAME, + }, }; use futures::{executor::block_on, future::poll_fn, task::Poll}; + use crate::tests::BeefyLinkHalf; use sc_client_api::HeaderBackend; use sc_network::NetworkService; use sc_network_test::{PeersFullClient, TestNetFactory}; @@ -672,10 +642,40 @@ pub(crate) mod tests { Backend, }; - #[derive(Clone)] - pub struct TestModifiers { - pub active_validators: ValidatorSet, - pub corrupt_mmr_roots: bool, + fn create_beefy_worker( + peer: &BeefyPeer, + key: &Keyring, + min_block_delta: u32, + ) -> BeefyWorker>> { + let keystore = create_beefy_keystore(*key); + + let (signed_commitment_sender, signed_commitment_stream) = + BeefySignedCommitmentStream::::channel(); + let (beefy_best_block_sender, beefy_best_block_stream) = + BeefyBestBlockStream::::channel(); + let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream }; + *peer.data.beefy_link_half.lock() = Some(beefy_link_half); + + let api = Arc::new(TestApi {}); + let network = peer.network_service().clone(); + let sync_oracle = network.clone(); + let gossip_validator = Arc::new(crate::gossip::GossipValidator::new()); + let gossip_engine = + GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None); + let worker_params = crate::worker::WorkerParams { + client: peer.client().as_client(), + backend: peer.client().as_backend(), + runtime: api, + key_store: Some(keystore).into(), + signed_commitment_sender, + beefy_best_block_sender, + gossip_engine, + gossip_validator, + min_block_delta, + metrics: None, + sync_oracle, + }; + BeefyWorker::<_, _, _, _, _>::new(worker_params) } #[test] @@ -825,7 +825,6 @@ pub(crate) mod tests { let keys = &[Keyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); let mut net = BeefyTestNet::new(1, 0); - net.peer(0).data.use_validator_set(&validator_set); let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); // rounds not initialized -> should vote: `None` @@ -833,8 +832,9 @@ pub(crate) mod tests { let set_up = |worker: &mut BeefyWorker< Block, - PeersFullClient, Backend, + PeersFullClient, + TestApi, Arc>, >, best_grandpa: u64, @@ -889,7 +889,6 @@ pub(crate) mod tests { let keys = &[Keyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); let mut net = BeefyTestNet::new(1, 0); - net.peer(0).data.use_validator_set(&validator_set); let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); // keystore doesn't contain other keys than validators' @@ -913,7 +912,6 @@ pub(crate) mod tests { let keys = &[Keyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); let mut net = BeefyTestNet::new(1, 0); - net.peer(0).data.use_validator_set(&validator_set); let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys); @@ -939,7 +937,7 @@ pub(crate) mod tests { // generate 2 blocks, try again expect success let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys); let mut best_block_stream = best_block_streams.drain(..).next().unwrap(); - net.generate_blocks(2, 10, &validator_set); + net.generate_blocks(2, 10, &validator_set, false); worker.set_best_beefy_block(2); assert_eq!(worker.best_beefy_block, Some(2)); @@ -961,7 +959,6 @@ pub(crate) mod tests { let keys = &[Keyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); let mut net = BeefyTestNet::new(1, 0); - net.peer(0).data.use_validator_set(&validator_set); let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); assert!(worker.rounds.is_none()); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 6a8a025f1f45f..4f504651f952c 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -106,6 +106,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = sp_state_machine::TrieBackend>>, HashFor>; +#[cfg(feature = "with-parity-db")] /// Length of a [`DbHash`]. const DB_HASH_LEN: usize = 32; diff --git a/frame/beefy-mmr/Cargo.toml b/frame/beefy-mmr/Cargo.toml index 19703fa79863a..a32b1dc6dbd64 100644 --- a/frame/beefy-mmr/Cargo.toml +++ b/frame/beefy-mmr/Cargo.toml @@ -18,7 +18,6 @@ serde = { version = "1.0.136", optional = true } frame-support = { version = "4.0.0-dev", path = "../support", default-features = false } frame-system = { version = "4.0.0-dev", path = "../system", default-features = false } pallet-mmr = { version = "4.0.0-dev", path = "../merkle-mountain-range", default-features = false } -pallet-mmr-primitives = { version = "4.0.0-dev", path = "../merkle-mountain-range/primitives", default-features = false } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } @@ -46,7 +45,6 @@ std = [ "k256/std", "log/std", "pallet-beefy/std", - "pallet-mmr-primitives/std", "pallet-mmr/std", "pallet-session/std", "serde", diff --git a/frame/beefy-mmr/src/lib.rs b/frame/beefy-mmr/src/lib.rs index 9ee7bd770f64c..640ebeb7d49cc 100644 --- a/frame/beefy-mmr/src/lib.rs +++ b/frame/beefy-mmr/src/lib.rs @@ -37,7 +37,7 @@ use sp_runtime::traits::{Convert, Hash, Member}; use sp_std::prelude::*; use beefy_primitives::mmr::{BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion}; -use pallet_mmr::primitives::LeafDataProvider; +use pallet_mmr::{LeafDataProvider, ParentNumberAndHash}; use frame_support::traits::Get; @@ -150,7 +150,7 @@ where fn leaf_data() -> Self::LeafData { MmrLeaf { version: T::LeafVersion::get(), - parent_number_and_hash: frame_system::Pallet::::leaf_data(), + parent_number_and_hash: ParentNumberAndHash::::leaf_data(), leaf_extra: T::BeefyDataProvider::extra_data(), beefy_next_authority_set: Pallet::::update_beefy_next_authority_set(), } diff --git a/frame/beefy-mmr/src/tests.rs b/frame/beefy-mmr/src/tests.rs index fd383adb1d4a2..37f571cd842ee 100644 --- a/frame/beefy-mmr/src/tests.rs +++ b/frame/beefy-mmr/src/tests.rs @@ -49,7 +49,7 @@ fn offchain_key(pos: usize) -> Vec { } fn read_mmr_leaf(ext: &mut TestExternalities, index: usize) -> MmrLeaf { - type Node = pallet_mmr_primitives::DataOrHash; + type Node = pallet_mmr::primitives::DataOrHash; ext.persist_offchain_overlay(); let offchain_db = ext.offchain_db(); offchain_db diff --git a/frame/merkle-mountain-range/Cargo.toml b/frame/merkle-mountain-range/Cargo.toml index 796ab98dc2c32..8b67174e8385d 100644 --- a/frame/merkle-mountain-range/Cargo.toml +++ b/frame/merkle-mountain-range/Cargo.toml @@ -18,6 +18,7 @@ mmr-lib = { package = "ckb-merkle-mountain-range", default-features = false, ver sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/merkle-mountain-range" } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } @@ -25,8 +26,6 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = " frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -pallet-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "./primitives" } - [dev-dependencies] env_logger = "0.9" hex-literal = "0.3" @@ -39,12 +38,12 @@ std = [ "mmr-lib/std", "sp-core/std", "sp-io/std", + "sp-mmr-primitives/std", "sp-runtime/std", "sp-std/std", "frame-benchmarking/std", "frame-support/std", "frame-system/std", - "pallet-mmr-primitives/std", ] runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/merkle-mountain-range/rpc/Cargo.toml b/frame/merkle-mountain-range/rpc/Cargo.toml index 9ac26c2ed54b2..94c895ea91517 100644 --- a/frame/merkle-mountain-range/rpc/Cargo.toml +++ b/frame/merkle-mountain-range/rpc/Cargo.toml @@ -22,9 +22,8 @@ serde = { version = "1.0.136", features = ["derive"] } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/merkle-mountain-range" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } -pallet-mmr-primitives = { version = "4.0.0-dev", path = "../primitives" } - [dev-dependencies] serde_json = "1.0.79" diff --git a/frame/merkle-mountain-range/rpc/src/lib.rs b/frame/merkle-mountain-range/rpc/src/lib.rs index bf3eb3b694e39..99359bfea8eb6 100644 --- a/frame/merkle-mountain-range/rpc/src/lib.rs +++ b/frame/merkle-mountain-range/rpc/src/lib.rs @@ -26,13 +26,13 @@ use jsonrpc_core::{Error, ErrorCode, Result}; use jsonrpc_derive::rpc; use serde::{Deserialize, Serialize}; -use pallet_mmr_primitives::{Error as MmrError, Proof}; use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; use sp_core::Bytes; +use sp_mmr_primitives::{Error as MmrError, LeafIndex, Proof}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; -pub use pallet_mmr_primitives::{LeafIndex, MmrApi as MmrRuntimeApi}; +pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi; /// Retrieved MMR leaf and its proof. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -42,7 +42,7 @@ pub struct LeafProof { pub block_hash: BlockHash, /// SCALE-encoded leaf data. pub leaf: Bytes, - /// SCALE-encoded proof data. See [pallet_mmr_primitives::Proof]. + /// SCALE-encoded proof data. See [sp_mmr_primitives::Proof]. pub proof: Bytes, } diff --git a/frame/merkle-mountain-range/src/lib.rs b/frame/merkle-mountain-range/src/lib.rs index f904428e02048..855eb0a7436dc 100644 --- a/frame/merkle-mountain-range/src/lib.rs +++ b/frame/merkle-mountain-range/src/lib.rs @@ -58,7 +58,7 @@ use codec::Encode; use frame_support::weights::Weight; -use sp_runtime::traits; +use sp_runtime::traits::{self, One, Saturating}; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarking; @@ -70,7 +70,30 @@ mod mock; mod tests; pub use pallet::*; -pub use pallet_mmr_primitives::{self as primitives, NodeIndex}; +pub use sp_mmr_primitives::{self as primitives, Error, LeafDataProvider, LeafIndex, NodeIndex}; + +/// The most common use case for MMRs is to store historical block hashes, +/// so that any point in time in the future we can receive a proof about some past +/// blocks without using excessive on-chain storage. +/// +/// Hence we implement the [LeafDataProvider] for [ParentNumberAndHash] which is a +/// crate-local wrapper over [frame_system::Pallet]. Since the current block hash +/// is not available (since the block is not finished yet), +/// we use the `parent_hash` here along with parent block number. +pub struct ParentNumberAndHash { + _phanthom: sp_std::marker::PhantomData, +} + +impl LeafDataProvider for ParentNumberAndHash { + type LeafData = (::BlockNumber, ::Hash); + + fn leaf_data() -> Self::LeafData { + ( + frame_system::Pallet::::block_number().saturating_sub(One::one()), + frame_system::Pallet::::parent_hash(), + ) + } +} pub trait WeightInfo { fn on_initialize(peaks: NodeIndex) -> Weight; @@ -161,7 +184,7 @@ pub mod pallet { /// Current size of the MMR (number of leaves). #[pallet::storage] #[pallet::getter(fn mmr_leaves)] - pub type NumberOfLeaves = StorageValue<_, NodeIndex, ValueQuery>; + pub type NumberOfLeaves = StorageValue<_, LeafIndex, ValueQuery>; /// Hashes of the nodes in the MMR. /// @@ -240,7 +263,7 @@ impl, I: 'static> Pallet { /// all the leaves to be present. /// It may return an error or panic if used incorrectly. pub fn generate_proof( - leaf_index: NodeIndex, + leaf_index: LeafIndex, ) -> Result<(LeafOf, primitives::Proof<>::Hash>), primitives::Error> { let mmr: ModuleMmr = mmr::Mmr::new(Self::mmr_leaves()); mmr.generate_proof(leaf_index) @@ -272,4 +295,9 @@ impl, I: 'static> Pallet { Err(primitives::Error::Verify.log_debug("The proof is incorrect.")) } } + + /// Return the on-chain MMR root hash. + pub fn mmr_root() -> >::Hash { + Self::mmr_root_hash() + } } diff --git a/frame/merkle-mountain-range/src/mmr/mod.rs b/frame/merkle-mountain-range/src/mmr/mod.rs index 1a729b08b966f..1cb4e8535b991 100644 --- a/frame/merkle-mountain-range/src/mmr/mod.rs +++ b/frame/merkle-mountain-range/src/mmr/mod.rs @@ -19,7 +19,7 @@ mod mmr; pub mod storage; pub mod utils; -use crate::primitives::FullLeaf; +use sp_mmr_primitives::{DataOrHash, FullLeaf}; use sp_runtime::traits; pub use self::mmr::{verify_leaf_proof, Mmr}; @@ -28,7 +28,7 @@ pub use self::mmr::{verify_leaf_proof, Mmr}; pub type NodeOf = Node<>::Hashing, L>; /// A node stored in the MMR. -pub type Node = crate::primitives::DataOrHash; +pub type Node = DataOrHash; /// Default Merging & Hashing behavior for MMR. pub struct Hasher(sp_std::marker::PhantomData<(H, L)>); diff --git a/frame/merkle-mountain-range/src/mock.rs b/frame/merkle-mountain-range/src/mock.rs index 56d3c9c0d77d8..b2b6821fcd054 100644 --- a/frame/merkle-mountain-range/src/mock.rs +++ b/frame/merkle-mountain-range/src/mock.rs @@ -20,8 +20,8 @@ use crate::*; use codec::{Decode, Encode}; use frame_support::traits::{ConstU32, ConstU64}; -use pallet_mmr_primitives::{Compact, LeafDataProvider}; use sp_core::H256; +use sp_mmr_primitives::{Compact, LeafDataProvider}; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup, Keccak256}, @@ -74,7 +74,7 @@ impl Config for Test { type Hashing = Keccak256; type Hash = H256; - type LeafData = Compact, LeafData)>; + type LeafData = Compact, LeafData)>; type OnNewRoot = (); type WeightInfo = (); } diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 576a7ace8f1c0..70d1395aa94d5 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -19,11 +19,11 @@ use crate::{mmr::utils, mock::*, *}; use frame_support::traits::OnInitialize; use mmr_lib::helper; -use pallet_mmr_primitives::{Compact, Proof}; use sp_core::{ offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt}, H256, }; +use sp_mmr_primitives::{Compact, Proof}; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { frame_system::GenesisConfig::default().build_storage::().unwrap().into() diff --git a/frame/merkle-mountain-range/primitives/Cargo.toml b/primitives/merkle-mountain-range/Cargo.toml similarity index 68% rename from frame/merkle-mountain-range/primitives/Cargo.toml rename to primitives/merkle-mountain-range/Cargo.toml index 3ce2caa7762c0..ddb820f9e6cc4 100644 --- a/frame/merkle-mountain-range/primitives/Cargo.toml +++ b/primitives/merkle-mountain-range/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "pallet-mmr-primitives" +name = "sp-mmr-primitives" version = "4.0.0-dev" authors = ["Parity Technologies "] edition = "2021" license = "Apache-2.0" homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" -description = "FRAME Merkle Mountain Range primitives." +description = "Merkle Mountain Range primitives." [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] @@ -16,13 +16,11 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = log = { version = "0.4.14", default-features = false } serde = { version = "1.0.136", optional = true, features = ["derive"] } -sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } -sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } - -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } +sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-debug-derive = { version = "4.0.0", default-features = false, path = "../debug-derive" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } [dev-dependencies] hex-literal = "0.3" @@ -35,8 +33,7 @@ std = [ "serde", "sp-api/std", "sp-core/std", + "sp-debug-derive/std", "sp-runtime/std", "sp-std/std", - "frame-support/std", - "frame-system/std", ] diff --git a/frame/merkle-mountain-range/primitives/src/lib.rs b/primitives/merkle-mountain-range/src/lib.rs similarity index 95% rename from frame/merkle-mountain-range/primitives/src/lib.rs rename to primitives/merkle-mountain-range/src/lib.rs index cc78dfefefe60..a27536b8d35b7 100644 --- a/frame/merkle-mountain-range/primitives/src/lib.rs +++ b/primitives/merkle-mountain-range/src/lib.rs @@ -20,8 +20,8 @@ #![cfg_attr(not(feature = "std"), no_std)] #![warn(missing_docs)] -use frame_support::RuntimeDebug; -use sp_runtime::traits::{self, One, Saturating}; +use sp_debug_derive::RuntimeDebug; +use sp_runtime::traits; use sp_std::fmt; #[cfg(not(feature = "std"))] use sp_std::prelude::Vec; @@ -57,21 +57,6 @@ impl LeafDataProvider for () { } } -/// The most common use case for MMRs is to store historical block hashes, -/// so that any point in time in the future we can receive a proof about some past -/// blocks without using excessive on-chain storage. -/// -/// Hence we implement the [LeafDataProvider] for [frame_system::Pallet]. Since the -/// current block hash is not available (since the block is not finished yet), -/// we use the `parent_hash` here along with parent block number. -impl LeafDataProvider for frame_system::Pallet { - type LeafData = (::BlockNumber, ::Hash); - - fn leaf_data() -> Self::LeafData { - (Self::block_number().saturating_sub(One::one()), Self::parent_hash()) - } -} - /// New MMR root notification hook. pub trait OnNewRoot { /// Function called by the pallet in case new MMR root has been computed. @@ -83,6 +68,17 @@ impl OnNewRoot for () { fn on_new_root(_root: &Hash) {} } +/// A MMR proof data for one of the leaves. +#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)] +pub struct Proof { + /// The index of the leaf the proof is for. + pub leaf_index: LeafIndex, + /// Number of leaves in MMR, when the proof was generated. + pub leaf_count: NodeIndex, + /// Proof elements (hashes of siblings of inner nodes on the path to the leaf). + pub items: Vec, +} + /// A full leaf content stored in the offchain-db. pub trait FullLeaf: Clone + PartialEq + fmt::Debug { /// Encode the leaf either in it's full or compact form. @@ -97,6 +93,80 @@ impl FullLeaf } } +/// A helper type to allow using arbitrary SCALE-encoded leaf data in the RuntimeApi. +/// +/// The point is to be able to verify MMR proofs from external MMRs, where we don't +/// know the exact leaf type, but it's enough for us to have it SCALE-encoded. +/// +/// Note the leaf type should be encoded in its compact form when passed through this type. +/// See [FullLeaf] documentation for details. +/// +/// This type does not implement SCALE encoding/decoding on purpose to avoid confusion, +/// it would have to be SCALE-compatible with the concrete leaf type, but due to SCALE limitations +/// it's not possible to know how many bytes the encoding of concrete leaf type uses. +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[derive(RuntimeDebug, Clone, PartialEq)] +pub struct OpaqueLeaf( + /// Raw bytes of the leaf type encoded in its compact form. + /// + /// NOTE it DOES NOT include length prefix (like `Vec` encoding would). + #[cfg_attr(feature = "std", serde(with = "sp_core::bytes"))] + pub Vec, +); + +impl OpaqueLeaf { + /// Convert a concrete MMR leaf into an opaque type. + pub fn from_leaf(leaf: &T) -> Self { + let encoded_leaf = leaf.using_encoded(|d| d.to_vec(), true); + OpaqueLeaf::from_encoded_leaf(encoded_leaf) + } + + /// Create a `OpaqueLeaf` given raw bytes of compact-encoded leaf. + pub fn from_encoded_leaf(encoded_leaf: Vec) -> Self { + OpaqueLeaf(encoded_leaf) + } + + /// Attempt to decode the leaf into expected concrete type. + pub fn try_decode(&self) -> Option { + codec::Decode::decode(&mut &*self.0).ok() + } +} + +impl FullLeaf for OpaqueLeaf { + fn using_encoded R>(&self, f: F, _compact: bool) -> R { + f(&self.0) + } +} + +/// A type-safe wrapper for the concrete leaf type. +/// +/// This structure serves merely to avoid passing raw `Vec` around. +/// It must be `Vec`-encoding compatible. +/// +/// It is different from [`OpaqueLeaf`], because it does implement `Codec` +/// and the encoding has to match raw `Vec` encoding. +#[derive(codec::Encode, codec::Decode, RuntimeDebug, PartialEq, Eq)] +pub struct EncodableOpaqueLeaf(pub Vec); + +impl EncodableOpaqueLeaf { + /// Convert a concrete leaf into encodable opaque version. + pub fn from_leaf(leaf: &T) -> Self { + let opaque = OpaqueLeaf::from_leaf(leaf); + Self::from_opaque_leaf(opaque) + } + + /// Given an opaque leaf, make it encodable. + pub fn from_opaque_leaf(opaque: OpaqueLeaf) -> Self { + Self(opaque.0) + } + + /// Try to convert into a [OpaqueLeaf]. + pub fn into_opaque_leaf(self) -> OpaqueLeaf { + // wrap into `OpaqueLeaf` type + OpaqueLeaf::from_encoded_leaf(self.0) + } +} + /// An element representing either full data or it's hash. /// /// See [Compact] to see how it may be used in practice to reduce the size @@ -281,17 +351,6 @@ impl_leaf_data_for_tuple!(A:0, B:1, C:2); impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3); impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3, E:4); -/// A MMR proof data for one of the leaves. -#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)] -pub struct Proof { - /// The index of the leaf the proof is for. - pub leaf_index: LeafIndex, - /// Number of leaves in MMR, when the proof was generated. - pub leaf_count: NodeIndex, - /// Proof elements (hashes of siblings of inner nodes on the path to the leaf). - pub items: Vec, -} - /// Merkle Mountain Range operation error. #[derive(RuntimeDebug, codec::Encode, codec::Decode, PartialEq, Eq)] pub enum Error { @@ -334,80 +393,6 @@ impl Error { } } -/// A helper type to allow using arbitrary SCALE-encoded leaf data in the RuntimeApi. -/// -/// The point is to be able to verify MMR proofs from external MMRs, where we don't -/// know the exact leaf type, but it's enough for us to have it SCALE-encoded. -/// -/// Note the leaf type should be encoded in its compact form when passed through this type. -/// See [FullLeaf] documentation for details. -/// -/// This type does not implement SCALE encoding/decoding on purpose to avoid confusion, -/// it would have to be SCALE-compatible with the concrete leaf type, but due to SCALE limitations -/// it's not possible to know how many bytes the encoding of concrete leaf type uses. -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] -#[derive(RuntimeDebug, Clone, PartialEq)] -pub struct OpaqueLeaf( - /// Raw bytes of the leaf type encoded in its compact form. - /// - /// NOTE it DOES NOT include length prefix (like `Vec` encoding would). - #[cfg_attr(feature = "std", serde(with = "sp_core::bytes"))] - pub Vec, -); - -impl OpaqueLeaf { - /// Convert a concrete MMR leaf into an opaque type. - pub fn from_leaf(leaf: &T) -> Self { - let encoded_leaf = leaf.using_encoded(|d| d.to_vec(), true); - OpaqueLeaf::from_encoded_leaf(encoded_leaf) - } - - /// Create a `OpaqueLeaf` given raw bytes of compact-encoded leaf. - pub fn from_encoded_leaf(encoded_leaf: Vec) -> Self { - OpaqueLeaf(encoded_leaf) - } - - /// Attempt to decode the leaf into expected concrete type. - pub fn try_decode(&self) -> Option { - codec::Decode::decode(&mut &*self.0).ok() - } -} - -impl FullLeaf for OpaqueLeaf { - fn using_encoded R>(&self, f: F, _compact: bool) -> R { - f(&self.0) - } -} - -/// A type-safe wrapper for the concrete leaf type. -/// -/// This structure serves merely to avoid passing raw `Vec` around. -/// It must be `Vec`-encoding compatible. -/// -/// It is different from [`OpaqueLeaf`], because it does implement `Codec` -/// and the encoding has to match raw `Vec` encoding. -#[derive(codec::Encode, codec::Decode, RuntimeDebug, PartialEq, Eq)] -pub struct EncodableOpaqueLeaf(pub Vec); - -impl EncodableOpaqueLeaf { - /// Convert a concrete leaf into encodable opaque version. - pub fn from_leaf(leaf: &T) -> Self { - let opaque = OpaqueLeaf::from_leaf(leaf); - Self::from_opaque_leaf(opaque) - } - - /// Given an opaque leaf, make it encodable. - pub fn from_opaque_leaf(opaque: OpaqueLeaf) -> Self { - Self(opaque.0) - } - - /// Try to convert into a [OpaqueLeaf]. - pub fn into_opaque_leaf(self) -> OpaqueLeaf { - // wrap into `OpaqueLeaf` type - OpaqueLeaf::from_encoded_leaf(self.0) - } -} - sp_api::decl_runtime_apis! { /// API to interact with MMR pallet. pub trait MmrApi { @@ -429,6 +414,9 @@ sp_api::decl_runtime_apis! { /// The leaf data is expected to be encoded in it's compact form. fn verify_proof_stateless(root: Hash, leaf: EncodableOpaqueLeaf, proof: Proof) -> Result<(), Error>; + + /// Return the on-chain MMR root hash. + fn mmr_root() -> Result; } }