Feature/update to polkadot v0 9 0

.github/workflows/monthly-tag.yml

@@ -0,0 +1,43 @@
+name: Monthly Snapshot Tag
+
+on:
+  schedule:
+    - cron: "0 1 1 * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Take Snapshot
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the tags by date
+        id: tags
+        run: |
+          echo "::set-output name=new::$(date +'monthly-%Y-%m')"
+          echo "::set-output name=old::$(date -d'1 month ago' +'monthly-%Y-%m')"
+      - name: Checkout branch "master"
+        uses: actions/checkout@v2
+        with:
+          ref: 'master'
+          fetch-depth: 0
+      - name: Generate changelog
+        id: changelog
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "# Automatic snapshot pre-release ${{ steps.tags.outputs.new }}" > Changelog.md
+          echo "" >> Changelog.md
+          echo "## Changes since last snapshot (${{ steps.tags.outputs.old }})" >> Changelog.md
+          echo "" >> Changelog.md
+          ./.maintain/gitlab/generate_changelog.sh ${{ steps.tags.outputs.old }} >>  Changelog.md
+      - name: Release snapshot
+        id: release-snapshot
+        uses: actions/create-release@latest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.tags.outputs.new }}
+          release_name: ${{ steps.tags.outputs.new }}
+          draft: false
+          prerelease: true
+          body_path: Changelog.md

.gitlab-ci.yml

@@ -26,7 +26,6 @@ stages:
   - build
   - publish
   - deploy
-  - flaming-fir
 
 workflow:
   rules:
@@ -60,12 +59,15 @@ default:
     - kubernetes-parity-build
   interruptible:                   true
 
+.rust-info-script:                 &rust-info-script
+  - rustup show
+  - cargo --version
+  - sccache -s
+
 .docker-env:                       &docker-env
   image:                           "${CI_IMAGE}"
   before_script:
-    - rustup show
-    - cargo --version
-    - sccache -s
+    - *rust-info-script
   retry:
     max: 2
     when:
@@ -81,7 +83,6 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
@@ -92,38 +93,49 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
-.build-refs:                       &build-refs
+.test-refs-no-trigger-prs-only:   &test-refs-no-trigger-prs-only
   rules:
-    # .publish-refs with manual on PRs
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: manual
-      allow_failure: true
 
-.publish-refs:                     &publish-refs
+.build-refs:                       &build-refs
   rules:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
 .nightly-pipeline:                 &nightly-pipeline
   rules:
     # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
 
+.merge-ref-into-master-script:     &merge-ref-into-master-script
+  - if [ $CI_COMMIT_REF_NAME != "master" ]; then
+      git fetch origin +master:master;
+      git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
+      git checkout master;
+      git config user.email "ci@gitlab.parity.io";
+      git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
+    fi
+
+.cargo-check-benches-script:       &cargo-check-benches-script
+  - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
+  - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
+  - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
+    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
+  - 'cargo run --release -p node-bench -- ::trie::read::small --json
+    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
+  - sccache -s
+
 #### stage:                       .pre
 
 skip-if-draft:
@@ -222,12 +234,37 @@ cargo-deny:
 cargo-check-benches:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *test-refs-no-trigger
+  <<:                              *test-refs
+  <<:                              *collect-artifacts
+  before_script:
+    # merges in the master branch on PRs
+    - *merge-ref-into-master-script
+    - *rust-info-script
   script:
-    - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
-    - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small
-    - cargo run --release -p node-bench -- ::trie::read::small
-    - sccache -s
+    - *cargo-check-benches-script
+
+node-bench-regression-guard:
+  # it's not belong to `build` semantically, but dag jobs can't depend on each other
+  # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
+  # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
+  stage:                           build
+  <<:                              *docker-env
+  <<:                              *test-refs-no-trigger-prs-only
+  needs:
+    # this is a DAG
+    - job:                         cargo-check-benches
+      artifacts:                   true
+    # this does not like a DAG, just polls the artifact
+    - project:                     $CI_PROJECT_PATH
+      job:                         cargo-check-benches
+      ref:                         master
+      artifacts:                   true
+  variables:
+    CI_IMAGE:                      "paritytech/node-bench-regression-guard:latest"
+  before_script: [""]
+  script:
+    - 'node-bench-regression-guard --reference artifacts/benches/master-* 
+       --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
 
 cargo-check-subkey:
   stage:                           test
@@ -238,6 +275,14 @@ cargo-check-subkey:
     - SKIP_WASM_BUILD=1 time cargo check --release
     - sccache -s
 
+cargo-check-try-runtime:
+  stage:                           test
+  <<:                              *docker-env
+  <<:                              *test-refs
+  script:
+    - time cargo check --features try-runtime
+    - sccache -s
+
 test-deterministic-wasm:
   stage:                           test
   <<:                              *docker-env
@@ -272,7 +317,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
-    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml # does not reuse cache 1 min 44 sec
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -283,7 +328,6 @@ unleash-check:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   script:
     - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
@@ -336,7 +380,7 @@ check-web-wasm:
     # Note: we don't need to test crates imported in `bin/node/cli`
     - time cargo build --manifest-path=client/consensus/aura/Cargo.toml --target=wasm32-unknown-unknown --features getrandom
     # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
-    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget
+    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown
     # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
@@ -407,7 +451,7 @@ test-browser-node:
     CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner"
     WASM_BINDGEN_TEST_TIMEOUT:                  120
   script:
-    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget
+    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing
 
 build-linux-substrate:             &build-binary
   stage:                           build
@@ -491,7 +535,7 @@ build-rust-doc:
 #### stage:                        publish
 
 .build-push-docker-image:          &build-push-docker-image
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   image:                           quay.io/buildah/stable
   variables:                       &docker-build-vars
@@ -519,15 +563,16 @@ build-rust-doc:
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
-    # pass artifacts to the trigget-simnet job
-    - echo "VERSION=${VERSION}" > build.env
-    - echo "TRIGGERER=${CI_PROJECT_NAME}" >> build.env
   after_script:
     - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the trigger-simnet job
+    - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/$PRODUCT/build.env
+    - echo "IMAGE_TAG=${VERSION}" >> ./artifacts/$PRODUCT/build.env
 
 publish-docker-substrate:
   stage:                           publish
   <<:                              *build-push-docker-image
+  <<:                              *build-refs
   needs:
     - job:                         build-linux-substrate
       artifacts:                   true
@@ -538,7 +583,7 @@ publish-docker-substrate:
     reports:
       # this artifact is used in trigger-simnet job
       # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-      dotenv: artifacts/substrate/build.env
+      dotenv: ./artifacts/substrate/build.env
 
 publish-docker-subkey:
   stage:                           publish
@@ -552,7 +597,7 @@ publish-docker-subkey:
 
 publish-s3-release:
   stage:                           publish
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   needs:
     - job:                         build-linux-substrate
@@ -581,7 +626,7 @@ publish-s3-doc:
       artifacts:                   true
     - job:                         build-linux-substrate
       artifacts:                   false
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   variables:
     GIT_STRATEGY:                  none
@@ -651,10 +696,21 @@ deploy-prometheus-alerting-rules:
 
 trigger-simnet:
   stage:                           deploy
-  <<:                              *nightly-pipeline
+  image:                           paritytech/tools:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
   needs:
     - job:                         publish-docker-substrate
-  trigger:
-    project:                       parity/simnet
-    branch:                        master
-    strategy:                      depend
+  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here, 
+  # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
+  variables:
+    TRGR_PROJECT:                  ${CI_PROJECT_NAME}
+    TRGR_REF:                      ${CI_COMMIT_REF_NAME}
+    # simnet project ID
+    DWNSTRM_ID:                    332
+  script:
+    # API trigger for a simnet job
+    - ./scripts/gitlab/trigger_pipeline.sh

.maintain/gitlab/check_polkadot_companion_build.sh

@@ -89,7 +89,4 @@ fi
 diener patch --crates-to-patch ../ --substrate --path Cargo.toml
 
 # Test Polkadot pr or master branch with this Substrate commit.
-time cargo test --all --release --verbose --features=real-overseer
-
-cd parachain/test-parachains/adder/collator/
-time cargo test --release --verbose --locked --features=real-overseer
+time cargo test --all --release --verbose

.maintain/gitlab/check_polkadot_companion_status.sh

@@ -56,28 +56,14 @@ fi
 boldprint "companion pr: #${pr_companion}"
 
 # check the status of that pull request - needs to be
-# mergable and approved
+# approved and mergable
 
 curl -H "${github_header}" -sS -o companion_pr.json \
   ${github_api_polkadot_pull_url}/${pr_companion}
 
 pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json)
 boldprint "Polkadot PR's HEAD SHA: $pr_head_sha"
 
-if jq -e .merged < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} already merged"
-  exit 0
-fi
-
-if jq -e '.mergeable' < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} mergeable"
-else
-  boldprint "polkadot pr #${pr_companion} not mergeable"
-  exit 1
-fi
-
 curl -H "${github_header}" -sS -o companion_pr_reviews.json \
   ${github_api_polkadot_pull_url}/${pr_companion}/reviews
 
@@ -98,6 +84,19 @@ if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews
 fi
 
 boldprint "polkadot pr #${pr_companion} state APPROVED"
-exit 0
 
+if jq -e .merged < companion_pr.json >/dev/null
+then
+  boldprint "polkadot pr #${pr_companion} already merged"
+  exit 0
+fi
+
+if jq -e '.mergeable' < companion_pr.json >/dev/null
+then
+  boldprint "polkadot pr #${pr_companion} mergeable"
+else
+  boldprint "polkadot pr #${pr_companion} not mergeable"
+  exit 1
+fi
 
+exit 0

.maintain/gitlab/generate_changelog.sh

@@ -32,7 +32,7 @@ $line"
     runtime_changes="$runtime_changes
 $line"
   fi
-  if has_label 'paritytech/substrate' "$pr_id" 'D1-runtime-migration'; then
+  if has_label 'paritytech/substrate' "$pr_id" 'E1-runtime-migration'; then
     migrations="$migrations
 $line"
   fi