From 082246a328fd0e8101167ebf7e62a879442afd61 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 11:22:58 +0200 Subject: [PATCH 01/18] [Do not merge] [ci] publish parachain-implementers-guide --- .gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a46f3b5915e8..412ad7e24393 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,14 @@ default: paths: - ./artifacts/ +.collect-artifacts-short: &collect-artifacts-short + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" + when: on_success + expire_in: 1 days + paths: + - ./artifacts/ + .kubernetes-env: &kubernetes-env retry: max: 2 @@ -559,7 +567,7 @@ build-rustdoc: script: # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo +nightly doc --workspace --verbose + - time cargo +nightly doc --workspace --verbose --no-deps - rm -f ./target/doc/.lock - mv ./target/doc ./crate-docs # FIXME: remove me after CI image gets nonroot @@ -575,11 +583,16 @@ generate-impl-guide: artifacts: false <<: *test-refs <<: *docker-env + <<: *collect-artifacts-short image: name: michaelfbryan/mdbook-docker-image:v0.4.4 entrypoint: [""] script: - mdbook build ./roadmap/implementers-guide + - mkdir -p artifacts + - mv roadmap/implementers-guide/book artifacts/ + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot artifacts/ check-try-runtime: stage: stage3 From 508ca5d20a683c7a538b4aab1e4fa2ab70dde12f Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 11:28:45 +0200 Subject: [PATCH 02/18] fix .cancel-pipeline --- .gitlab-ci.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 412ad7e24393..59e511f87010 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -932,13 +932,8 @@ short-benchmark-westend: # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. -cancel-pipeline: +.cancel-pipeline-template: stage: .post - needs: - - job: test-linux-stable - artifacts: false - - job: check-try-runtime - artifacts: false rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: on_failure @@ -947,4 +942,14 @@ cancel-pipeline: PIPELINE_ID: "${CI_PIPELINE_ID}" trigger: "parity/infrastructure/ci_cd/pipeline-stopper" +cancel-pipeline-test-linux-stable: + extends: .cancel-pipeline-template + needs: + - job: test-linux-stable + artifacts: false +cancel-pipeline-check-try-runtime: + extends: .cancel-pipeline-template + needs: + - job: check-try-runtime + artifacts: false \ No newline at end of file From 1411bb477040beda3e8d9ef231b51e7e0544228d Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 11:36:32 +0200 Subject: [PATCH 03/18] rename build-implementers-guide --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59e511f87010..93cb8230bcf7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -575,12 +575,12 @@ build-rustdoc: - echo "" > ./crate-docs/index.html - sccache -s -generate-impl-guide: - stage: stage3 +build-implementers-guide: + stage: stage1 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-transaction-versions - artifacts: false + # needs: + # - job: check-transaction-versions + # artifacts: false <<: *test-refs <<: *docker-env <<: *collect-artifacts-short From d27bcbcb7a11ee11a314548efb3e74fa09c9f05d Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 11:44:54 +0200 Subject: [PATCH 04/18] fix --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93cb8230bcf7..85d245e43206 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -591,8 +591,7 @@ build-implementers-guide: - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot artifacts/ + check-try-runtime: stage: stage3 From 7b4e4c6e1028b0761c29d7d6e269a7d0cad3f226 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:00:42 +0200 Subject: [PATCH 05/18] debig docs --- .gitlab-ci.yml | 1407 ++++++++++++++++++++++++------------------------ 1 file changed, 706 insertions(+), 701 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85d245e43206..0e1b8691926e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,393 +167,657 @@ default: #### stage: stage1 -check-runtime: - stage: stage1 - image: paritytech/tools:latest - <<: *kubernetes-env - <<: *test-refs - variables: - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./scripts/ci/gitlab/check_runtime.sh - allow_failure: true - -cargo-fmt: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly --version - - cargo +nightly fmt --all -- --check - allow_failure: true - -build-linux-stable: - stage: stage1 - <<: *docker-env - <<: *compiler-info - <<: *collect-artifacts - <<: *common-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - - mv ./target/testnet/polkadot ./artifacts/. - - pushd artifacts - - sha256sum polkadot | tee polkadot.sha256 - - shasum -c polkadot.sha256 - - popd - - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - - echo -n ${VERSION} > ./artifacts/VERSION - - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - - cp -r scripts/* ./artifacts - -test-linux-stable: - stage: stage1 - <<: *docker-env - <<: *compiler-info - <<: *common-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics - - - -spellcheck: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo spellcheck --version - # compare with the commit parent to the PR, given it's from a default branch - - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} - - echo "___Spellcheck is going to check your diff___" - - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 - $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - allow_failure: true - -build-test-collators: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --profile testnet --verbose -p test-parachain-undying-collator - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/adder-collator ./artifacts/. - - mv ./target/testnet/undying-collator ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-malus: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p polkadot-test-malus - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/malus ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-staking-miner: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *common-refs - script: - - time cargo build --locked --release --package staking-miner - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/release/staking-miner ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -#### stage: stage2 - -.check-dependent-project: &check-dependent-project - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-runtime - artifacts: false - <<: *docker-env - <<: *test-pr-refs - script: - - git clone - --depth=1 - "--branch=$PIPELINE_SCRIPTS_TAG" - https://github.com/paritytech/pipeline-scripts - - ./pipeline-scripts/check_dependent_project.sh - --org paritytech - --dependent-repo "$DEPENDENT_REPO" - --github-api-token "$GITHUB_PR_TOKEN" - --extra-dependencies "$EXTRA_DEPENDENCIES" - --companion-overrides "$COMPANION_OVERRIDES" - -check-dependent-cumulus: - <<: *check-dependent-project - variables: - DEPENDENT_REPO: cumulus - EXTRA_DEPENDENCIES: substrate - COMPANION_OVERRIDES: | - polkadot: release-v* - cumulus: polkadot-v* - -test-node-metrics: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-runtime - artifacts: false - <<: *docker-env - <<: *compiler-info - <<: *test-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics - -test-deterministic-wasm: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - - ./scripts/ci/gitlab/test_deterministic_wasm.sh - -check-transaction-versions: - stage: stage2 - <<: *test-refs - <<: *docker-env - image: node:15 - needs: - - job: build-linux-stable - artifacts: true - before_script: - - apt-get -y update; apt-get -y install jq lsof - - npm install --ignore-scripts -g @polkadot/metadata-cmp - # Set git config - - git config remote.origin.url "https://github.com/paritytech/polkadot.git" - - git fetch origin release - script: - - ./scripts/ci/gitlab/check_extrinsics_ordering.sh - - - -# This image is used in testnets -# Release image is handled by the Github Action here: -# .github/workflows/publish-docker-release.yml -publish-polkadot-debug-image: - stage: stage2 - <<: *build-push-image - rules: - # Don't run when triggered from another pipeline - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - variables: - <<: *image-variables - # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile - DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile - IMAGE_NAME: docker.io/paritypr/polkadot-debug - needs: - - job: build-linux-stable - artifacts: true - after_script: - # pass artifacts to the zombienet-tests job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env - - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/parachains.env - expire_in: 1 days - -publish-test-collators-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/collator_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/colander - needs: - - job: build-test-collators - artifacts: true - after_script: - - buildah logout --all - # pass artifacts to the zombienet-tests job - - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env - - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/collator.env - -publish-malus-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/malus_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/malus - needs: - - job: build-malus - artifacts: true - after_script: - - buildah logout "$IMAGE_NAME" - # pass artifacts to the zombienet-tests job - - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env - - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/malus.env - -publish-staking-miner-image: - stage: stage2 - <<: *build-push-image - <<: *publish-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - IMAGE_NAME: docker.io/paritytech/staking-miner - GIT_STRATEGY: none - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} - needs: - - job: build-staking-miner - artifacts: true - - -publish-s3-release: &publish-s3 - stage: stage3 - needs: - - job: build-linux-stable - artifacts: true - <<: *kubernetes-env - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - # publishing binaries nightly - - if: $CI_PIPELINE_SOURCE == "schedule" - before_script: - - *check-versions - script: - - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" - - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ - - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" - - find ./artifacts -type f | while read file; do - name="${file#./artifacts/}"; - aws s3api copy-object - --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} - --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; - done - - | - cat <<-EOM - | - | polkadot binary paths: - | - | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot - | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot - | - EOM - after_script: - - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ - --recursive --human-readable --summarize - -update_polkadot_weights: &update-weights - stage: stage2 - when: manual - variables: - RUNTIME: polkadot - artifacts: - paths: - - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - script: - - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME - - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - # uses the "shell" executors - tags: - - weights - -update_kusama_weights: - <<: *update-weights - variables: - RUNTIME: kusama - -update_westend_weights: - <<: *update-weights - variables: - RUNTIME: westend +# check-runtime: +# stage: stage1 +# image: paritytech/tools:latest +# <<: *kubernetes-env +# <<: *test-refs +# variables: +# GITLAB_API: "https://gitlab.parity.io/api/v4" +# GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" +# script: +# - ./scripts/ci/gitlab/check_runtime.sh +# allow_failure: true + +# cargo-fmt: +# stage: stage1 +# <<: *docker-env +# <<: *test-refs +# script: +# - cargo +nightly --version +# - cargo +nightly fmt --all -- --check +# allow_failure: true + +# build-linux-stable: +# stage: stage1 +# <<: *docker-env +# <<: *compiler-info +# <<: *collect-artifacts +# <<: *common-refs +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# # Ensure we run the UI tests. +# RUN_UI_TESTS: 1 +# script: +# - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name +# - mv ./target/testnet/polkadot ./artifacts/. +# - pushd artifacts +# - sha256sum polkadot | tee polkadot.sha256 +# - shasum -c polkadot.sha256 +# - popd +# - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" +# - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" +# - echo -n ${VERSION} > ./artifacts/VERSION +# - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG +# - cp -r scripts/* ./artifacts + +# test-linux-stable: +# stage: stage1 +# <<: *docker-env +# <<: *compiler-info +# <<: *common-refs +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# script: +# - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics + + + +# spellcheck: +# stage: stage1 +# <<: *docker-env +# <<: *test-refs +# script: +# - cargo spellcheck --version +# # compare with the commit parent to the PR, given it's from a default branch +# - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} +# - echo "___Spellcheck is going to check your diff___" +# - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) +# - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 +# $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) +# allow_failure: true + +# build-test-collators: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# script: +# - time cargo build --profile testnet --verbose -p test-parachain-adder-collator +# - time cargo build --profile testnet --verbose -p test-parachain-undying-collator +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/testnet/adder-collator ./artifacts/. +# - mv ./target/testnet/undying-collator ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# build-malus: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# script: +# - time cargo build --profile testnet --verbose -p polkadot-test-malus +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/testnet/malus ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# build-staking-miner: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *common-refs +# script: +# - time cargo build --locked --release --package staking-miner +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/release/staking-miner ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# #### stage: stage2 + +# .check-dependent-project: &check-dependent-project +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: check-runtime +# artifacts: false +# <<: *docker-env +# <<: *test-pr-refs +# script: +# - git clone +# --depth=1 +# "--branch=$PIPELINE_SCRIPTS_TAG" +# https://github.com/paritytech/pipeline-scripts +# - ./pipeline-scripts/check_dependent_project.sh +# --org paritytech +# --dependent-repo "$DEPENDENT_REPO" +# --github-api-token "$GITHUB_PR_TOKEN" +# --extra-dependencies "$EXTRA_DEPENDENCIES" +# --companion-overrides "$COMPANION_OVERRIDES" + +# check-dependent-cumulus: +# <<: *check-dependent-project +# variables: +# DEPENDENT_REPO: cumulus +# EXTRA_DEPENDENCIES: substrate +# COMPANION_OVERRIDES: | +# polkadot: release-v* +# cumulus: polkadot-v* + +# test-node-metrics: +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: check-runtime +# artifacts: false +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# script: +# - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics + +# test-deterministic-wasm: +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: cargo-fmt +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# - ./scripts/ci/gitlab/test_deterministic_wasm.sh + +# check-transaction-versions: +# stage: stage2 +# <<: *test-refs +# <<: *docker-env +# image: node:15 +# needs: +# - job: build-linux-stable +# artifacts: true +# before_script: +# - apt-get -y update; apt-get -y install jq lsof +# - npm install --ignore-scripts -g @polkadot/metadata-cmp +# # Set git config +# - git config remote.origin.url "https://github.com/paritytech/polkadot.git" +# - git fetch origin release +# script: +# - ./scripts/ci/gitlab/check_extrinsics_ordering.sh + + + +# # This image is used in testnets +# # Release image is handled by the Github Action here: +# # .github/workflows/publish-docker-release.yml +# publish-polkadot-debug-image: +# stage: stage2 +# <<: *build-push-image +# rules: +# # Don't run when triggered from another pipeline +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# - if: $CI_PIPELINE_SOURCE == "web" +# - if: $CI_PIPELINE_SOURCE == "schedule" +# - if: $CI_COMMIT_REF_NAME == "master" +# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +# DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile +# IMAGE_NAME: docker.io/paritypr/polkadot-debug +# needs: +# - job: build-linux-stable +# artifacts: true +# after_script: +# # pass artifacts to the zombienet-tests job +# # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance +# - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env +# - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/parachains.env +# expire_in: 1 days + +# publish-test-collators-image: +# # service image for Simnet +# stage: stage2 +# <<: *build-push-image +# <<: *zombienet-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/collator_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile +# IMAGE_NAME: docker.io/paritypr/colander +# needs: +# - job: build-test-collators +# artifacts: true +# after_script: +# - buildah logout --all +# # pass artifacts to the zombienet-tests job +# - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env +# - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/collator.env + +# publish-malus-image: +# # service image for Simnet +# stage: stage2 +# <<: *build-push-image +# <<: *zombienet-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/malus_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile +# IMAGE_NAME: docker.io/paritypr/malus +# needs: +# - job: build-malus +# artifacts: true +# after_script: +# - buildah logout "$IMAGE_NAME" +# # pass artifacts to the zombienet-tests job +# - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env +# - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/malus.env + +# publish-staking-miner-image: +# stage: stage2 +# <<: *build-push-image +# <<: *publish-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +# IMAGE_NAME: docker.io/paritytech/staking-miner +# GIT_STRATEGY: none +# DOCKER_USER: ${Docker_Hub_User_Parity} +# DOCKER_PASS: ${Docker_Hub_Pass_Parity} +# needs: +# - job: build-staking-miner +# artifacts: true + + +# publish-s3-release: &publish-s3 +# stage: stage3 +# needs: +# - job: build-linux-stable +# artifacts: true +# <<: *kubernetes-env +# image: paritytech/awscli:latest +# variables: +# GIT_STRATEGY: none +# PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" +# rules: +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# # publishing binaries nightly +# - if: $CI_PIPELINE_SOURCE == "schedule" +# before_script: +# - *check-versions +# script: +# - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" +# - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ +# - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" +# - find ./artifacts -type f | while read file; do +# name="${file#./artifacts/}"; +# aws s3api copy-object +# --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} +# --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; +# done +# - | +# cat <<-EOM +# | +# | polkadot binary paths: +# | +# | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot +# | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot +# | +# EOM +# after_script: +# - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ +# --recursive --human-readable --summarize + +# update_polkadot_weights: &update-weights +# stage: stage2 +# when: manual +# variables: +# RUNTIME: polkadot +# artifacts: +# paths: +# - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch +# script: +# - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME +# - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch +# # uses the "shell" executors +# tags: +# - weights + +# update_kusama_weights: +# <<: *update-weights +# variables: +# RUNTIME: kusama + +# update_westend_weights: +# <<: *update-weights +# variables: +# RUNTIME: westend + +# update_rococo_weights: +# <<: *update-weights +# variables: +# RUNTIME: rococo + +# #### stage: stage3 + + + + +# check-try-runtime: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-node-metrics +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# # Check that everything compiles with `try-runtime` feature flag. +# - cargo check --features try-runtime --all +# - sccache -s + +# check-no-default-features: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# # Check that polkadot-cli will compile no default features. +# - pushd ./node/service && cargo check --no-default-features && popd +# - pushd ./cli && cargo check --no-default-features --features "service" && popd +# - sccache -s + +# build-short-benchmark: +# stage: stage3 +# <<: *test-refs +# <<: *docker-env +# <<: *collect-artifacts +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-node-metrics +# artifacts: false +# script: +# - cargo +nightly build --profile release --locked --features=runtime-benchmarks +# - mkdir artifacts +# - cp ./target/release/polkadot ./artifacts/ + +# deploy-parity-testnet: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *deploy-testnet-refs +# variables: +# POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" +# POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" +# allow_failure: false +# trigger: "parity/infrastructure/parity-testnet" + +# zombienet-tests-parachains-smoke-test: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-parachains-smoke-test.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-parachains-pvf: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-parachains-pvf.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-parachains-disputes: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# - job: publish-malus-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0002-parachains-disputes.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-test-parachains-upgrade-smoke-test: +# stage: stage3 +# <<: *kubernetes-env +# <<: *zombienet-refs +# image: "docker.io/paritytech/zombienet:v1.2.45" +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' +# before_script: +# - echo "ZombieNet Tests Config" +# - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" +# - echo "docker.io/parity/polkadot-collator:latest" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0002-parachains-upgrade-smoke-test.feature" +# allow_failure: true +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-misc-paritydb: +# stage: stage3 +# <<: *kubernetes-env +# <<: *zombienet-refs +# image: "docker.io/paritytech/zombienet:v1.2.49" +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# artifacts: true +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-paritydb.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-malus-dispute-valid: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie* +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-dispute-valid-block.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-deregister-register-validator: +# stage: stage3 +# image: "docker.io/paritytech/zombienet:v1.2.47" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# artifacts: true +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie* +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0003-deregister-register-validator-smoke.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test -update_rococo_weights: - <<: *update-weights - variables: - RUNTIME: rococo - -#### stage: stage3 +#### stage: stage4 build-rustdoc: - stage: stage3 + stage: stage1 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false + # needs: + # - job: test-deterministic-wasm + # artifacts: false <<: *docker-env <<: *test-refs variables: @@ -579,7 +843,7 @@ build-implementers-guide: stage: stage1 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs # needs: - # - job: check-transaction-versions + # - job: test-deterministic-wasm # artifacts: false <<: *test-refs <<: *docker-env @@ -592,284 +856,25 @@ build-implementers-guide: - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ - -check-try-runtime: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - # Check that everything compiles with `try-runtime` feature flag. - - cargo check --features try-runtime --all - - sccache -s - -check-no-default-features: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - # Check that polkadot-cli will compile no default features. - - pushd ./node/service && cargo check --no-default-features && popd - - pushd ./cli && cargo check --no-default-features --features "service" && popd - - sccache -s - -build-short-benchmark: - stage: stage3 - <<: *test-refs - <<: *docker-env - <<: *collect-artifacts - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - script: - - cargo +nightly build --profile release --locked --features=runtime-benchmarks - - mkdir artifacts - - cp ./target/release/polkadot ./artifacts/ - -deploy-parity-testnet: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *deploy-testnet-refs - variables: - POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" - allow_failure: false - trigger: "parity/infrastructure/parity-testnet" - -zombienet-tests-parachains-smoke-test: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-smoke-test.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-pvf: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-pvf.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-disputes.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-test-parachains-upgrade-smoke-test: - stage: stage3 - <<: *kubernetes-env - <<: *zombienet-refs - image: "docker.io/paritytech/zombienet:v1.2.45" - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' - before_script: - - echo "ZombieNet Tests Config" - - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" - - echo "docker.io/parity/polkadot-collator:latest" - - echo "${ZOMBIENET_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-upgrade-smoke-test.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-paritydb: - stage: stage3 - <<: *kubernetes-env - <<: *zombienet-refs - image: "docker.io/paritytech/zombienet:v1.2.49" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-paritydb.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-malus-dispute-valid: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-dispute-valid-block.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-deregister-register-validator: - stage: stage3 - image: "docker.io/paritytech/zombienet:v1.2.47" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-deregister-register-validator-smoke.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -#### stage: stage4 - publish-rustdoc: stage: stage4 <<: *kubernetes-env image: paritytech/tools:latest variables: GIT_DEPTH: 100 - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" + <<: *test-refs + # rules: + # - if: $CI_PIPELINE_SOURCE == "pipeline" + # when: never + # - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" + # - if: $CI_COMMIT_REF_NAME == "master" # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other # artifacts from being dowloaded by this job. needs: - job: build-rustdoc artifacts: true + - job: build-rustdoc + artifacts: true script: # setup ssh - eval $(ssh-agent) @@ -902,53 +907,53 @@ publish-rustdoc: after_script: - rm -rf .git/ ./* -# Run all pallet benchmarks only once to check if there are any errors -short-benchmark-polkadot: &short-bench - stage: stage4 - <<: *test-pr-refs - <<: *docker-env - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-short-benchmark - artifacts: true - variables: - RUNTIME: polkadot - script: - - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 1 --repeat 1 - -short-benchmark-kusama: - <<: *short-bench - variables: - RUNTIME: kusama - -short-benchmark-westend: - <<: *short-bench - variables: - RUNTIME: westend - -#### stage: .post - -# This job cancels the whole pipeline if any of provided jobs fail. -# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests -# to fail the pipeline as soon as possible to shorten the feedback loop. -.cancel-pipeline-template: - stage: .post - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: on_failure - variables: - PROJECT_ID: "${CI_PROJECT_ID}" - PIPELINE_ID: "${CI_PIPELINE_ID}" - trigger: "parity/infrastructure/ci_cd/pipeline-stopper" - -cancel-pipeline-test-linux-stable: - extends: .cancel-pipeline-template - needs: - - job: test-linux-stable - artifacts: false - -cancel-pipeline-check-try-runtime: - extends: .cancel-pipeline-template - needs: - - job: check-try-runtime - artifacts: false \ No newline at end of file +# # Run all pallet benchmarks only once to check if there are any errors +# short-benchmark-polkadot: &short-bench +# stage: stage4 +# <<: *test-pr-refs +# <<: *docker-env +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: build-short-benchmark +# artifacts: true +# variables: +# RUNTIME: polkadot +# script: +# - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 1 --repeat 1 + +# short-benchmark-kusama: +# <<: *short-bench +# variables: +# RUNTIME: kusama + +# short-benchmark-westend: +# <<: *short-bench +# variables: +# RUNTIME: westend + +# #### stage: .post + +# # This job cancels the whole pipeline if any of provided jobs fail. +# # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# # to fail the pipeline as soon as possible to shorten the feedback loop. +# .cancel-pipeline-template: +# stage: .post +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +# when: on_failure +# variables: +# PROJECT_ID: "${CI_PROJECT_ID}" +# PIPELINE_ID: "${CI_PIPELINE_ID}" +# trigger: "parity/infrastructure/ci_cd/pipeline-stopper" + +# cancel-pipeline-test-linux-stable: +# extends: .cancel-pipeline-template +# needs: +# - job: test-linux-stable +# artifacts: false + +# cancel-pipeline-check-try-runtime: +# extends: .cancel-pipeline-template +# needs: +# - job: check-try-runtime +# artifacts: false From 525a83c9abc982840a2714c194678588f99f6a8f Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:01:35 +0200 Subject: [PATCH 06/18] fix needs --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e1b8691926e..714604f50702 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -873,7 +873,7 @@ publish-rustdoc: needs: - job: build-rustdoc artifacts: true - - job: build-rustdoc + - job: build-implementers-guide artifacts: true script: # setup ssh From c4bc61e92280f9df392014cc528a49c9f22fd959 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:05:09 +0200 Subject: [PATCH 07/18] debug publish --- .gitlab-ci.yml | 60 ++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 714604f50702..a8100e330998 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -876,35 +876,37 @@ publish-rustdoc: - job: build-implementers-guide artifacts: true script: - # setup ssh - - eval $(ssh-agent) - - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} - - mkdir ~/.ssh && touch ~/.ssh/known_hosts - - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - # Set git config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp ./README.md /tmp/doc/ - - git checkout gh-pages - # Remove everything and restore generated docs and README - - rm -rf ./* - - mv /tmp/doc/* . - # Upload files - - git add --all --force - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" - after_script: + - ls -la artifacts/ + - ls -la crate-docs/ + # # setup ssh + # - eval $(ssh-agent) + # - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + # - mkdir ~/.ssh && touch ~/.ssh/known_hosts + # - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # # Set git config + # - git config user.email "devops-team@parity.io" + # - git config user.name "${GITHUB_USER}" + # - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" + # - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + # - git fetch origin gh-pages + # # Save README and docs + # - cp -r ./crate-docs/ /tmp/doc/ + # - cp ./README.md /tmp/doc/ + # - git checkout gh-pages + # # Remove everything and restore generated docs and README + # - rm -rf ./* + # - mv /tmp/doc/* . + # # Upload files + # - git add --all --force + # # `git commit` has an exit code of > 0 if there is nothing to commit. + # # This causes GitLab to exit immediately and marks this job failed. + # # We don't want to mark the entire job failed if there's nothing to + # # publish though, hence the `|| true`. + # - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || + # echo "___Nothing to commit___" + # - git push origin gh-pages --force + # - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" + # after_script: - rm -rf .git/ ./* # # Run all pallet benchmarks only once to check if there are any errors From 7133ad027e1a5441415c178b98b2b224682b73c6 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:10:16 +0200 Subject: [PATCH 08/18] change image for impl guide job --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8100e330998..fe07bae40baa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -848,13 +848,13 @@ build-implementers-guide: <<: *test-refs <<: *docker-env <<: *collect-artifacts-short - image: - name: michaelfbryan/mdbook-docker-image:v0.4.4 - entrypoint: [""] script: + - cargo install mdbook - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot artifacts/ publish-rustdoc: stage: stage4 From b97a47c4b1dd061772b429535a7715b364369ddc Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:13:01 +0200 Subject: [PATCH 09/18] fix mdbook --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe07bae40baa..83ca88da59ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -849,7 +849,7 @@ build-implementers-guide: <<: *docker-env <<: *collect-artifacts-short script: - - cargo install mdbook + - cargo install mdbook mdbook-graphviz mdbook-mermaid mdbook-linkcheck - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ From 23db5986c2d39079dc90dc053edd8e697dbe2c96 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:19:14 +0200 Subject: [PATCH 10/18] add folder --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83ca88da59ad..8acb1dbe2a62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -851,6 +851,7 @@ build-implementers-guide: script: - cargo install mdbook mdbook-graphviz mdbook-mermaid mdbook-linkcheck - mdbook build ./roadmap/implementers-guide + - mkdir -p roadmap/implementers-guide/book - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ # FIXME: remove me after CI image gets nonroot From 247b995534d9279a2d287c819e47df81421e2ac8 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:45:53 +0200 Subject: [PATCH 11/18] remove mdbook-graphviz --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8acb1dbe2a62..32479267438c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -849,7 +849,7 @@ build-implementers-guide: <<: *docker-env <<: *collect-artifacts-short script: - - cargo install mdbook mdbook-graphviz mdbook-mermaid mdbook-linkcheck + - cargo install mdbook mdbook-mermaid mdbook-linkcheck - mdbook build ./roadmap/implementers-guide - mkdir -p roadmap/implementers-guide/book - mkdir -p artifacts From 56af02b1389749c371534704330f4df192340d40 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:51:26 +0200 Subject: [PATCH 12/18] fix publish --- .gitlab-ci.yml | 62 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32479267438c..71f9aadca13a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -877,37 +877,37 @@ publish-rustdoc: - job: build-implementers-guide artifacts: true script: - - ls -la artifacts/ - - ls -la crate-docs/ - # # setup ssh - # - eval $(ssh-agent) - # - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} - # - mkdir ~/.ssh && touch ~/.ssh/known_hosts - # - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - # # Set git config - # - git config user.email "devops-team@parity.io" - # - git config user.name "${GITHUB_USER}" - # - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - # - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - # - git fetch origin gh-pages - # # Save README and docs - # - cp -r ./crate-docs/ /tmp/doc/ - # - cp ./README.md /tmp/doc/ - # - git checkout gh-pages - # # Remove everything and restore generated docs and README - # - rm -rf ./* - # - mv /tmp/doc/* . - # # Upload files - # - git add --all --force - # # `git commit` has an exit code of > 0 if there is nothing to commit. - # # This causes GitLab to exit immediately and marks this job failed. - # # We don't want to mark the entire job failed if there's nothing to - # # publish though, hence the `|| true`. - # - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - # echo "___Nothing to commit___" - # - git push origin gh-pages --force - # - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" - # after_script: + # setup ssh + - eval $(ssh-agent) + - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + - mkdir ~/.ssh && touch ~/.ssh/known_hosts + - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # Set git config + - git config user.email "devops-team@parity.io" + - git config user.name "${GITHUB_USER}" + - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch origin gh-pages + # Save README and docs + - cp -r ./crate-docs/ /tmp/doc/ + - cp -r ./artifacts/ /tmp/ + - ls -la /tmp + - cp ./README.md /tmp/doc/ + - git checkout gh-pages + # Remove everything and restore generated docs and README + - rm -rf ./* + - mv /tmp/doc/* . + # Upload files + - git add --all --force + # `git commit` has an exit code of > 0 if there is nothing to commit. + # This causes GitLab to exit immediately and marks this job failed. + # We don't want to mark the entire job failed if there's nothing to + # publish though, hence the `|| true`. + - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || + echo "___Nothing to commit___" + - git push origin gh-pages --force + - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" + after_script: - rm -rf .git/ ./* # # Run all pallet benchmarks only once to check if there are any errors From e0ee5140ecfb9421babb0af486b784b6cd90f722 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 12:59:56 +0200 Subject: [PATCH 13/18] fix publish --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71f9aadca13a..74a8f0a4922b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -851,7 +851,6 @@ build-implementers-guide: script: - cargo install mdbook mdbook-mermaid mdbook-linkcheck - mdbook build ./roadmap/implementers-guide - - mkdir -p roadmap/implementers-guide/book - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ # FIXME: remove me after CI image gets nonroot @@ -877,6 +876,11 @@ publish-rustdoc: - job: build-implementers-guide artifacts: true script: + # Save README and docs + - cp -r ./crate-docs/ /tmp/doc/ + - cp -r ./artifacts/ /tmp/ + - ls -la /tmp + - cp ./README.md /tmp/doc/ # setup ssh - eval $(ssh-agent) - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} @@ -888,15 +892,11 @@ publish-rustdoc: - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch origin gh-pages - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp -r ./artifacts/ /tmp/ - - ls -la /tmp - - cp ./README.md /tmp/doc/ - git checkout gh-pages # Remove everything and restore generated docs and README - rm -rf ./* - - mv /tmp/doc/* . + - mv /tmp/doc . + - mv /tmp/book . # Upload files - git add --all --force # `git commit` has an exit code of > 0 if there is nothing to commit. From 47c9fe077f8eadfce2bd44e4b6798e923fa1db1e Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 13:20:11 +0200 Subject: [PATCH 14/18] debug publish --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74a8f0a4922b..2d3a6507b776 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -837,7 +837,6 @@ build-rustdoc: # FIXME: remove me after CI image gets nonroot - chown -R nonroot:nonroot ./crate-docs - echo "" > ./crate-docs/index.html - - sccache -s build-implementers-guide: stage: stage1 @@ -878,8 +877,10 @@ publish-rustdoc: script: # Save README and docs - cp -r ./crate-docs/ /tmp/doc/ - - cp -r ./artifacts/ /tmp/ + - cp -r ./artifacts/book/ /tmp/ - ls -la /tmp + - ls -la /tmp/doc/ + - ls -la /tmp/book/ - cp ./README.md /tmp/doc/ # setup ssh - eval $(ssh-agent) From 849dda500a05e63ab9224ea9b63b485851e1611a Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 13:46:47 +0200 Subject: [PATCH 15/18] remove debug --- .gitlab-ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d3a6507b776..30df33d4b355 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -878,10 +878,8 @@ publish-rustdoc: # Save README and docs - cp -r ./crate-docs/ /tmp/doc/ - cp -r ./artifacts/book/ /tmp/ - - ls -la /tmp - - ls -la /tmp/doc/ - - ls -la /tmp/book/ - - cp ./README.md /tmp/doc/ + - cp README.md /tmp + - cp index.html /tmp # setup ssh - eval $(ssh-agent) - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} @@ -896,8 +894,14 @@ publish-rustdoc: - git checkout gh-pages # Remove everything and restore generated docs and README - rm -rf ./* - - mv /tmp/doc . - - mv /tmp/book . + # dir for rustdoc + - mkdir -p doc + # dir for implementors guide + - mkdir -p book + - mv /tmp/doc/* doc/ + - mv /tmp/book/html/* book/ + - mv /tmp/index.html . + - mv /tmp/README.md . # Upload files - git add --all --force # `git commit` has an exit code of > 0 if there is nothing to commit. From 81dd839df13c794338c28bb388d7b14d2be12c18 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 14:03:22 +0200 Subject: [PATCH 16/18] fix script order --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30df33d4b355..2985788caf01 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -878,8 +878,6 @@ publish-rustdoc: # Save README and docs - cp -r ./crate-docs/ /tmp/doc/ - cp -r ./artifacts/book/ /tmp/ - - cp README.md /tmp - - cp index.html /tmp # setup ssh - eval $(ssh-agent) - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} @@ -893,6 +891,8 @@ publish-rustdoc: - git fetch origin gh-pages - git checkout gh-pages # Remove everything and restore generated docs and README + - cp index.html /tmp + - cp README.md /tmp - rm -rf ./* # dir for rustdoc - mkdir -p doc From 5eb944a5c94fb9bc5a24ae52d7010f2d22e4d9f9 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 14:30:00 +0200 Subject: [PATCH 17/18] return pipeline --- .gitlab-ci.yml | 1409 ++++++++++++++++++++++++------------------------ 1 file changed, 703 insertions(+), 706 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2985788caf01..a4d234d4a5a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,657 +167,393 @@ default: #### stage: stage1 -# check-runtime: -# stage: stage1 -# image: paritytech/tools:latest -# <<: *kubernetes-env -# <<: *test-refs -# variables: -# GITLAB_API: "https://gitlab.parity.io/api/v4" -# GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" -# script: -# - ./scripts/ci/gitlab/check_runtime.sh -# allow_failure: true - -# cargo-fmt: -# stage: stage1 -# <<: *docker-env -# <<: *test-refs -# script: -# - cargo +nightly --version -# - cargo +nightly fmt --all -- --check -# allow_failure: true - -# build-linux-stable: -# stage: stage1 -# <<: *docker-env -# <<: *compiler-info -# <<: *collect-artifacts -# <<: *common-refs -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# # Ensure we run the UI tests. -# RUN_UI_TESTS: 1 -# script: -# - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name -# - mv ./target/testnet/polkadot ./artifacts/. -# - pushd artifacts -# - sha256sum polkadot | tee polkadot.sha256 -# - shasum -c polkadot.sha256 -# - popd -# - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" -# - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" -# - echo -n ${VERSION} > ./artifacts/VERSION -# - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG -# - cp -r scripts/* ./artifacts - -# test-linux-stable: -# stage: stage1 -# <<: *docker-env -# <<: *compiler-info -# <<: *common-refs -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# script: -# - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics - - - -# spellcheck: -# stage: stage1 -# <<: *docker-env -# <<: *test-refs -# script: -# - cargo spellcheck --version -# # compare with the commit parent to the PR, given it's from a default branch -# - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} -# - echo "___Spellcheck is going to check your diff___" -# - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) -# - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 -# $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) -# allow_failure: true - -# build-test-collators: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# script: -# - time cargo build --profile testnet --verbose -p test-parachain-adder-collator -# - time cargo build --profile testnet --verbose -p test-parachain-undying-collator -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/testnet/adder-collator ./artifacts/. -# - mv ./target/testnet/undying-collator ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# build-malus: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# script: -# - time cargo build --profile testnet --verbose -p polkadot-test-malus -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/testnet/malus ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# build-staking-miner: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *common-refs -# script: -# - time cargo build --locked --release --package staking-miner -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/release/staking-miner ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# #### stage: stage2 - -# .check-dependent-project: &check-dependent-project -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: check-runtime -# artifacts: false -# <<: *docker-env -# <<: *test-pr-refs -# script: -# - git clone -# --depth=1 -# "--branch=$PIPELINE_SCRIPTS_TAG" -# https://github.com/paritytech/pipeline-scripts -# - ./pipeline-scripts/check_dependent_project.sh -# --org paritytech -# --dependent-repo "$DEPENDENT_REPO" -# --github-api-token "$GITHUB_PR_TOKEN" -# --extra-dependencies "$EXTRA_DEPENDENCIES" -# --companion-overrides "$COMPANION_OVERRIDES" - -# check-dependent-cumulus: -# <<: *check-dependent-project -# variables: -# DEPENDENT_REPO: cumulus -# EXTRA_DEPENDENCIES: substrate -# COMPANION_OVERRIDES: | -# polkadot: release-v* -# cumulus: polkadot-v* - -# test-node-metrics: -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: check-runtime -# artifacts: false -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# script: -# - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics - -# test-deterministic-wasm: -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: cargo-fmt -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# - ./scripts/ci/gitlab/test_deterministic_wasm.sh - -# check-transaction-versions: -# stage: stage2 -# <<: *test-refs -# <<: *docker-env -# image: node:15 -# needs: -# - job: build-linux-stable -# artifacts: true -# before_script: -# - apt-get -y update; apt-get -y install jq lsof -# - npm install --ignore-scripts -g @polkadot/metadata-cmp -# # Set git config -# - git config remote.origin.url "https://github.com/paritytech/polkadot.git" -# - git fetch origin release -# script: -# - ./scripts/ci/gitlab/check_extrinsics_ordering.sh - - - -# # This image is used in testnets -# # Release image is handled by the Github Action here: -# # .github/workflows/publish-docker-release.yml -# publish-polkadot-debug-image: -# stage: stage2 -# <<: *build-push-image -# rules: -# # Don't run when triggered from another pipeline -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# - if: $CI_PIPELINE_SOURCE == "web" -# - if: $CI_PIPELINE_SOURCE == "schedule" -# - if: $CI_COMMIT_REF_NAME == "master" -# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile -# DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile -# IMAGE_NAME: docker.io/paritypr/polkadot-debug -# needs: -# - job: build-linux-stable -# artifacts: true -# after_script: -# # pass artifacts to the zombienet-tests job -# # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance -# - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env -# - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/parachains.env -# expire_in: 1 days - -# publish-test-collators-image: -# # service image for Simnet -# stage: stage2 -# <<: *build-push-image -# <<: *zombienet-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/collator_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/colander -# needs: -# - job: build-test-collators -# artifacts: true -# after_script: -# - buildah logout --all -# # pass artifacts to the zombienet-tests job -# - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env -# - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/collator.env - -# publish-malus-image: -# # service image for Simnet -# stage: stage2 -# <<: *build-push-image -# <<: *zombienet-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/malus_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/malus -# needs: -# - job: build-malus -# artifacts: true -# after_script: -# - buildah logout "$IMAGE_NAME" -# # pass artifacts to the zombienet-tests job -# - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env -# - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/malus.env - -# publish-staking-miner-image: -# stage: stage2 -# <<: *build-push-image -# <<: *publish-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# IMAGE_NAME: docker.io/paritytech/staking-miner -# GIT_STRATEGY: none -# DOCKER_USER: ${Docker_Hub_User_Parity} -# DOCKER_PASS: ${Docker_Hub_Pass_Parity} -# needs: -# - job: build-staking-miner -# artifacts: true - - -# publish-s3-release: &publish-s3 -# stage: stage3 -# needs: -# - job: build-linux-stable -# artifacts: true -# <<: *kubernetes-env -# image: paritytech/awscli:latest -# variables: -# GIT_STRATEGY: none -# PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" -# rules: -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# # publishing binaries nightly -# - if: $CI_PIPELINE_SOURCE == "schedule" -# before_script: -# - *check-versions -# script: -# - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" -# - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ -# - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" -# - find ./artifacts -type f | while read file; do -# name="${file#./artifacts/}"; -# aws s3api copy-object -# --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} -# --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; -# done -# - | -# cat <<-EOM -# | -# | polkadot binary paths: -# | -# | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot -# | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot -# | -# EOM -# after_script: -# - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ -# --recursive --human-readable --summarize - -# update_polkadot_weights: &update-weights -# stage: stage2 -# when: manual -# variables: -# RUNTIME: polkadot -# artifacts: -# paths: -# - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch -# script: -# - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME -# - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch -# # uses the "shell" executors -# tags: -# - weights - -# update_kusama_weights: -# <<: *update-weights -# variables: -# RUNTIME: kusama - -# update_westend_weights: -# <<: *update-weights -# variables: -# RUNTIME: westend - -# update_rococo_weights: -# <<: *update-weights -# variables: -# RUNTIME: rococo - -# #### stage: stage3 - - - - -# check-try-runtime: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-node-metrics -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# # Check that everything compiles with `try-runtime` feature flag. -# - cargo check --features try-runtime --all -# - sccache -s - -# check-no-default-features: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# # Check that polkadot-cli will compile no default features. -# - pushd ./node/service && cargo check --no-default-features && popd -# - pushd ./cli && cargo check --no-default-features --features "service" && popd -# - sccache -s - -# build-short-benchmark: -# stage: stage3 -# <<: *test-refs -# <<: *docker-env -# <<: *collect-artifacts -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-node-metrics -# artifacts: false -# script: -# - cargo +nightly build --profile release --locked --features=runtime-benchmarks -# - mkdir artifacts -# - cp ./target/release/polkadot ./artifacts/ - -# deploy-parity-testnet: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *deploy-testnet-refs -# variables: -# POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" -# POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" -# allow_failure: false -# trigger: "parity/infrastructure/parity-testnet" - -# zombienet-tests-parachains-smoke-test: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-parachains-smoke-test.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-parachains-pvf: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-parachains-pvf.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-parachains-disputes: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# - job: publish-malus-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0002-parachains-disputes.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-test-parachains-upgrade-smoke-test: -# stage: stage3 -# <<: *kubernetes-env -# <<: *zombienet-refs -# image: "docker.io/paritytech/zombienet:v1.2.45" -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' -# before_script: -# - echo "ZombieNet Tests Config" -# - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" -# - echo "docker.io/parity/polkadot-collator:latest" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0002-parachains-upgrade-smoke-test.feature" -# allow_failure: true -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-misc-paritydb: -# stage: stage3 -# <<: *kubernetes-env -# <<: *zombienet-refs -# image: "docker.io/paritytech/zombienet:v1.2.49" -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# artifacts: true -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-paritydb.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-malus-dispute-valid: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie* -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-dispute-valid-block.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-deregister-register-validator: -# stage: stage3 -# image: "docker.io/paritytech/zombienet:v1.2.47" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# artifacts: true -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie* -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0003-deregister-register-validator-smoke.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test +check-runtime: + stage: stage1 + image: paritytech/tools:latest + <<: *kubernetes-env + <<: *test-refs + variables: + GITLAB_API: "https://gitlab.parity.io/api/v4" + GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" + script: + - ./scripts/ci/gitlab/check_runtime.sh + allow_failure: true -#### stage: stage4 +cargo-fmt: + stage: stage1 + <<: *docker-env + <<: *test-refs + script: + - cargo +nightly --version + - cargo +nightly fmt --all -- --check + allow_failure: true -build-rustdoc: +build-linux-stable: + stage: stage1 + <<: *docker-env + <<: *compiler-info + <<: *collect-artifacts + <<: *common-refs + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + # Ensure we run the UI tests. + RUN_UI_TESTS: 1 + script: + - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name + - mv ./target/testnet/polkadot ./artifacts/. + - pushd artifacts + - sha256sum polkadot | tee polkadot.sha256 + - shasum -c polkadot.sha256 + - popd + - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" + - echo -n ${VERSION} > ./artifacts/VERSION + - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG + - cp -r scripts/* ./artifacts + +test-linux-stable: + stage: stage1 + <<: *docker-env + <<: *compiler-info + <<: *common-refs + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics + + + +spellcheck: + stage: stage1 + <<: *docker-env + <<: *test-refs + script: + - cargo spellcheck --version + # compare with the commit parent to the PR, given it's from a default branch + - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} + - echo "___Spellcheck is going to check your diff___" + - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 + $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + allow_failure: true + +build-test-collators: + stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *test-refs + script: + - time cargo build --profile testnet --verbose -p test-parachain-adder-collator + - time cargo build --profile testnet --verbose -p test-parachain-undying-collator + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/adder-collator ./artifacts/. + - mv ./target/testnet/undying-collator ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-malus: stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *test-refs + script: + - time cargo build --profile testnet --verbose -p polkadot-test-malus + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/malus ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-staking-miner: + stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *common-refs + script: + - time cargo build --locked --release --package staking-miner + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/release/staking-miner ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +#### stage: stage2 + +.check-dependent-project: &check-dependent-project + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-runtime + artifacts: false + <<: *docker-env + <<: *test-pr-refs + script: + - git clone + --depth=1 + "--branch=$PIPELINE_SCRIPTS_TAG" + https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/check_dependent_project.sh + --org paritytech + --dependent-repo "$DEPENDENT_REPO" + --github-api-token "$GITHUB_PR_TOKEN" + --extra-dependencies "$EXTRA_DEPENDENCIES" + --companion-overrides "$COMPANION_OVERRIDES" + +check-dependent-cumulus: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: cumulus + EXTRA_DEPENDENCIES: substrate + COMPANION_OVERRIDES: | + polkadot: release-v* + cumulus: polkadot-v* + +test-node-metrics: + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-runtime + artifacts: false + <<: *docker-env + <<: *compiler-info + <<: *test-refs + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics + +test-deterministic-wasm: + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: cargo-fmt + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + - ./scripts/ci/gitlab/test_deterministic_wasm.sh + +check-transaction-versions: + stage: stage2 + <<: *test-refs + <<: *docker-env + image: node:15 + needs: + - job: build-linux-stable + artifacts: true + before_script: + - apt-get -y update; apt-get -y install jq lsof + - npm install --ignore-scripts -g @polkadot/metadata-cmp + # Set git config + - git config remote.origin.url "https://github.com/paritytech/polkadot.git" + - git fetch origin release + script: + - ./scripts/ci/gitlab/check_extrinsics_ordering.sh + + + +# This image is used in testnets +# Release image is handled by the Github Action here: +# .github/workflows/publish-docker-release.yml +publish-polkadot-debug-image: + stage: stage2 + <<: *build-push-image + rules: + # Don't run when triggered from another pipeline + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + variables: + <<: *image-variables + # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile + DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile + IMAGE_NAME: docker.io/paritypr/polkadot-debug + needs: + - job: build-linux-stable + artifacts: true + after_script: + # pass artifacts to the zombienet-tests job + # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance + - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env + - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/parachains.env + expire_in: 1 days + +publish-test-collators-image: + # service image for Simnet + stage: stage2 + <<: *build-push-image + <<: *zombienet-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/collator_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/colander + needs: + - job: build-test-collators + artifacts: true + after_script: + - buildah logout --all + # pass artifacts to the zombienet-tests job + - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env + - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/collator.env + +publish-malus-image: + # service image for Simnet + stage: stage2 + <<: *build-push-image + <<: *zombienet-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/malus_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/malus + needs: + - job: build-malus + artifacts: true + after_script: + - buildah logout "$IMAGE_NAME" + # pass artifacts to the zombienet-tests job + - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env + - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/malus.env + +publish-staking-miner-image: + stage: stage2 + <<: *build-push-image + <<: *publish-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + IMAGE_NAME: docker.io/paritytech/staking-miner + GIT_STRATEGY: none + DOCKER_USER: ${Docker_Hub_User_Parity} + DOCKER_PASS: ${Docker_Hub_Pass_Parity} + needs: + - job: build-staking-miner + artifacts: true + + +publish-s3-release: &publish-s3 + stage: stage3 + needs: + - job: build-linux-stable + artifacts: true + <<: *kubernetes-env + image: paritytech/awscli:latest + variables: + GIT_STRATEGY: none + PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + # publishing binaries nightly + - if: $CI_PIPELINE_SOURCE == "schedule" + before_script: + - *check-versions + script: + - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" + - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ + - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" + - find ./artifacts -type f | while read file; do + name="${file#./artifacts/}"; + aws s3api copy-object + --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} + --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; + done + - | + cat <<-EOM + | + | polkadot binary paths: + | + | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot + | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot + | + EOM + after_script: + - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ + --recursive --human-readable --summarize + +update_polkadot_weights: &update-weights + stage: stage2 + when: manual + variables: + RUNTIME: polkadot + artifacts: + paths: + - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + script: + - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME + - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + # uses the "shell" executors + tags: + - weights + +update_kusama_weights: + <<: *update-weights + variables: + RUNTIME: kusama + +update_westend_weights: + <<: *update-weights + variables: + RUNTIME: westend + +update_rococo_weights: + <<: *update-weights + variables: + RUNTIME: rococo + +#### stage: stage3 + +build-rustdoc: + stage: stage3 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # needs: - # - job: test-deterministic-wasm - # artifacts: false + needs: + - job: test-deterministic-wasm + artifacts: false <<: *docker-env <<: *test-refs variables: @@ -839,11 +575,11 @@ build-rustdoc: - echo "" > ./crate-docs/index.html build-implementers-guide: - stage: stage1 + stage: stage3 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # needs: - # - job: test-deterministic-wasm - # artifacts: false + needs: + - job: test-deterministic-wasm + artifacts: false <<: *test-refs <<: *docker-env <<: *collect-artifacts-short @@ -855,6 +591,267 @@ build-implementers-guide: # FIXME: remove me after CI image gets nonroot - chown -R nonroot:nonroot artifacts/ +check-try-runtime: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-node-metrics + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + # Check that everything compiles with `try-runtime` feature flag. + - cargo check --features try-runtime --all + - sccache -s + +check-no-default-features: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + # Check that polkadot-cli will compile no default features. + - pushd ./node/service && cargo check --no-default-features && popd + - pushd ./cli && cargo check --no-default-features --features "service" && popd + - sccache -s + +build-short-benchmark: + stage: stage3 + <<: *test-refs + <<: *docker-env + <<: *collect-artifacts + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-node-metrics + artifacts: false + script: + - cargo +nightly build --profile release --locked --features=runtime-benchmarks + - mkdir artifacts + - cp ./target/release/polkadot ./artifacts/ + +deploy-parity-testnet: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *deploy-testnet-refs + variables: + POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" + POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" + allow_failure: false + trigger: "parity/infrastructure/parity-testnet" + +zombienet-tests-parachains-smoke-test: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-smoke-test.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-pvf: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-pvf.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-disputes: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-disputes.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-test-parachains-upgrade-smoke-test: + stage: stage3 + <<: *kubernetes-env + <<: *zombienet-refs + image: "docker.io/paritytech/zombienet:v1.2.45" + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' + before_script: + - echo "ZombieNet Tests Config" + - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" + - echo "docker.io/parity/polkadot-collator:latest" + - echo "${ZOMBIENET_IMAGE}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-upgrade-smoke-test.feature" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-misc-paritydb: + stage: stage3 + <<: *kubernetes-env + <<: *zombienet-refs + image: "docker.io/paritytech/zombienet:v1.2.49" + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-paritydb.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-malus-dispute-valid: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-dispute-valid-block.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-deregister-register-validator: + stage: stage3 + image: "docker.io/paritytech/zombienet:v1.2.47" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-deregister-register-validator-smoke.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +#### stage: stage4 + publish-rustdoc: stage: stage4 <<: *kubernetes-env @@ -862,11 +859,11 @@ publish-rustdoc: variables: GIT_DEPTH: 100 <<: *test-refs - # rules: - # - if: $CI_PIPELINE_SOURCE == "pipeline" - # when: never - # - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - # - if: $CI_COMMIT_REF_NAME == "master" + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "master" # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other # artifacts from being dowloaded by this job. needs: @@ -915,53 +912,53 @@ publish-rustdoc: after_script: - rm -rf .git/ ./* -# # Run all pallet benchmarks only once to check if there are any errors -# short-benchmark-polkadot: &short-bench -# stage: stage4 -# <<: *test-pr-refs -# <<: *docker-env -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: build-short-benchmark -# artifacts: true -# variables: -# RUNTIME: polkadot -# script: -# - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 1 --repeat 1 - -# short-benchmark-kusama: -# <<: *short-bench -# variables: -# RUNTIME: kusama - -# short-benchmark-westend: -# <<: *short-bench -# variables: -# RUNTIME: westend - -# #### stage: .post - -# # This job cancels the whole pipeline if any of provided jobs fail. -# # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests -# # to fail the pipeline as soon as possible to shorten the feedback loop. -# .cancel-pipeline-template: -# stage: .post -# rules: -# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -# when: on_failure -# variables: -# PROJECT_ID: "${CI_PROJECT_ID}" -# PIPELINE_ID: "${CI_PIPELINE_ID}" -# trigger: "parity/infrastructure/ci_cd/pipeline-stopper" - -# cancel-pipeline-test-linux-stable: -# extends: .cancel-pipeline-template -# needs: -# - job: test-linux-stable -# artifacts: false - -# cancel-pipeline-check-try-runtime: -# extends: .cancel-pipeline-template -# needs: -# - job: check-try-runtime -# artifacts: false +# Run all pallet benchmarks only once to check if there are any errors +short-benchmark-polkadot: &short-bench + stage: stage4 + <<: *test-pr-refs + <<: *docker-env + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-short-benchmark + artifacts: true + variables: + RUNTIME: polkadot + script: + - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 1 --repeat 1 + +short-benchmark-kusama: + <<: *short-bench + variables: + RUNTIME: kusama + +short-benchmark-westend: + <<: *short-bench + variables: + RUNTIME: westend + +#### stage: .post + +# This job cancels the whole pipeline if any of provided jobs fail. +# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# to fail the pipeline as soon as possible to shorten the feedback loop. +.cancel-pipeline-template: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: on_failure + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + trigger: "parity/infrastructure/ci_cd/pipeline-stopper" + +cancel-pipeline-test-linux-stable: + extends: .cancel-pipeline-template + needs: + - job: test-linux-stable + artifacts: false + +cancel-pipeline-check-try-runtime: + extends: .cancel-pipeline-template + needs: + - job: check-try-runtime + artifacts: false From 76966656ba24f345b74ae477ea6bfd01ebfac236 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Fri, 22 Jul 2022 14:31:32 +0200 Subject: [PATCH 18/18] remove test ref from publish --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4d234d4a5a7..7d7d9e7cbdc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -858,7 +858,6 @@ publish-rustdoc: image: paritytech/tools:latest variables: GIT_DEPTH: 100 - <<: *test-refs rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never