From 0c15da4024c11cb0808dcc574b10a8bdcb3037a3 Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 3 Jan 2023 13:37:38 +0100 Subject: [PATCH 1/8] A0-1619 New GH check for runtime metadata --- .github/actions/run-e2e-test/action.yml | 5 +++- .github/workflows/e2e-tests-main-devnet.yml | 27 ++++++++++++++++++--- aleph-client/.dockerignore | 4 +++ aleph-client/docker/Dockerfile | 15 ++++++++++++ aleph-client/docker/docker_entrypoint.sh | 13 ++++++++++ aleph-client/rustfmt.toml | 7 ++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 aleph-client/.dockerignore create mode 100644 aleph-client/docker/Dockerfile create mode 100644 aleph-client/docker/docker_entrypoint.sh create mode 100644 aleph-client/rustfmt.toml diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index f066979c56..92f08da54b 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -12,7 +12,7 @@ inputs: default: '4' test-case: description: 'Name of test to run.' - required: true + required: false randomized: description: 'Whether to use randomized test params.' required: false @@ -57,15 +57,18 @@ runs: run: docker logs Node0 --follow & - name: Download artifact with the test suite image + if: "${{ github.event.inputs.test-case != '' }}" uses: actions/download-artifact@v2 with: name: aleph-e2e-client - name: Load test suite docker image + if: "${{ github.event.inputs.test-case != '' }}" shell: bash run: docker load -i aleph-e2e-client.tar - name: Run single e2e test + if: "${{ github.event.inputs.test-case != '' }}" shell: bash run: | ARGS=( diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index 839b64b956..0d00bab444 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -68,7 +68,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.6.1' + version: '3.6.1' - name: Restore cache uses: ./.github/actions/restore-cache @@ -115,7 +115,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.6.1' + version: '3.6.1' - name: Install WASM target run: rustup target add wasm32-unknown-unknown @@ -153,7 +153,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.6.1' + version: '3.6.1' - name: Restore cache uses: ./.github/actions/restore-cache @@ -182,6 +182,25 @@ jobs: if-no-files-found: error retention-days: 7 + run-aleph-client-subxt-codegen-check: + needs: [build-test-docker, build-test-client] + name: Checks if runtime file in aleph-client is up-to-date + runs-on: ubuntu-20.04 + steps: + - name: Checkout source code + uses: actions/checkout@v2 + + - name: Run one node in the background + uses: ./.github/actions/run-e2e-test + with: + node-count: 1 + min-validator-count: 1 + + - name: check if runtime metadata matches + run: | + cd aleph-client/ + docker build --tag subxt:latest -f docker/Dockerfile . + docker run -it --rm --network host subxt:latest run-e2e-finalization-test: needs: [build-test-docker, build-test-client] @@ -839,7 +858,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.6.1' + version: '3.6.1' - name: Install WASM target run: rustup target add wasm32-unknown-unknown diff --git a/aleph-client/.dockerignore b/aleph-client/.dockerignore new file mode 100644 index 0000000000..3c2f8ba28f --- /dev/null +++ b/aleph-client/.dockerignore @@ -0,0 +1,4 @@ +** +!src/aleph_zero.rs +!docker/docker_entrypoint.sh +!rustfmt.toml diff --git a/aleph-client/docker/Dockerfile b/aleph-client/docker/Dockerfile new file mode 100644 index 0000000000..4c026cc9bc --- /dev/null +++ b/aleph-client/docker/Dockerfile @@ -0,0 +1,15 @@ +FROM rustlang/rust:nightly-slim + +WORKDIR subxt + +RUN cargo install subxt-cli +RUN rustup component add rustfmt --toolchain nightly + +COPY docker/docker_entrypoint.sh /subxt/docker_entrypoint.sh +COPY src/aleph_zero.rs /subxt/aleph_zero_current.rs +COPY rustfmt.toml /subxt/rustfmt.toml + +RUN chmod +x /subxt/docker_entrypoint.sh +RUN rustc --version + +ENTRYPOINT ["./docker_entrypoint.sh"] diff --git a/aleph-client/docker/docker_entrypoint.sh b/aleph-client/docker/docker_entrypoint.sh new file mode 100644 index 0000000000..10f7b8974c --- /dev/null +++ b/aleph-client/docker/docker_entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +subxt codegen --derive Clone --derive Debug --derive Eq --derive PartialEq | rustfmt --edition=2021 > aleph_zero.rs +diff -y --suppress-common-lines aleph_zero.rs aleph_zero_current.rs +diff_exit_code=$? +if [[ ! $diff_exit_code -eq 0 ]]; then + echo "Current runtime metadata is different than versioned in git!" + echo "Run subxt codegen --derive Clone --derive Debug --derive Eq --derive PartialEq | rustfmt --edition=2021 >" \ +"src/aleph_zero.rs from aleph-client directory and commit to git." + exit 1 +fi +echo "Current runtime metadata and versioned in git matches." diff --git a/aleph-client/rustfmt.toml b/aleph-client/rustfmt.toml new file mode 100644 index 0000000000..af85000b72 --- /dev/null +++ b/aleph-client/rustfmt.toml @@ -0,0 +1,7 @@ +edition = "2021" +use_field_init_shorthand = true +reorder_modules = true + +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +reorder_imports = true From b921c979a837161cb36bc2b109cde22ca338313a Mon Sep 17 00:00:00 2001 From: Marcin Date: Tue, 3 Jan 2023 14:52:16 +0100 Subject: [PATCH 2/8] A0-1619 Removed interactive mode from the new runtime check. Fixed steps dependencies. --- .github/workflows/e2e-tests-main-devnet.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index 0d00bab444..a7659f3c73 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -183,7 +183,7 @@ jobs: retention-days: 7 run-aleph-client-subxt-codegen-check: - needs: [build-test-docker, build-test-client] + needs: [build-test-docker] name: Checks if runtime file in aleph-client is up-to-date runs-on: ubuntu-20.04 steps: @@ -200,7 +200,7 @@ jobs: run: | cd aleph-client/ docker build --tag subxt:latest -f docker/Dockerfile . - docker run -it --rm --network host subxt:latest + docker run --rm --network host subxt:latest run-e2e-finalization-test: needs: [build-test-docker, build-test-client] From 15386e72b41960bc55076641125b68ede5f16a8e Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 4 Jan 2023 10:51:14 +0100 Subject: [PATCH 3/8] A0-1619 Attempt to fix workflows. --- .github/actions/run-e2e-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 92f08da54b..82754a5dbd 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -57,18 +57,18 @@ runs: run: docker logs Node0 --follow & - name: Download artifact with the test suite image - if: "${{ github.event.inputs.test-case != '' }}" + if: github.event.inputs.test-case != '' uses: actions/download-artifact@v2 with: name: aleph-e2e-client - name: Load test suite docker image - if: "${{ github.event.inputs.test-case != '' }}" + if: github.event.inputs.test-case != '' shell: bash run: docker load -i aleph-e2e-client.tar - name: Run single e2e test - if: "${{ github.event.inputs.test-case != '' }}" + if: github.event.inputs.test-case != '' shell: bash run: | ARGS=( From 7fbd118adad7bd276edb640626dd8900e2d00ef4 Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 4 Jan 2023 11:39:27 +0100 Subject: [PATCH 4/8] A0-1619 use pre-build subxt image --- .github/workflows/e2e-tests-main-devnet.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index a7659f3c73..b85aaa30a2 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -198,9 +198,9 @@ jobs: - name: check if runtime metadata matches run: | - cd aleph-client/ - docker build --tag subxt:latest -f docker/Dockerfile . - docker run --rm --network host subxt:latest + # see Dockerfile in aleph-client/Dockerfile for reference + docker pull public.ecr.aws/p6e8q1z1/subxt-client-integration:latest + docker run --rm --network host subxt-client-integration:latest run-e2e-finalization-test: needs: [build-test-docker, build-test-client] From d2673e114589e8f180d6b80c5a1cb563d860022a Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 4 Jan 2023 12:41:30 +0100 Subject: [PATCH 5/8] A0-1619 Fixed docker entrypoint. Regenerated runtime metadata as it was outdated. --- aleph-client/docker/docker_entrypoint.sh | 1 - aleph-client/src/aleph_zero.rs | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/aleph-client/docker/docker_entrypoint.sh b/aleph-client/docker/docker_entrypoint.sh index 10f7b8974c..dd761c6822 100644 --- a/aleph-client/docker/docker_entrypoint.sh +++ b/aleph-client/docker/docker_entrypoint.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -euo pipefail subxt codegen --derive Clone --derive Debug --derive Eq --derive PartialEq | rustfmt --edition=2021 > aleph_zero.rs diff -y --suppress-common-lines aleph_zero.rs aleph_zero_current.rs diff --git a/aleph-client/src/aleph_zero.rs b/aleph-client/src/aleph_zero.rs index 96dc79aca0..c3d5c0f443 100644 --- a/aleph-client/src/aleph_zero.rs +++ b/aleph-client/src/aleph_zero.rs @@ -6409,10 +6409,9 @@ pub mod api { "FinalityVersion", vec![], [ - 134u8, 19u8, 94u8, 247u8, 125u8, 18u8, 148u8, 160u8, 167u8, 235u8, - 174u8, 4u8, 107u8, 69u8, 55u8, 187u8, 249u8, 13u8, 129u8, 99u8, 116u8, - 158u8, 38u8, 29u8, 239u8, 112u8, 150u8, 92u8, 151u8, 197u8, 223u8, - 30u8, + 99u8, 158u8, 103u8, 180u8, 128u8, 32u8, 84u8, 110u8, 229u8, 2u8, 3u8, + 114u8, 95u8, 125u8, 230u8, 210u8, 56u8, 85u8, 38u8, 136u8, 49u8, 206u8, + 6u8, 136u8, 193u8, 164u8, 251u8, 60u8, 125u8, 91u8, 205u8, 144u8, ], ) } @@ -19533,9 +19532,9 @@ pub mod api { let runtime_metadata_hash = client.metadata().metadata_hash(&PALLETS); if runtime_metadata_hash != [ - 155u8, 236u8, 207u8, 138u8, 149u8, 114u8, 96u8, 168u8, 143u8, 247u8, 148u8, 106u8, - 148u8, 203u8, 48u8, 129u8, 14u8, 48u8, 155u8, 234u8, 78u8, 212u8, 73u8, 65u8, - 212u8, 201u8, 174u8, 194u8, 72u8, 70u8, 240u8, 233u8, + 10u8, 121u8, 157u8, 11u8, 147u8, 107u8, 235u8, 73u8, 90u8, 254u8, 82u8, 183u8, + 112u8, 64u8, 213u8, 99u8, 23u8, 17u8, 10u8, 91u8, 124u8, 231u8, 209u8, 172u8, 59u8, + 160u8, 15u8, 142u8, 149u8, 200u8, 95u8, 164u8, ] { Err(::subxt::error::MetadataError::IncompatibleMetadata) From b9f3968b4d55556f4c712835fd245b9be8edd968 Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 4 Jan 2023 12:55:00 +0100 Subject: [PATCH 6/8] A0-1619 Attempt 4 to fix workflows --- .github/actions/run-e2e-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml index 82754a5dbd..c45467baa5 100644 --- a/.github/actions/run-e2e-test/action.yml +++ b/.github/actions/run-e2e-test/action.yml @@ -57,18 +57,18 @@ runs: run: docker logs Node0 --follow & - name: Download artifact with the test suite image - if: github.event.inputs.test-case != '' + if: inputs.test-case != '' uses: actions/download-artifact@v2 with: name: aleph-e2e-client - name: Load test suite docker image - if: github.event.inputs.test-case != '' + if: inputs.test-case != '' shell: bash run: docker load -i aleph-e2e-client.tar - name: Run single e2e test - if: github.event.inputs.test-case != '' + if: inputs.test-case != '' shell: bash run: | ARGS=( From 901b7f0d6ce14f5e5a9dcb4c33e9e812090d17fd Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 4 Jan 2023 13:52:45 +0100 Subject: [PATCH 7/8] A0-1619 Attempt 5 to fix workflows --- .github/workflows/e2e-tests-main-devnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index b85aaa30a2..3cd9c12d9a 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -200,7 +200,7 @@ jobs: run: | # see Dockerfile in aleph-client/Dockerfile for reference docker pull public.ecr.aws/p6e8q1z1/subxt-client-integration:latest - docker run --rm --network host subxt-client-integration:latest + docker run --rm --network host public.ecr.aws/p6e8q1z1/subxt-client-integration:latest run-e2e-finalization-test: needs: [build-test-docker, build-test-client] From 65233b4b8200cf54927a9e518e1f43b9acb3549d Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 5 Jan 2023 09:37:26 +0100 Subject: [PATCH 8/8] A0-1619 Hopefully better attempt to the Dockerfile --- .github/workflows/e2e-tests-main-devnet.yml | 4 ++-- aleph-client/.dockerignore | 4 +--- aleph-client/docker/Dockerfile | 15 --------------- aleph-client/docker/README.md | 18 ++++++++++++++++++ ...oint.sh => subxt-integration-entrypoint.sh} | 4 ++-- .../docker/subxt-integration.Dockerfile | 13 +++++++++++++ 6 files changed, 36 insertions(+), 22 deletions(-) delete mode 100644 aleph-client/docker/Dockerfile create mode 100644 aleph-client/docker/README.md rename aleph-client/docker/{docker_entrypoint.sh => subxt-integration-entrypoint.sh} (71%) create mode 100644 aleph-client/docker/subxt-integration.Dockerfile diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index 3cd9c12d9a..9c75f40971 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -198,9 +198,9 @@ jobs: - name: check if runtime metadata matches run: | - # see Dockerfile in aleph-client/Dockerfile for reference + cd aleph-client docker pull public.ecr.aws/p6e8q1z1/subxt-client-integration:latest - docker run --rm --network host public.ecr.aws/p6e8q1z1/subxt-client-integration:latest + docker run --rm --network host --mount type=bind,source="$(pwd)/..",target=/subxt/aleph-node public.ecr.aws/p6e8q1z1/subxt-client-integration:latest run-e2e-finalization-test: needs: [build-test-docker, build-test-client] diff --git a/aleph-client/.dockerignore b/aleph-client/.dockerignore index 3c2f8ba28f..11e58e6c04 100644 --- a/aleph-client/.dockerignore +++ b/aleph-client/.dockerignore @@ -1,4 +1,2 @@ ** -!src/aleph_zero.rs -!docker/docker_entrypoint.sh -!rustfmt.toml +!docker/subxt-integration-entrypoint.sh diff --git a/aleph-client/docker/Dockerfile b/aleph-client/docker/Dockerfile deleted file mode 100644 index 4c026cc9bc..0000000000 --- a/aleph-client/docker/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM rustlang/rust:nightly-slim - -WORKDIR subxt - -RUN cargo install subxt-cli -RUN rustup component add rustfmt --toolchain nightly - -COPY docker/docker_entrypoint.sh /subxt/docker_entrypoint.sh -COPY src/aleph_zero.rs /subxt/aleph_zero_current.rs -COPY rustfmt.toml /subxt/rustfmt.toml - -RUN chmod +x /subxt/docker_entrypoint.sh -RUN rustc --version - -ENTRYPOINT ["./docker_entrypoint.sh"] diff --git a/aleph-client/docker/README.md b/aleph-client/docker/README.md new file mode 100644 index 0000000000..207b432454 --- /dev/null +++ b/aleph-client/docker/README.md @@ -0,0 +1,18 @@ +This directory contains following files: +### `subxt-integration.Dockerfile` +This is not a main `aleph-client`, rather it is a helper Dockerfile to run on GH, which has `subxt` tool. + +It requires: +* an `aleph-node` chain to be run in the background (ie `127.0.0.1:9944` port must be opened), +* access to `rustfmt.toml`, +* access to current `aleph_zero.rs` file + +The docker checks whether a `subxt`-generated runtime metadata is the same as from the current commit. + +It needs to be run only from `aleph-client` directory and in network host mode: +```bash + docker run --network host --mount type=bind,source="$(pwd)/..",target=/subxt/aleph-node subxt:latest +``` + +### `subxt-integration-entrypoint.sh` +An entrypoint for above Dockerfile diff --git a/aleph-client/docker/docker_entrypoint.sh b/aleph-client/docker/subxt-integration-entrypoint.sh similarity index 71% rename from aleph-client/docker/docker_entrypoint.sh rename to aleph-client/docker/subxt-integration-entrypoint.sh index dd761c6822..947010e7a5 100644 --- a/aleph-client/docker/docker_entrypoint.sh +++ b/aleph-client/docker/subxt-integration-entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -subxt codegen --derive Clone --derive Debug --derive Eq --derive PartialEq | rustfmt --edition=2021 > aleph_zero.rs -diff -y --suppress-common-lines aleph_zero.rs aleph_zero_current.rs +subxt codegen --derive Clone --derive Debug --derive Eq --derive PartialEq | rustfmt --edition=2021 --config-path aleph-node/rustfmt.toml > aleph_zero.rs +diff -y -W 200 --suppress-common-lines aleph_zero.rs aleph-node/aleph-client/src/aleph_zero.rs diff_exit_code=$? if [[ ! $diff_exit_code -eq 0 ]]; then echo "Current runtime metadata is different than versioned in git!" diff --git a/aleph-client/docker/subxt-integration.Dockerfile b/aleph-client/docker/subxt-integration.Dockerfile new file mode 100644 index 0000000000..54c2049df7 --- /dev/null +++ b/aleph-client/docker/subxt-integration.Dockerfile @@ -0,0 +1,13 @@ +FROM rustlang/rust:nightly-slim + +WORKDIR subxt + +RUN cargo install subxt-cli +RUN rustup component add rustfmt --toolchain nightly + +COPY docker/subxt-integration-entrypoint.sh /subxt/subxt-integration-entrypoint.sh + +RUN chmod +x /subxt/subxt-integration-entrypoint.sh +RUN rustc --version + +ENTRYPOINT ["./subxt-integration-entrypoint.sh"]