From 20d8f1db0be85e88d2db45cfdf02d7b75a37a34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 10:42:47 +0100 Subject: [PATCH 1/8] CI: Rewrite `check-each-crate` in python This is a much better readable version of the script and it should also work on Macos and not siltently fail ;) --- scripts/ci/gitlab/check-each-crate.py | 45 ++++++++++++++++++++++ scripts/ci/gitlab/check-each-crate.sh | 54 --------------------------- scripts/ci/gitlab/pipeline/test.yml | 4 +- 3 files changed, 47 insertions(+), 56 deletions(-) create mode 100755 scripts/ci/gitlab/check-each-crate.py delete mode 100755 scripts/ci/gitlab/check-each-crate.sh diff --git a/scripts/ci/gitlab/check-each-crate.py b/scripts/ci/gitlab/check-each-crate.py new file mode 100755 index 0000000000000..c954ec5bc8555 --- /dev/null +++ b/scripts/ci/gitlab/check-each-crate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +# A script that checks each workspace crate individually. +# It's relevant to check workspace crates individually because otherwise their compilation problems +# due to feature misconfigurations won't be caught, as exemplified by +# https://github.com/paritytech/substrate/issues/12705 + +import subprocess, sys + +# Get all crates +output = subprocess.check_output(["cargo", "tree", "--workspace", "--depth", "0", "--prefix", "none"]) + +# Convert the output into a proper list +crates = [] +for line in output.splitlines(): + if line != b"": + crates.append(line.decode('utf8').split(" ")[0]) + +# Make the list unique and sorted +crates = list(set(crates)) +crates.sort() + +target_group = int(sys.argv[1]) - 1 +groups_total = int(sys.argv[2]) + +if len(crates) == 0: + print("No crates detected!") + sys.exit(1) + +print(f"Total crates: {len(crates)}") + +crates_per_group = (len(crates) + groups_total - 1) // groups_total + +print(f"Crates per group: {crates_per_group}") + +# Check each crate +for i in range(0, crates_per_group): + print(f"Checking {crates[crates_per_group * target_group + i]}") + + res = subprocess.run( + ["cargo", "check", "--locked", "-p", crates[crates_per_group * target_group + i]], + ) + + if res.returncode != 0: + sys.exit(1) diff --git a/scripts/ci/gitlab/check-each-crate.sh b/scripts/ci/gitlab/check-each-crate.sh deleted file mode 100755 index 24cad67007e73..0000000000000 --- a/scripts/ci/gitlab/check-each-crate.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -## A script that checks each workspace crate individually. -## It's relevant to check workspace crates individually because otherwise their compilation problems -## due to feature misconfigurations won't be caught, as exemplified by -## https://github.com/paritytech/substrate/issues/12705 - -set -Eeu -o pipefail -shopt -s inherit_errexit - -set -vx - -target_group="$1" -groups_total="$2" - -readarray -t workspace_crates < <(\ - cargo tree --workspace --depth 0 --prefix none | - awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }' | - sort | - uniq -) - -crates_total=${#workspace_crates[*]} -if [ "$crates_total" -lt 1 ]; then - >&2 echo "No crates detected for $PWD" - exit 1 -fi - -if [ "$crates_total" -lt "$groups_total" ]; then - # `crates_total / groups_total` would result in 0, so round it up to 1 - crates_per_group=1 -else - # We add `crates_total % groups_total > 0` (which evaluates to 1 in case there's a remainder for - # `crates_total % groups_total`) to round UP `crates_total / groups_total` 's - # potentially-fractional result to the nearest integer. Doing that ensures that we'll not miss any - # crate in case `crates_total / groups_total` would normally result in a fractional number, since - # in those cases Bash would round DOWN the result to the nearest integer. For example, if - # `crates_total = 5` and `groups_total = 2`, then `crates_total / groups_total` would round down - # to 2; since the loop below would then step by 2, we'd miss the 5th crate. - crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) )) -fi - -group=1 -for ((i=0; i < crates_total; i += crates_per_group)); do - if [ $group -eq "$target_group" ]; then - crates_in_group=("${workspace_crates[@]:$i:$crates_per_group}") - echo "crates in the group: ${crates_in_group[*]}" >/dev/null # >/dev/null due to "set -x" - for crate in "${crates_in_group[@]}"; do - cargo check --locked --release -p "$crate" - done - break - fi - group=$(( group + 1 )) -done diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index f00857ffa9935..28bc238bbb9d8 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -411,7 +411,7 @@ cargo-check-each-crate: CI_JOB_NAME: cargo-check-each-crate script: - rusty-cachier snapshot create - - time ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL" + - time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL" # need to update cache only from one job - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi parallel: 2 @@ -449,6 +449,6 @@ cargo-check-each-crate-macos: script: # TODO: enable rusty-cachier once it supports Mac # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available - - time ./scripts/ci/gitlab/check-each-crate.sh 1 1 + - time ./scripts/ci/gitlab/check-each-crate.py 1 1 tags: - osx From ee8ba350a3b6f6c31d8c88026366699bfbefb463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 12:04:42 +0100 Subject: [PATCH 2/8] Fix dumb bugs and print everything to stderr --- scripts/ci/gitlab/check-each-crate.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/ci/gitlab/check-each-crate.py b/scripts/ci/gitlab/check-each-crate.py index c954ec5bc8555..cc33dc4b09ea2 100755 --- a/scripts/ci/gitlab/check-each-crate.py +++ b/scripts/ci/gitlab/check-each-crate.py @@ -4,6 +4,11 @@ # It's relevant to check workspace crates individually because otherwise their compilation problems # due to feature misconfigurations won't be caught, as exemplified by # https://github.com/paritytech/substrate/issues/12705 +# +# `check-each-crate.py target_group groups_total` +# +# - `target_group`: Integer starting from 1, the group this script should execute. +# - `groups_total`: Integer starting from 1, total number of groups. import subprocess, sys @@ -24,22 +29,25 @@ groups_total = int(sys.argv[2]) if len(crates) == 0: - print("No crates detected!") + print("No crates detected!", file=sys.stderr) sys.exit(1) -print(f"Total crates: {len(crates)}") +print(f"Total crates: {len(crates)}", file=sys.stderr) -crates_per_group = (len(crates) + groups_total - 1) // groups_total +crates_per_group = len(crates) // groups_total -print(f"Crates per group: {crates_per_group}") +print(f"Crates per group: {crates_per_group}", file=sys.stderr) # Check each crate for i in range(0, crates_per_group): - print(f"Checking {crates[crates_per_group * target_group + i]}") + crate = crates_per_group * target_group + i - res = subprocess.run( - ["cargo", "check", "--locked", "-p", crates[crates_per_group * target_group + i]], - ) + if crate >= len(crates): + break + + print(f"Checking {crates[crate]}", file=sys.stderr) + + res = subprocess.run(["cargo", "check", "--locked", "-p", crates[crate]]) if res.returncode != 0: sys.exit(1) From c4b4bfc239f11076e84e99f8fdf2078fceb1da7c Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Wed, 25 Jan 2023 15:10:08 +0400 Subject: [PATCH 3/8] Don't buffer Python output --- scripts/ci/gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 28bc238bbb9d8..26113f6e797e9 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -411,7 +411,7 @@ cargo-check-each-crate: CI_JOB_NAME: cargo-check-each-crate script: - rusty-cachier snapshot create - - time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL" + - PYTHONUNBUFFERED=x time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL" # need to update cache only from one job - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi parallel: 2 From 1b1fa84535d33d48fe905cf6987ae96847c01317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 14:29:35 +0100 Subject: [PATCH 4/8] :facepalm: --- scripts/ci/gitlab/check-each-crate.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/ci/gitlab/check-each-crate.py b/scripts/ci/gitlab/check-each-crate.py index cc33dc4b09ea2..37302afb8df09 100755 --- a/scripts/ci/gitlab/check-each-crate.py +++ b/scripts/ci/gitlab/check-each-crate.py @@ -36,15 +36,19 @@ crates_per_group = len(crates) // groups_total +# If this is the last runner, we need to take care of crates +# after the group that we lost because of the integer division. +if target_group + 1 == groups_total: + overflow_crates = len(crates) % groups_total +else: + overflow_crates = 0 + print(f"Crates per group: {crates_per_group}", file=sys.stderr) # Check each crate -for i in range(0, crates_per_group): +for i in range(0, crates_per_group + rest_crates): crate = crates_per_group * target_group + i - if crate >= len(crates): - break - print(f"Checking {crates[crate]}", file=sys.stderr) res = subprocess.run(["cargo", "check", "--locked", "-p", crates[crate]]) From c96e997d5b6c8fddc1a67ba7c87a942719aa6e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 14:55:15 +0100 Subject: [PATCH 5/8] :facepalm: :facepalm: --- scripts/ci/gitlab/check-each-crate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/check-each-crate.py b/scripts/ci/gitlab/check-each-crate.py index 37302afb8df09..04df4befaf429 100755 --- a/scripts/ci/gitlab/check-each-crate.py +++ b/scripts/ci/gitlab/check-each-crate.py @@ -46,7 +46,7 @@ print(f"Crates per group: {crates_per_group}", file=sys.stderr) # Check each crate -for i in range(0, crates_per_group + rest_crates): +for i in range(0, crates_per_group + overflow_crates): crate = crates_per_group * target_group + i print(f"Checking {crates[crate]}", file=sys.stderr) From 16d5d6dba2fdb276129e7f43167cdb22726fb8a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 20:29:01 +0100 Subject: [PATCH 6/8] Use check all until we have more macos runners --- scripts/ci/gitlab/pipeline/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 26113f6e797e9..838f7d51f7161 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -449,6 +449,7 @@ cargo-check-each-crate-macos: script: # TODO: enable rusty-cachier once it supports Mac # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available - - time ./scripts/ci/gitlab/check-each-crate.py 1 1 + # - time ./scripts/ci/gitlab/check-each-crate.py 1 1 + - time cargo check --all --locked tags: - osx From 5fbfbe7aebfa5d1230ddee37345d86a3a01dc138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 22:41:57 +0100 Subject: [PATCH 7/8] Update scripts/ci/gitlab/check-each-crate.py Co-authored-by: Oliver Tale-Yazdi --- scripts/ci/gitlab/check-each-crate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/check-each-crate.py b/scripts/ci/gitlab/check-each-crate.py index 04df4befaf429..adad4f5bd5835 100755 --- a/scripts/ci/gitlab/check-each-crate.py +++ b/scripts/ci/gitlab/check-each-crate.py @@ -13,7 +13,7 @@ import subprocess, sys # Get all crates -output = subprocess.check_output(["cargo", "tree", "--workspace", "--depth", "0", "--prefix", "none"]) +output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"]) # Convert the output into a proper list crates = [] From d793016a0447b7133f0971528ac417ccfa2f4cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Jan 2023 22:42:39 +0100 Subject: [PATCH 8/8] Update scripts/ci/gitlab/pipeline/test.yml --- scripts/ci/gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 838f7d51f7161..02e05752fd01f 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -450,6 +450,6 @@ cargo-check-each-crate-macos: # TODO: enable rusty-cachier once it supports Mac # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available # - time ./scripts/ci/gitlab/check-each-crate.py 1 1 - - time cargo check --all --locked + - time cargo check --workspace --locked tags: - osx