From 66a18a4a14ddd65f8e0a1fafeaf9baa5deac3398 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Fri, 28 Jun 2024 15:12:05 +0900 Subject: [PATCH 1/5] Fix false negatives in dcou by using patched cargo --- scripts/check-dev-context-only-utils.sh | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 65382d20663e46..8ccb79a58ea201 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -157,7 +157,34 @@ fi export RUSTFLAGS="-D warnings -Z threads=8 $RUSTFLAGS" if [[ $mode = "check-bins" || $mode = "full" ]]; then - _ cargo "+${rust_nightly}" hack check --bins + # Until https://github.com/rust-lang/cargo/pull/14163 is applied for our + # chosen nightly toolchain, dcou needs custom-built cargo to avoid + # false-negatives. + + # This clone will fail every time we update our nightly toolchain because + # ${rust_nightly} is embedded in the branch name. This is intentional to + # force to re-cheery-pick the patch for the new nightly. + # + # Steps: + # 1. Clone the forked cargo repo. + # 2. Check out the commit hash printed by `/solana/cargo nightly --version + # --verbose` _using the new toolchain_. + # 3. Cherry-pick old branch's tip commit titled as "Unify no-library-target + # error into no-target warn". + # 4. Push new branch with its name updated with the new nightly version. + _ git clone --depth 1 --no-tags \ + --branch "no-no-library-target-error-${rust_nightly}" \ + https://github.com/anza-xyz/cargo.git cargo-for-dcou + + # Now build the patched cargo + ( + cd ./cargo-for-dcou/ + _ cargo "+${rust_nightly}" build --release --bin cargo + ) + + # Use `cargo "+${rust_nightly}" hack ..` once we stop using custom-built one. + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib --bins fi if [[ $mode = "check-all-targets" || $mode = "full" ]]; then _ cargo "+${rust_nightly}" hack check --all-targets From 4b0e10bb896ce6e50dd3cd0330081136f60293a1 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 9 Jul 2024 09:17:23 +0900 Subject: [PATCH 2/5] Separate cargo checks --- scripts/check-dev-context-only-utils.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 8ccb79a58ea201..795be699373876 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -184,7 +184,9 @@ if [[ $mode = "check-bins" || $mode = "full" ]]; then # Use `cargo "+${rust_nightly}" hack ..` once we stop using custom-built one. PATH="./cargo-for-dcou/target/release:$PATH" \ - RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib --bins + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins fi if [[ $mode = "check-all-targets" || $mode = "full" ]]; then _ cargo "+${rust_nightly}" hack check --all-targets From ceda83ef1ffa273d953d21b8737848810c6b0f1f Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 9 Jul 2024 12:56:25 +0900 Subject: [PATCH 3/5] Update check-dev-context-only-utils.sh --- scripts/check-dev-context-only-utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 795be699373876..909ddabbf3ff9a 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -183,10 +183,10 @@ if [[ $mode = "check-bins" || $mode = "full" ]]; then ) # Use `cargo "+${rust_nightly}" hack ..` once we stop using custom-built one. - PATH="./cargo-for-dcou/target/release:$PATH" \ - RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib PATH="./cargo-for-dcou/target/release:$PATH" \ RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib fi if [[ $mode = "check-all-targets" || $mode = "full" ]]; then _ cargo "+${rust_nightly}" hack check --all-targets From 3c4086c786019e2a2390cf526567367d9dd8d047 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 9 Jul 2024 13:28:15 +0900 Subject: [PATCH 4/5] seperate jobs --- ci/buildkite-pipeline.sh | 1 + scripts/check-dev-context-only-utils.sh | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index 711551805d2de9..12f4c5216375ae 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -182,6 +182,7 @@ wait_step() { all_test_steps() { command_step checks1 "ci/docker-run-default-image.sh ci/test-checks.sh" 20 check command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins" 15 check + command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-lib" 15 check command_step checks3 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-all-targets" 15 check command_step miri "ci/docker-run-default-image.sh ci/test-miri.sh" 5 check wait_step diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 909ddabbf3ff9a..713a5451dc5fd4 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -43,7 +43,7 @@ allowed="${allowed%,}" mode=${1:-full} case "$mode" in - tree | check-bins | check-all-targets | full) + tree | check-bins | check-lib | check-all-targets | full) ;; *) echo "$0: unrecognized mode: $mode"; @@ -156,7 +156,7 @@ fi # consistency with other CI steps and for the possibility of new similar lints. export RUSTFLAGS="-D warnings -Z threads=8 $RUSTFLAGS" -if [[ $mode = "check-bins" || $mode = "full" ]]; then +if [[ $mode = "check-bins" || $mode = "check-lib" || $mode = "full" ]]; then # Until https://github.com/rust-lang/cargo/pull/14163 is applied for our # chosen nightly toolchain, dcou needs custom-built cargo to avoid # false-negatives. @@ -183,10 +183,13 @@ if [[ $mode = "check-bins" || $mode = "full" ]]; then ) # Use `cargo "+${rust_nightly}" hack ..` once we stop using custom-built one. - PATH="./cargo-for-dcou/target/release:$PATH" \ - RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins - PATH="./cargo-for-dcou/target/release:$PATH" \ - RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib + if [[ $mode = "check-bins" ]]; then + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins + else + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib + fi fi if [[ $mode = "check-all-targets" || $mode = "full" ]]; then _ cargo "+${rust_nightly}" hack check --all-targets From c07110182cf5ffe17f597cda8ab63af02044670f Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Sun, 14 Jul 2024 15:38:15 +0900 Subject: [PATCH 5/5] add bins-and-lib --- ci/buildkite-pipeline.sh | 5 +++-- scripts/check-dev-context-only-utils.sh | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index 12f4c5216375ae..2557a098984e73 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -181,8 +181,9 @@ wait_step() { all_test_steps() { command_step checks1 "ci/docker-run-default-image.sh ci/test-checks.sh" 20 check - command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins" 15 check - command_step checks2 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-lib" 15 check + command_step checks2b "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins" 15 check + command_step checks2l "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-lib" 15 check + command_step checks2bl "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-bins-and-lib" 15 check command_step checks3 "ci/docker-run-default-image.sh ci/test-dev-context-only-utils.sh check-all-targets" 15 check command_step miri "ci/docker-run-default-image.sh ci/test-miri.sh" 5 check wait_step diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 713a5451dc5fd4..efa72883e0026a 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -43,7 +43,7 @@ allowed="${allowed%,}" mode=${1:-full} case "$mode" in - tree | check-bins | check-lib | check-all-targets | full) + tree | check-bins | check-lib | check-bins-and-lib | check-all-targets | full) ;; *) echo "$0: unrecognized mode: $mode"; @@ -156,7 +156,7 @@ fi # consistency with other CI steps and for the possibility of new similar lints. export RUSTFLAGS="-D warnings -Z threads=8 $RUSTFLAGS" -if [[ $mode = "check-bins" || $mode = "check-lib" || $mode = "full" ]]; then +if [[ $mode = "check-bins" || $mode = "check-lib" || $mode = "check-bins-and-lib" || $mode = "full" ]]; then # Until https://github.com/rust-lang/cargo/pull/14163 is applied for our # chosen nightly toolchain, dcou needs custom-built cargo to avoid # false-negatives. @@ -186,9 +186,12 @@ if [[ $mode = "check-bins" || $mode = "check-lib" || $mode = "full" ]]; then if [[ $mode = "check-bins" ]]; then PATH="./cargo-for-dcou/target/release:$PATH" \ RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins - else + elif [[ $mode = "check-lib" ]]; then PATH="./cargo-for-dcou/target/release:$PATH" \ RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --lib + else + PATH="./cargo-for-dcou/target/release:$PATH" \ + RUSTUP_TOOLCHAIN="${rust_nightly}" _ cargo hack check --bins --lib fi fi if [[ $mode = "check-all-targets" || $mode = "full" ]]; then