From c572a7ca344efd09392d2a0312c6e3c345ae6512 Mon Sep 17 00:00:00 2001 From: Denis P Date: Mon, 4 Feb 2019 18:31:20 +0100 Subject: [PATCH 01/12] CI optimizations --- .gitlab-ci.yml | 29 +++++++---------------------- scripts/gitlab/build-unix.sh | 30 ++++++++++++++++++------------ scripts/gitlab/cargo-audit.sh | 7 ------- scripts/gitlab/test-all.sh | 25 +------------------------ test.sh | 23 +---------------------- 5 files changed, 27 insertions(+), 87 deletions(-) delete mode 100755 scripts/gitlab/cargo-audit.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1a1c979987..7f4f1179788 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,9 +4,11 @@ stages: - publish - optional -image: parity/rust:gitlab-ci +image: parity/rust-ci:latest variables: + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: recursive CI_SERVER_NAME: "GitLab CI" CARGO_HOME: "${CI_PROJECT_DIR}/.cargo" CARGO_TARGET: x86_64-unknown-linux-gnu @@ -40,14 +42,16 @@ test-linux: variables: RUN_TESTS: all script: - - scripts/gitlab/test-all.sh stable + - scripts/gitlab/test-all.sh tags: - rust-stable test-audit: stage: test script: - - scripts/gitlab/cargo-audit.sh + - set -e + - set -u + - cargo audit tags: - rust-stable @@ -150,22 +154,3 @@ build-android: allow_failure: true <<: *collect_artifacts -test-beta: - stage: optional - variables: - RUN_TESTS: cargo - script: - - scripts/gitlab/test-all.sh beta - tags: - - rust-beta - allow_failure: true - -test-nightly: - stage: optional - variables: - RUN_TESTS: all - script: - - scripts/gitlab/test-all.sh nightly - tags: - - rust-nightly - allow_failure: true diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index 57c6c858583..0989bad1edf 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -9,9 +9,8 @@ echo "CARGO_HOME: " $CARGO_HOME echo "CARGO_TARGET: " $CARGO_TARGET echo "CC: " $CC echo "CXX: " $CXX -#strip ON -export RUSTFLAGS=" -C link-arg=-s" +echo "_____ Building target: "$CARGO_TARGET" _____" echo "_____ Building target: "$CARGO_TARGET" _____" if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then @@ -27,20 +26,27 @@ fi echo "_____ Post-processing binaries _____" rm -rf artifacts -mkdir -p artifacts -cd artifacts -mkdir -p $CARGO_TARGET -cd $CARGO_TARGET +mkdir -p artifacts/$CARGO_TARGET + if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then # only thing we need for android - cp -v ../../target/$CARGO_TARGET/release/libparity.so ./libparity.so + cp -v target/$CARGO_TARGET/release/libparity.so ./libparity.so +else + cp -v target/$CARGO_TARGET/release/parity ./parity + cp -v target/$CARGO_TARGET/release/parity-evm ./parity-evm + cp -v target/$CARGO_TARGET/release/ethstore ./ethstore + cp -v target/$CARGO_TARGET/release/ethkey ./ethkey + cp -v target/$CARGO_TARGET/release/whisper ./whisper +fi + +# stripping can also be done on release build time +# export RUSTFLAGS="${RUSTFLAGS} -C link-arg=-s" +if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] +then + arm-linux-androideabi-strip -v ./* else - cp -v ../../target/$CARGO_TARGET/release/parity ./parity - cp -v ../../target/$CARGO_TARGET/release/parity-evm ./parity-evm - cp -v ../../target/$CARGO_TARGET/release/ethstore ./ethstore - cp -v ../../target/$CARGO_TARGET/release/ethkey ./ethkey - cp -v ../../target/$CARGO_TARGET/release/whisper ./whisper + strip -v ./* fi echo "_____ Calculating checksums _____" diff --git a/scripts/gitlab/cargo-audit.sh b/scripts/gitlab/cargo-audit.sh deleted file mode 100755 index 16f0dc934ad..00000000000 --- a/scripts/gitlab/cargo-audit.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e # fail on any error -set -u # treat unset variables as error - -cargo install cargo-audit -cargo audit diff --git a/scripts/gitlab/test-all.sh b/scripts/gitlab/test-all.sh index 15c22870626..41fad3a71d7 100755 --- a/scripts/gitlab/test-all.sh +++ b/scripts/gitlab/test-all.sh @@ -1,34 +1,11 @@ #!/bin/bash -# ARGUMENT $1 Rust flavor to test with (stable/beta/nightly) set -e # fail on any error set -u # treat unset variables as error git log --graph --oneline --decorate=short -n 10 -case ${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}} in - (beta|stable) - export GIT_COMPARE=origin/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}~ - ;; - (master|nightly) - export GIT_COMPARE=master~ - ;; - (*) - export GIT_COMPARE=master - ;; -esac - -export RUST_FILES_MODIFIED="$(git --no-pager diff --name-only $GIT_COMPARE...$CI_COMMIT_SHA | grep -v -e ^\\. -e ^LICENSE -e ^README.md -e ^CHANGELOG.md -e ^test.sh -e ^scripts/ -e ^docs/ -e ^docker/ -e ^snap/ | wc -l | tr -d ' ')" -echo "RUST_FILES_MODIFIED: $RUST_FILES_MODIFIED" - -if [ "${RUST_FILES_MODIFIED}" = "0" ] -then - echo "__________Skipping Rust tests since no Rust files modified__________"; - exit 0 -fi - -rustup default $1 - +# FIXME: git submodule update --init --recursive rustup show diff --git a/test.sh b/test.sh index 1b05194e471..a25f41d9a94 100755 --- a/test.sh +++ b/test.sh @@ -1,33 +1,12 @@ #!/bin/sh # Running Parity Full Test Suite +echo "________Running test.sh________" FEATURES="json-tests,ci-skip-issue" OPTIONS="--release" VALIDATE=1 THREADS=8 -case $1 in - --no-json) - FEATURES="ipc" - shift # past argument=value - ;; - --no-release) - OPTIONS="" - shift - ;; - --no-validate) - VALIDATE=0 - shift - ;; - --no-run) - OPTIONS="--no-run" - shift - ;; - *) - # unknown option - ;; -esac - set -e From 543c254cb68535cfb8f8ff374cb1df5feb002647 Mon Sep 17 00:00:00 2001 From: Denis P Date: Mon, 4 Feb 2019 19:34:22 +0100 Subject: [PATCH 02/12] fix stripping --- scripts/gitlab/build-unix.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index 0989bad1edf..e2ac81bf5c9 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -9,8 +9,9 @@ echo "CARGO_HOME: " $CARGO_HOME echo "CARGO_TARGET: " $CARGO_TARGET echo "CC: " $CC echo "CXX: " $CXX +#strip ON +export RUSTFLAGS=" -C link-arg=-s" -echo "_____ Building target: "$CARGO_TARGET" _____" echo "_____ Building target: "$CARGO_TARGET" _____" if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then @@ -40,15 +41,6 @@ else cp -v target/$CARGO_TARGET/release/whisper ./whisper fi -# stripping can also be done on release build time -# export RUSTFLAGS="${RUSTFLAGS} -C link-arg=-s" -if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] -then - arm-linux-androideabi-strip -v ./* -else - strip -v ./* -fi - echo "_____ Calculating checksums _____" for binary in $(ls) do From b1905b470b1f5c53c9f2efb6742eb7a23c376950 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 5 Feb 2019 10:44:52 +0100 Subject: [PATCH 03/12] new dockerfile --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f4f1179788..07ede4068c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - publish - optional -image: parity/rust-ci:latest +image: parity/rust:gitlab-ci variables: GIT_STRATEGY: fetch From 6ec2f2b5db2d8b4da58774cac2137b702be753b1 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 5 Feb 2019 10:59:51 +0100 Subject: [PATCH 04/12] no need n submodule upd --- scripts/gitlab/test-all.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/gitlab/test-all.sh b/scripts/gitlab/test-all.sh index 41fad3a71d7..72554906740 100755 --- a/scripts/gitlab/test-all.sh +++ b/scripts/gitlab/test-all.sh @@ -5,8 +5,6 @@ set -u # treat unset variables as error git log --graph --oneline --decorate=short -n 10 -# FIXME: -git submodule update --init --recursive rustup show exec ./test.sh From dea89362e7167c94ad09d31d3111b79c2ae79c65 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 5 Feb 2019 17:25:52 +0100 Subject: [PATCH 05/12] review --- scripts/gitlab/build-unix.sh | 16 +++++++--------- scripts/gitlab/test-all.sh | 22 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index e2ac81bf5c9..7ebe93b6868 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -15,7 +15,6 @@ export RUSTFLAGS=" -C link-arg=-s" echo "_____ Building target: "$CARGO_TARGET" _____" if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then -# only thing we need for android time cargo build --target $CARGO_TARGET --release -p parity-clib --features final else time cargo build --target $CARGO_TARGET --release --features final @@ -26,19 +25,18 @@ else fi echo "_____ Post-processing binaries _____" -rm -rf artifacts mkdir -p artifacts/$CARGO_TARGET +cd artifacts/$CARGO_TARGET if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then -# only thing we need for android - cp -v target/$CARGO_TARGET/release/libparity.so ./libparity.so + cp -v ../../target/$CARGO_TARGET/release/libparity.so ./libparity.so else - cp -v target/$CARGO_TARGET/release/parity ./parity - cp -v target/$CARGO_TARGET/release/parity-evm ./parity-evm - cp -v target/$CARGO_TARGET/release/ethstore ./ethstore - cp -v target/$CARGO_TARGET/release/ethkey ./ethkey - cp -v target/$CARGO_TARGET/release/whisper ./whisper + cp -v ../../target/$CARGO_TARGET/release/parity ./parity + cp -v ../../target/$CARGO_TARGET/release/parity-evm ./parity-evm + cp -v ../../target/$CARGO_TARGET/release/ethstore ./ethstore + cp -v ../../target/$CARGO_TARGET/release/ethkey ./ethkey + cp -v ../../target/$CARGO_TARGET/release/whisper ./whisper fi echo "_____ Calculating checksums _____" diff --git a/scripts/gitlab/test-all.sh b/scripts/gitlab/test-all.sh index 72554906740..925124b7ac2 100755 --- a/scripts/gitlab/test-all.sh +++ b/scripts/gitlab/test-all.sh @@ -1,10 +1,30 @@ #!/bin/bash - set -e # fail on any error set -u # treat unset variables as error git log --graph --oneline --decorate=short -n 10 +case ${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}} in + (beta|stable) + export GIT_COMPARE=origin/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}~ + ;; + (master|nightly) + export GIT_COMPARE=master~ + ;; + (*) + export GIT_COMPARE=master + ;; +esac + +export RUST_FILES_MODIFIED="$(git --no-pager diff --name-only $GIT_COMPARE...$CI_COMMIT_SHA | grep -v -e ^\\. -e ^LICENSE -e ^README.md -e ^CHANGELOG.md -e ^test.sh -e ^scripts/ -e ^docs/ -e ^docker/ -e ^snap/ | wc -l | tr -d ' ')" +echo "RUST_FILES_MODIFIED: $RUST_FILES_MODIFIED" + +if [ "${RUST_FILES_MODIFIED}" = "0" ] +then + echo "__________Skipping Rust tests since no Rust files modified__________"; + exit 0 +fi + rustup show exec ./test.sh From aeaa4a1f687def08d794314d6445424532a4f027 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 5 Feb 2019 18:51:06 +0100 Subject: [PATCH 06/12] moved dockerfile --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07ede4068c5..53c328a1b5e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - publish - optional -image: parity/rust:gitlab-ci +image: parity/rust-linux:stretch variables: GIT_STRATEGY: fetch From 9b938708ee7395c1cc5f1c4c928ec22fc4d769c3 Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: [PATCH 07/12] it becomes large --- .gitlab-ci.yml | 62 ++++++++++++++++--- ...blish-awss3.sh => publish-onnet-update.sh} | 16 ----- 2 files changed, 54 insertions(+), 24 deletions(-) rename scripts/gitlab/{publish-awss3.sh => publish-onnet-update.sh} (73%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53c328a1b5e..6e7b859a943 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - publish - optional -image: parity/rust-linux:stretch +image: parity/rust-parity-ethereum-build:stretch variables: GIT_STRATEGY: fetch @@ -44,7 +44,7 @@ test-linux: script: - scripts/gitlab/test-all.sh tags: - - rust-stable + - linux-docker test-audit: stage: test @@ -53,7 +53,7 @@ test-audit: - set -u - cargo audit tags: - - rust-stable + - linux-docker build-linux: stage: build @@ -62,7 +62,7 @@ build-linux: - scripts/gitlab/build-unix.sh <<: *collect_artifacts tags: - - rust-stable + - linux-docker build-darwin: stage: build @@ -116,19 +116,65 @@ publish-snap: allow_failure: true <<: *collect_artifacts -publish-awss3: +publish:onnet:update: stage: publish only: *releaseable_branches - cache: {} + cache: {} dependencies: - build-linux - build-darwin - build-windows before_script: *determine_version script: - - scripts/gitlab/publish-awss3.sh + - scripts/gitlab/publish-onnet-update.sh tags: - - shell + - linux-docker + +# configures aws for fast uploads/syncs +.s3_before_script: &s3_before_script + before_script: + - mkdir -p ${HOME}/.aws + - | + cat > ${HOME}/.aws/config < Date: Thu, 7 Feb 2019 17:06:41 +0100 Subject: [PATCH 08/12] onchain update depends on s3 --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e7b859a943..a97f1470882 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,6 +124,7 @@ publish:onnet:update: - build-linux - build-darwin - build-windows + - publish:awss3:release before_script: *determine_version script: - scripts/gitlab/publish-onnet-update.sh From 4073a3d7463fb33fe775a1ddacfef8b87a6911e7 Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 7 Feb 2019 17:19:44 +0100 Subject: [PATCH 09/12] fix dependency --- .gitlab-ci.yml | 3 ++- scripts/gitlab/build-unix.sh | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a97f1470882..f699700b51d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - test - build - publish + - publish-onchain - optional image: parity/rust-parity-ethereum-build:stretch @@ -117,7 +118,7 @@ publish-snap: <<: *collect_artifacts publish:onnet:update: - stage: publish + stage: publish-onchain only: *releaseable_branches cache: {} dependencies: diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index 7ebe93b6868..bd64bb1d5de 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -51,3 +51,6 @@ do fi done +echo "_____ Cache status _____" +sccache -s + From cb038d77b2b0c68f350977845b36b876427ac4c7 Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 7 Feb 2019 17:36:31 +0100 Subject: [PATCH 10/12] fix cache status --- scripts/gitlab/build-unix.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index bd64bb1d5de..44b014299af 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -37,6 +37,8 @@ else cp -v ../../target/$CARGO_TARGET/release/ethstore ./ethstore cp -v ../../target/$CARGO_TARGET/release/ethkey ./ethkey cp -v ../../target/$CARGO_TARGET/release/whisper ./whisper + echo "_____ Cache status _____" + sccache -s fi echo "_____ Calculating checksums _____" @@ -50,7 +52,3 @@ do ./parity tools hash $binary > $binary.sha3 fi done - -echo "_____ Cache status _____" -sccache -s - From 0bcd560d3a6217a6f4d9d6b63c44e4fcfc05755b Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 7 Feb 2019 17:37:51 +0100 Subject: [PATCH 11/12] fix cache status --- scripts/gitlab/build-unix.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/gitlab/build-unix.sh b/scripts/gitlab/build-unix.sh index 44b014299af..6244dc8460c 100755 --- a/scripts/gitlab/build-unix.sh +++ b/scripts/gitlab/build-unix.sh @@ -37,8 +37,6 @@ else cp -v ../../target/$CARGO_TARGET/release/ethstore ./ethstore cp -v ../../target/$CARGO_TARGET/release/ethkey ./ethkey cp -v ../../target/$CARGO_TARGET/release/whisper ./whisper - echo "_____ Cache status _____" - sccache -s fi echo "_____ Calculating checksums _____" From e29476ba49774f52e0857ec33d222acad3feebfb Mon Sep 17 00:00:00 2001 From: Denis P Date: Thu, 7 Feb 2019 17:38:47 +0100 Subject: [PATCH 12/12] new cache status --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f699700b51d..90097a763e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,6 +61,7 @@ build-linux: only: *releaseable_branches script: - scripts/gitlab/build-unix.sh + - sccache -s <<: *collect_artifacts tags: - linux-docker