Speed up net.sh builds (solana-labs#16360)

* Speed up net.sh builds

* feedback

* Update net/net.sh

Co-authored-by: Tyera Eulberg <[email protected]>

* feedback

* fix

Co-authored-by: Trent Nelson <[email protected]>
Co-authored-by: Tyera Eulberg <[email protected]>

Author: 3 people

cargo

@@ -3,25 +3,22 @@
 # shellcheck source=ci/rust-version.sh
 here=$(dirname "$0")
 
-source "${here}"/ci/rust-version.sh all
-
 toolchain=
 case "$1" in
   stable)
+    source "${here}"/ci/rust-version.sh stable
     # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
     toolchain="$rust_stable"
     shift
     ;;
   nightly)
+    source "${here}"/ci/rust-version.sh nightly
     # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
     toolchain="$rust_nightly"
     shift
     ;;
-  +*)
-    toolchain="${1#+}"
-    shift
-    ;;
   *)
+    source "${here}"/ci/rust-version.sh stable
     # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
     toolchain="$rust_stable"
     ;;

ci/do-audit.sh

@@ -7,8 +7,6 @@ src_root="$(readlink -f "${here}/..")"
 
 cd "${src_root}"
 
-source ci/rust-version.sh stable
-
 cargo_audit_ignores=(
   # failure is officially deprecated/unmaintained
   #
@@ -42,4 +40,4 @@ cargo_audit_ignores=(
   --ignore RUSTSEC-2020-0146
 
 )
-scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
+scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/test-checks.sh

@@ -45,7 +45,7 @@ export RUSTFLAGS="-D warnings -A incomplete_features"
 # Only force up-to-date lock files on edge
 if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   # Exclude --benches as it's not available in rust stable yet
-  if _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" check --locked --tests --bins --examples; then
+  if _ scripts/cargo-for-all-lock-files.sh stable check --locked --tests --bins --examples; then
     true
   else
     check_status=$?
@@ -56,7 +56,7 @@ if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   fi
 
   # Ensure nightly and --benches
-  _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets
+  _ scripts/cargo-for-all-lock-files.sh nightly check --locked --all-targets
 else
   echo "Note: cargo-for-all-lock-files.sh skipped because $CI_BASE_BRANCH != $EDGE_CHANNEL"
 fi

net/net.sh

@@ -42,6 +42,7 @@ Operate a configured testnet
  startnode    - Start an individual node (previously stopped with stopNode)
  stopnode     - Stop an individual node
  startclients - Start client nodes only
+ prepare      - Prepare software deployment. (Build/download the software release)
  update       - Deploy a new software update to the cluster
  upgrade      - Upgrade software on bootstrap validator. (Restart bootstrap validator manually to run it)
 
@@ -185,12 +186,12 @@ build() {
 
     buildVariant=
     if $debugBuild; then
-      buildVariant=debug
+      buildVariant=--debug
     fi
 
     $MAYBE_DOCKER bash -c "
       set -ex
-      scripts/cargo-install-all.sh farf \"$buildVariant\"
+      scripts/cargo-install-all.sh farf $buildVariant --validator-only
     "
   )
 
@@ -1055,6 +1056,9 @@ start)
   prepareDeploy
   deploy
   ;;
+prepare)
+  prepareDeploy
+  ;;
 sanity)
   sanity
   ;;

scripts/cargo-install-all.sh

@@ -14,7 +14,7 @@ usage() {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [+<cargo version>] [--debug] <install directory>
+usage: $0 [+<cargo version>] [--debug] [--validator-only] <install directory>
 EOF
   exit $exitcode
 }
@@ -23,13 +23,17 @@ maybeRustVersion=
 installDir=
 buildVariant=release
 maybeReleaseFlag=--release
+validatorOnly=
 
 while [[ -n $1 ]]; do
   if [[ ${1:0:1} = - ]]; then
     if [[ $1 = --debug ]]; then
       maybeReleaseFlag=
       buildVariant=debug
       shift
+    elif [[ $1 = --validator-only ]]; then
+      validatorOnly=true
+      shift
     else
       usage "Unknown option: $1"
     fi
@@ -71,37 +75,37 @@ if [[ $CI_OS_NAME = windows ]]; then
   )
 else
   ./fetch-perf-libs.sh
-  (
-    set -x
-    # shellcheck disable=SC2086 # Don't want to double quote $rust_version
-    $cargo $maybeRustVersion build $maybeReleaseFlag
-  )
-
 
   BINS=(
-    cargo-build-bpf
-    cargo-test-bpf
     solana
     solana-bench-exchange
     solana-bench-tps
-    solana-dos
     solana-faucet
     solana-gossip
     solana-install
-    solana-install-init
     solana-keygen
     solana-ledger-tool
     solana-log-analyzer
     solana-net-shaper
-    solana-stake-accounts
-    solana-stake-monitor
     solana-sys-tuner
-    solana-test-validator
-    solana-tokens
     solana-validator
-    solana-watchtower
   )
 
+  # Speed up net.sh deploys by excluding unused binaries
+  if [[ -z "$validatorOnly" ]]; then
+    BINS+=(
+      cargo-build-bpf
+      cargo-test-bpf
+      solana-dos
+      solana-install-init
+      solana-stake-accounts
+      solana-stake-monitor
+      solana-test-validator
+      solana-tokens
+      solana-watchtower
+    )
+  fi
+
   #XXX: Ensure `solana-genesis` is built LAST!
   # See https://github.com/solana-labs/solana/issues/5826
   BINS+=(solana-genesis)
@@ -118,8 +122,12 @@ mkdir -p "$installDir/bin"
   set -x
   # shellcheck disable=SC2086 # Don't want to double quote $rust_version
   "$cargo" $maybeRustVersion build $maybeReleaseFlag "${binArgs[@]}"
-  # shellcheck disable=SC2086 # Don't want to double quote $rust_version
-  "$cargo" $maybeRustVersion install spl-token-cli --root "$installDir"
+
+  # Exclude `spl-token` binary for net.sh builds
+  if [[ -z "$validatorOnly" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote $rust_version
+    "$cargo" $maybeRustVersion install spl-token-cli --root "$installDir"
+  fi
 )
 
 for bin in "${BINS[@]}"; do