Moar rolling upgrade

multinode-demo/bootstrap-validator.sh

@@ -93,7 +93,6 @@ ledger_dir="$SOLANA_CONFIG_DIR"/bootstrap-validator
 args+=(
   --enable-rpc-exit
   --enable-rpc-set-log-filter
-  --require-tower
   --ledger "$ledger_dir"
   --rpc-port 8899
   --snapshot-interval-slots 200

multinode-demo/setup.sh

@@ -4,7 +4,7 @@ here=$(dirname "$0")
 # shellcheck source=multinode-demo/common.sh
 source "$here"/common.sh
 
-set -e
+set -ex
 
 rm -rf "$SOLANA_CONFIG_DIR"/bootstrap-validator
 mkdir -p "$SOLANA_CONFIG_DIR"/bootstrap-validator
@@ -48,4 +48,5 @@ default_arg --faucet-lamports 500000000000000000
 default_arg --hashes-per-tick auto
 default_arg --cluster-type development
 
+echo "genesis args: ${args[@]}"
 $solana_genesis "${args[@]}"

multinode-demo/validator.sh

@@ -228,7 +228,6 @@ default_arg --ledger "$ledger_dir"
 default_arg --log -
 default_arg --enable-rpc-exit
 default_arg --enable-rpc-set-log-filter
-default_arg --require-tower
 
 if [[ -n $SOLANA_CUDA ]]; then
   program=$solana_validator_cuda

net/net.sh

@@ -185,13 +185,44 @@ build() {
 
     buildVariant=
     if $debugBuild; then
-      buildVariant=debug
+      buildVariant=--debug
     fi
 
+    detached_build=false
+    if [[ -n $LOCAL_BUILD_BRANCH || -n $LOCAL_BUILD_REVISION ]]; then
+      detached_build=true
+      git status
+      git stash
+      if [[ -n $LOCAL_BUILD_BRANCH ]]; then
+        # for prs, BRANCH value will be like "pull/12390/head"
+        git fetch origin "$LOCAL_BUILD_BRANCH"
+      fi
+
+      # revision takes precedence
+      if [[ -n $LOCAL_BUILD_REVISION ]]; then
+        git reset --hard "$LOCAL_BUILD_REVISION"
+      else
+        git reset --hard "FETCH_HEAD"
+      fi
+      CI_COMMIT=$(git rev-parse HEAD)
+      export CI_COMMIT
+    fi
+
+    cargo_error_code=
+    source ci/rust-version.sh stable
     $MAYBE_DOCKER bash -c "
       set -ex
-      scripts/cargo-install-all.sh farf \"$buildVariant\"
-    "
+      scripts/cargo-install-all.sh +\"$rust_stable\" farf \"$buildVariant\"
+    " || cargo_error_code=$?
+
+    if $detached_build; then
+      git reset --hard "HEAD@{1}"
+      git stash pop || true
+    fi
+
+    if [[ -n $cargo_error_code ]]; then
+      (exit $cargo_error_code)
+    fi
   )
   echo "Build took $SECONDS seconds"
 }
@@ -289,6 +320,7 @@ startBootstrapLeader() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         > ~/solana/remote-node.log
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -360,6 +392,7 @@ startNode() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         > ~/solana/remote-node.log
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -900,6 +933,10 @@ while getopts "h?T:t:o:f:rc:Fn:i:d" opt "${shortArgs[@]}"; do
       releaseChannel=$OPTARG
       deployMethod=tar
       ;;
+    local)
+      # just pass-through to use default values of $deployMethod and $doBuild
+      true
+      ;;
     *)
       usage "Invalid release channel: $OPTARG"
       ;;

run.sh

@@ -104,7 +104,6 @@ args=(
   --enable-rpc-exit
   --enable-rpc-transaction-history
   --init-complete-file "$dataDir"/init-completed
-  --require-tower
 )
 solana-validator "${args[@]}" &
 validator=$!

system-test/rolling-upgrade/rolling_upgrade.sh

@@ -23,17 +23,42 @@ sleep_if_positive() {
   fi
 }
 
-# Fetch new software and upload it to the bootstrap validator
-declare -g sw_version_args
-get_net_launch_software_version_launch_args "$UPGRADE_CHANNEL" "upgrade-release" sw_version_args
-# shellcheck disable=2086 # $sw_version_args holds two args. Don't quote!
-"$NET_SH" upgrade $sw_version_args
+check_sanity() {
+  (
+    set -e
+    "$NET_SH" sanity
+  )
+}
+
+# Fetch or build new software and upload it to the bootstrap validator
+(
+  declare -g sw_version_args
+  get_net_launch_software_version_launch_args "$UPGRADE_CHANNEL" "upgrade-release" sw_version_args
+
+  # Override LOCAL_BUILD_* to build custom version to upgrade
+  if [[ -n $LOCAL_BUILD_BRANCH || -n $LOCAL_BUILD_REVISION ]]; then
+    unset LOCAL_BUILD_BRANCH
+    unset LOCAL_BUILD_REVISION
+    if [[ $UPGRADE_CHANNEL != "local" ]]; then
+      echo "Conflicting options: $UPGRADE_CHANNEL, $LOCAL_BUILD_BRANCH, $LOCAL_BUILD_REVISION"
+    fi
+  fi
+  if [[ -n $UPGRADE_LOCAL_BUILD_BRANCH ]]; then
+    export LOCAL_BUILD_BRANCH="$UPGRADE_LOCAL_BUILD_BRANCH"
+  fi
+  if [[ -n $UPGRADE_LOCAL_BUILD_REVISION ]]; then
+    export LOCAL_BUILD_REVISION="$UPGRADE_LOCAL_BUILD_REVISION"
+  fi
+
+  # shellcheck disable=2086 # $sw_version_args holds two args. Don't quote!
+  "$NET_SH" upgrade $sw_version_args
+)
 
 # Wait initial delay, if any
 sleep_if_positive "$UPGRADE_INITIAL_DELAY"
 
 if [[ "$UPGRADE_INITIAL_DELAY" -gt 0 ]]; then
-  "$NET_SH" sanity
+  check_sanity
 fi
 
 # Restart validators one by one
@@ -52,11 +77,11 @@ for i in "${!validatorIpList[@]}"; do
   # This could be replaced with something based on `solana catchup`
   sleep_if_positive "$UPGRADE_INTERVALIDATOR_DELAY"
 
-  "$NET_SH" sanity
+  check_sanity
 done
 
 sleep_if_positive "$UPGRADE_POST_TEST_DELAY"
 
 if [[ "$UPGRADE_POST_TEST_DELAY" -gt 0 ]]; then
-  "$NET_SH" sanity
+  check_sanity
 fi

system-test/testnet-automation.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
+NDEBUG="${NDEBUG-1}"
+export NDEBUG
+
 # shellcheck disable=SC1090
 # shellcheck disable=SC1091
 source "$(dirname "$0")"/automation_utils.sh
@@ -124,6 +127,10 @@ function launch_testnet() {
   declare -g version_args
   get_net_launch_software_version_launch_args "$CHANNEL" "solana-release" version_args
 
+  if [[ -z $NDEBUG ]]; then
+    maybeDebug=--debug
+  fi
+
   declare maybeWarpSlot
   if [[ -n "$WARP_SLOT" ]]; then
     maybeWarpSlot="--warp-slot $WARP_SLOT"
@@ -141,9 +148,10 @@ function launch_testnet() {
 
   # shellcheck disable=SC2068
   # shellcheck disable=SC2086
-  "${REPO_ROOT}"/net/net.sh start $version_args \
+  "${REPO_ROOT}"/net/net.sh start $version_args $maybeDebug \
     -c idle=$NUMBER_OF_CLIENT_NODES $maybeStartAllowBootFailures \
-    --gpu-mode $startGpuMode $maybeWarpSlot $maybeAsyncNodeInit $maybeExtraPrimordialStakes
+    --gpu-mode $startGpuMode $maybeWarpSlot $maybeAsyncNodeInit $maybeExtraPrimordialStakes \
+    $GENESIS_OPTIONS
 
   execution_step "Waiting for bootstrap validator's stake to fall below ${BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD}%"
   wait_for_bootstrap_validator_stake_drop "$BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD"
@@ -326,6 +334,7 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
                         PARTITION_ITERATION_COUNT \
                         TEST_TYPE \
                         CUSTOM_SCRIPT \
+                        GENESIS_OPTIONS \
                         )
 
 TEST_CONFIGURATION=