Merge pull request #11612 from Security-Onion-Solutions/upgrade/salt3…

…006.3

Upgrade/salt3006.3

salt/common/tools/sbin/so-common

@@ -152,15 +152,18 @@ check_salt_master_status() {
     return 0
 }
 
+# this is only intended to be used to check the status of the minion
 check_salt_minion_status() {
-	local timeout="${1:-5}"
-	echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
-	salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
+	local minion="$1"
+	local timeout="${2:-5}"
+	local logfile="${3:-'/dev/stdout'}"
+	echo "Checking if the salt minion will respond to jobs" >> "$logfile" 2>&1
+	salt "$minion" test.ping -t $timeout > /dev/null 2>&1
 	local status=$?
 	if [ $status -gt 0 ]; then
-		echo "  Minion did not respond" >> "$setup_log" 2>&1
+		echo "  Minion did not respond" >> "$logfile" 2>&1
 	else
-		echo "  Received job response from salt minion" >> "$setup_log" 2>&1
+		echo "  Received job response from salt minion" >> "$logfile" 2>&1
 	fi
 
 	return $status
@@ -440,6 +443,24 @@ run_check_net_err() {
 	fi
 }
 
+wait_for_salt_minion() {
+	local minion="$1"
+	local timeout="${2:-5}"
+	local logfile="${3:-'/dev/stdout'}"
+	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
+	local attempt=0
+	# each attempts would take about 15 seconds
+	local maxAttempts=20
+	until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
+		attempt=$((attempt+1))
+		if [[ $attempt -eq $maxAttempts ]]; then
+			return 1
+		fi
+		sleep 10
+	done
+	return 0
+}
+
 salt_minion_count() {
 	local MINIONDIR="/opt/so/saltstack/local/pillar/minions"
 	MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l)
@@ -452,19 +473,51 @@ set_os() {
 			OS=rocky
 			OSVER=9
 			is_rocky=true
+			is_rpm=true
 		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
 			OS=centos
 			OSVER=9
 			is_centos=true
-		elif grep -q "Oracle Linux Server release 9" /etc/system-release; then
-			OS=oel
+			is_rpm=true
+		elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
+			OS=alma
 			OSVER=9
-			is_oracle=true
+			is_alma=true
+			is_rpm=true
+		elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
+			if [ -f /etc/oracle-release ]; then
+				OS=oracle
+				OSVER=9
+				is_oracle=true
+				is_rpm=true
+			else
+				OS=rhel
+				OSVER=9
+				is_rhel=true
+				is_rpm=true
+			fi
 		fi
 		cron_service_name="crond"
-	else
-		OS=ubuntu
-		is_ubuntu=true
+	elif [ -f /etc/os-release ]; then
+		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+			OSVER=focal
+			UBVER=20.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
+			OSVER=jammy
+			UBVER=22.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
+			OSVER=bookworm
+			DEBVER=12
+			is_debian=true
+			OS=debian
+			is_deb=true
+		fi
 		cron_service_name="cron"
 	fi
 }

salt/manager/tools/sbin/soup

@@ -460,14 +460,14 @@ stop_salt_master() {
     echo ""
     echo "Killing any queued Salt jobs on the manager."
     pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1
-    set -e
 
     echo ""
     echo "Storing salt-master pid."
     MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess')
     echo "Found salt-master PID $MASTERPID"
     systemctl_func "stop" "salt-master"
     timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+    set -e
 }
 
 stop_salt_minion() {
@@ -480,14 +480,12 @@ stop_salt_minion() {
     echo ""
     echo "Killing Salt jobs on this node."
     salt-call saltutil.kill_all_jobs --local
-    set -e
 
     echo "Storing salt-minion pid."
     MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1)
     echo "Found salt-minion PID $MINIONPID"
     systemctl_func "stop" "salt-minion"
 
-    set +e
     timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
     set -e
 }
@@ -620,6 +618,7 @@ upgrade_check_salt() {
   if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
     echo "You are already running the correct version of Salt for Security Onion."
   else
+    echo "Salt needs to be upgraded to $NEWSALTVERSION."
     UPGRADESALT=1
   fi
 }   
@@ -628,22 +627,48 @@ upgrade_salt() {
   SALTUPGRADED=True
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
   echo ""
-  # If CentOS
-  if [[ $OS == 'centos' ]]; then
+  # If rhel family
+  if [[ $is_rpm ]]; then
     echo "Removing yum versionlock for Salt."
     echo ""
     yum versionlock delete "salt-*"
     echo "Updating Salt packages."
     echo ""
     set +e
-    run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
-    "Could not update salt, please check $SOUP_LOG for details."
+    # if oracle run with -r to ignore repos set by bootstrap
+    if [[ $OS == 'oracle' ]]; then
+      run_check_net_err \
+      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+      "Could not update salt, please check $SOUP_LOG for details."
+    # if another rhel family variant we want to run without -r to allow the bootstrap script to manage repos
+    else
+      run_check_net_err \
+      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+      "Could not update salt, please check $SOUP_LOG for details."
+    fi
     set -e
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
   # Else do Ubuntu things
+  elif [[ $is_deb ]]; then
+    echo "Removing apt hold for Salt."
+    echo ""
+    apt-mark unhold "salt-common"
+    apt-mark unhold "salt-master"
+    apt-mark unhold "salt-minion"
+    echo "Updating Salt packages."
+    echo ""
+    set +e
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update salt, please check $SOUP_LOG for details."
+    set -e
+    echo "Applying apt hold for Salt."
+    echo ""
+    apt-mark hold "salt-common"
+    apt-mark hold "salt-master"
+    apt-mark hold "salt-minion"
   fi
 
   echo "Checking if Salt was upgraded."
@@ -655,7 +680,7 @@ upgrade_salt() {
     echo "Once the issue is resolved, run soup again."
     echo "Exiting."
     echo ""
-    exit 0
+    exit 1
   else
     echo "Salt upgrade success."
     echo ""
@@ -785,7 +810,7 @@ main() {
   if [[ $is_airgap -eq 0 ]]; then
     yum clean all
     check_os_updates
-  elif [[ $OS == 'oel' ]]; then
+  elif [[ $OS == 'oracle' ]]; then
     # sync remote repo down to local if not airgap
     repo_sync
     check_os_updates
@@ -802,7 +827,8 @@ main() {
     echo "Hotfix applied"
     update_version
     enable_highstate
-    salt-call state.highstate -l info queue=True
+    (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
+    highstate
   else
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
@@ -838,6 +864,14 @@ main() {
       echo "Upgrading Salt"
       # Update the repo files so it can actually upgrade
       upgrade_salt
+
+      # for Debian based distro, we need to stop salt again after upgrade output below is from bootstrap-salt
+      # *  WARN: Not starting daemons on Debian based distributions
+      #    is not working mostly because starting them is the default behaviour.
+      if [[ $is_deb ]]; then
+        stop_salt_minion
+        stop_salt_master
+      fi
     fi
 
     preupgrade_changes
@@ -900,7 +934,8 @@ main() {
     echo ""
     echo "Running a highstate. This could take several minutes."
     set +e
-    salt-call state.highstate -l info queue=True
+    (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
+    highstate
     set -e
 
     stop_salt_master
@@ -915,7 +950,8 @@ main() {
     set -e
 
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-    salt-call state.highstate -l info queue=True
+    (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
+    highstate
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
 

salt/salt/map.jinja

@@ -23,7 +23,7 @@
   {% if grains.os|lower in ['Rocky', 'redhat', 'CentOS Stream'] %}
       {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %}
   {% elif grains.os_family|lower == 'debian' %}
-    {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %}
+    {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION %}
   {% endif %}
 {% else %}
   {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %}

salt/salt/master.defaults.yaml

@@ -2,4 +2,4 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   master:
-    version: 3006.1
+    version: 3006.3

salt/salt/minion.defaults.yaml

@@ -2,6 +2,6 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   minion:
-    version: 3006.1
+    version: 3006.3
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
     service_start_delay: 30 # in seconds.