fast-reboot

#!/bin/bash -e

REBOOT_USER=$(logname)
REBOOT_TIME=$(date)
REBOOT_CAUSE_FILE="/var/cache/sonic/reboot-cause.txt"
WARM_DIR=/host/warmboot
REDIS_FILE=dump.rdb
REBOOT_SCRIPT_NAME=$(basename $0)
REBOOT_TYPE="${REBOOT_SCRIPT_NAME}"
VERBOSE=no
FORCE=no
REBOOT_METHOD="/sbin/reboot"

# Check root privileges
if [[ "$EUID" -ne 0 ]]
then
    echo "This command must be run as root" >&2
    exit 1
fi

function debug()
{
    if [[ x"${VERBOSE}" == x"yes" ]]; then
        echo `date` $@
    fi
    logger "$@"
}

function showHelpAndExit()
{
    echo "Usage: ${REBOOT_SCRIPT_NAME} [options]"
    echo "    -h -? : get this help"
    echo "    -v    : turn on verbose"
    echo "    -f    : force execution"
    echo "    -r    : reboot with /sbin/reboot [default]"
    echo "    -k    : reboot with /sbin/kexec -e"

    exit 0
}

function parseOptions()
{
    while getopts "vfh?rk" opt; do
        case ${opt} in
            h )
                showHelpAndExit
                ;;
            \? )
                showHelpAndExit
                ;;
            v )
                VERBOSE=yes
                ;;
            f )
                FORCE=yes
                ;;
            r )
                REBOOT_METHOD="/sbin/reboot"
                ;;
            k )
                REBOOT_METHOD="/sbin/kexec -e"
                ;;
        esac
    done
}

sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)

function clear_warm_boot()
{
    debug "Failure ($?) cleanup ..."
    config warm_restart disable || /bin/true
    /sbin/kexec -u || /bin/true

    TIMESTAMP=`date +%Y%m%d-%H%M%S`
    if [[ -f ${WARM_DIR}/${REDIS_FILE} ]]; then
        mv -f ${WARM_DIR}/${REDIS_FILE} ${WARM_DIR}/${REDIS_FILE}.${TIMESTAMP} || /bin/true
    fi
}

function initialize_pre_shutdown()
{
    debug "Initialize pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    RESTORE_COUNT=`/usr/bin/redis-cli -n 6 hget "${TABLE}" restore_count`
    if [[ -z "$RESTORE_COUNT" ]]; then
        /usr/bin/redis-cli -n 6 hset "${TABLE}" "restore_count" "0" > /dev/null
    fi
    /usr/bin/redis-cli -n 6 hset "${TABLE}" "state" "requesting" > /dev/null
}

function request_pre_shutdown()
{
    debug "Requesting pre-shutdown ..."
    /usr/bin/docker exec -i syncd /usr/bin/syncd_request_shutdown --pre
}

function wait_for_pre_shutdown_complete_or_fail()
{
    debug "Waiting for pre-shutdown ..."
    TABLE="WARM_RESTART_TABLE|warm-shutdown"
    STATE="requesting"
    declare -i waitcount;
    waitcount=0
    # Wait up to 60 seconds for pre-shutdown to complete
    while [[ ${waitcount} -lt 600 ]]; do
        STATE=`/usr/bin/redis-cli -n 6 hget "${TABLE}" state`
        if [[ x"${STATE}" != x"requesting" ]]; then
            break
        fi
        sleep 0.1
        waitcount+=1
    done

    if [[ x"${STATE}" != x"pre-shutdown-succeeded" ]]; then
        debug "Syncd pre-shutdown failed: ${STATE} ..."
        exit 10
    fi
    debug "Pre-shutdown succeeded ..."
}

function backup_database()
{
    debug "Backing up database ..."
    # Dump redis content to a file 'dump.rdb' in warmboot directory
    mkdir -p $WARM_DIR
    # Delete keys in stateDB except FDB_TABLE|* and WARM_RESTA*
    redis-cli -n 6 eval "
        for _, k in ipairs(redis.call('keys', '*')) do
            if not string.match(k, 'FDB_TABLE|') and not string.match(k, 'WARM_RESTART_TABLE|') then
                redis.call('del', k)
            end
        end
    " 0
    redis-cli save
    docker cp database:/var/lib/redis/$REDIS_FILE $WARM_DIR
    docker exec -i database rm /var/lib/redis/$REDIS_FILE
}

parseOptions $@

# Check reboot type supported
BOOT_TYPE_ARG="cold"
case "$REBOOT_TYPE" in
    "fast-reboot")
        BOOT_TYPE_ARG=$REBOOT_TYPE
        ;;
    "warm-reboot")
        if [[ "$sonic_asic_type" == "mellanox" ]]; then
            REBOOT_TYPE="fastfast-reboot"
            BOOT_TYPE_ARG="fastfast"
            # source mlnx-ffb.sh file with
            # functions to check ISSU upgrade/do ISSU start
            source mlnx-ffb.sh
        else
            BOOT_TYPE_ARG="warm"
        fi
        trap clear_warm_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
        config warm_restart enable system
        ;;
    *)
        echo "Not supported reboot type: $REBOOT_TYPE" >&2
        exit 1
        ;;
esac

# Unload the previously loaded kernel if any loaded
if [[ "$(cat /sys/kernel/kexec_loaded)" -eq 1 ]]
then
    /sbin/kexec -u
fi

# Kernel and initrd image
NEXT_SONIC_IMAGE=$(sonic_installer list | grep "Next: " | cut -d ' ' -f 2)
if grep -q aboot_platform= /host/machine.conf; then
    IMAGE_PATH="/host/image-${NEXT_SONIC_IMAGE#SONiC-OS-}"
    KERNEL_IMAGE="$(ls $IMAGE_PATH/boot/vmlinuz-*)"
    BOOT_OPTIONS="$(cat "$IMAGE_PATH/kernel-cmdline" | tr '\n' ' ') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
elif grep -q onie_platform= /host/machine.conf; then
    KERNEL_OPTIONS=$(cat /host/grub/grub.cfg | sed "/$NEXT_SONIC_IMAGE'/,/}/"'!'"g" | grep linux)
    KERNEL_IMAGE="/host$(echo $KERNEL_OPTIONS | cut -d ' ' -f 2)"
    BOOT_OPTIONS="$(echo $KERNEL_OPTIONS | sed -e 's/\s*linux\s*/BOOT_IMAGE=/') SONIC_BOOT_TYPE=${BOOT_TYPE_ARG}"
else
    echo "Unknown bootloader. ${REBOOT_TYPE} is not supported."
    exit 1
fi
INITRD=$(echo $KERNEL_IMAGE | sed 's/vmlinuz/initrd.img/g')

# Install new FW for mellanox platforms before control plane goes down
# So on boot switch will not spend time to upgrade FW increasing the CP downtime
if [[ "$sonic_asic_type" == "mellanox" ]]; then
    MLNX_EXIT_SUCCESS="0"
    MLNX_EXIT_ERROR="1"

    MLNX_FW_UPGRADE_SCRIPT="/usr/bin/mlnx-fw-upgrade.sh"


    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        check_ffb || {
            echo "Warm reboot is not supported"
            exit "${MLNX_EXIT_ERROR}"
        }
    fi

    echo "Prepare MLNX ASIC to ${REBOOT_TYPE}: install new FW if required"

    ${MLNX_FW_UPGRADE_SCRIPT} --upgrade
    MLNX_EXIT_CODE="$?"
    if [[ "${MLNX_EXIT_CODE}" != "${MLNX_EXIT_SUCCESS}" ]]; then
        echo "Failed to burn MLNX FW: errno=${MLNX_EXIT_CODE}"
        exit "${MLNX_EXIT_ERROR}"
    fi
fi

# Load kernel into the memory
/sbin/kexec -l "$KERNEL_IMAGE" --initrd="$INITRD" --append="$BOOT_OPTIONS"

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    # Dump the ARP and FDB tables to files also as default routes for both IPv4 and IPv6
    # into /host/fast-reboot
    mkdir -p /host/fast-reboot
    /usr/bin/fast-reboot-dump.py -t /host/fast-reboot
fi

if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    # Freeze orchagent for warm restart
    # Try freeze 5 times, it is possible that the orchagent is in transient state and no opportunity to be freezed
    # Note: assume that 1 second is enough for orchagent to process the request and respone freeze or not
    debug "Pausing orchagent ..."
    for i in `seq 4 -1 0`; do
        docker exec -i swss /usr/bin/orchagent_restart_check -w 1000 && break
        echo "RESTARTCHECK failed $i" >&2
        if [[ "$i" = "0" ]]; then
            echo "RESTARTCHECK failed finally" >&2
            if [[ x"${FORCE}" == x"yes" ]]; then
                debug "Ignoring orchagent pausing failure ..."
                break;
            fi
            exit 10
        fi
        sleep 1
    done
fi

# Kill bgpd to start the bgp graceful restart procedure
debug "Stopping bgp ..."
docker exec -i bgp pkill -9 zebra
docker exec -i bgp pkill -9 bgpd
debug "Stopped  bgp ..."

# Kill lldp, otherwise it sends informotion about reboot
docker kill lldp > /dev/null

if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
    # Kill teamd, otherwise it gets down all LAGs
    # Note: teamd must be killed before syncd, because it will send the last packet through CPU port
    # TODO: stop teamd gracefully to allow teamd to send last valid update to be sure we'll have 90 seconds reboot time
    docker kill teamd > /dev/null
fi

# Kill swss dockers
docker kill swss > /dev/null

# Pre-shutdown syncd
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    initialize_pre_shutdown

    request_pre_shutdown

    wait_for_pre_shutdown_complete_or_fail

    # Warm reboot: dump state to host disk
    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
        redis-cli -n 1 FLUSHDB &> /dev/null
        redis-cli -n 2 FLUSHDB &> /dev/null
        redis-cli -n 5 FLUSHDB &> /dev/null
    fi

    # TODO: backup_database preserves FDB_TABLE
    # need to cleanup as well for fastfast boot case
    backup_database
fi

# Stop teamd gracefully
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
    debug "Stopping teamd ..."
    # Send USR1 signal to all teamd instances to stop them
    # It will prepare teamd for warm-reboot
    # Note: We must send USR1 signal before syncd, because it will send the last packet through CPU port
    docker exec -i teamd pkill -USR1 teamd || /bin/true > /dev/null
    debug "Stopped  teamd ..."
fi

debug "Stopping syncd ..."
systemctl stop syncd
debug "Stopped  syncd ..."

# Kill other containers to make the reboot faster
docker ps -q | xargs docker kill > /dev/null

# Stop the docker container engine. Otherwise we will have a broken docker storage
systemctl stop docker.service

if [[ "$REBOOT_TYPE" != "warm-reboot" ]]; then
    # Stop opennsl modules for Broadcom platform
    if [[ "$sonic_asic_type" = 'broadcom' ]];
    then
      service_name=$(systemctl list-units --plain --no-pager --no-legend --type=service | grep opennsl | cut -f 1 -d' ')
      systemctl stop "$service_name"
    fi
fi

# Stop kernel modules for Nephos platform
if [[ "$sonic_asic_type" = 'nephos' ]];
then
  systemctl stop nps-modules-`uname -r`.service
fi

# Update the reboot cause file to reflect that user issued this script
# Upon next boot, the contents of this file will be used to determine the
# cause of the previous reboot
echo "User issued '${REBOOT_SCRIPT_NAME}' command [User: ${REBOOT_USER}, Time: ${REBOOT_TIME}]" > ${REBOOT_CAUSE_FILE}

# Wait until all buffers synced with disk
sync
sleep 1
sync

# Reboot: explicity call Linux native reboot under sbin
debug "Rebooting with ${REBOOT_METHOD} to ${NEXT_SONIC_IMAGE} ..."
exec ${REBOOT_METHOD}

# Should never reach here
echo "${REBOOT_TYPE} failed!" >&2
exit 1