[Mellanox] Modified Platform API to support all firmware updates in s…

…ingle boot (#9608)

Why I did it
Requirements from Microsoft for fwutil update all state that all firmwares which support this upgrade flow must support upgrade within a single boot cycle. This conflicted with a number of Mellanox upgrade flows which have been revised to safely meet this requirement.

How I did it
Added --no-power-cycle flags to SSD and ONIE firmware scripts
Modified Platform API to call firmware upgrade flows with this new flag during fwutil update all
Added a script to our reboot plugin to handle installing firmwares in the correct order with prior to reboot
How to verify it
Populate platform_components.json with firmware for CPLD / BIOS / ONIE / SSD
Execute fwutil update all fw --boot cold
CPLD will burn / ONIE and BIOS images will stage / SSD will schedule for reboot
Reboot the switch
SSD will install / CPLD will refresh / switch will power cycle into ONIE
ONIE installer will upgrade ONIE and BIOS / switch will reboot back into SONiC
In SONiC run fwutil show status to check that all firmware upgrades were successful

device/mellanox/x86_64-mlnx_msn2700-r0/platform_reboot

@@ -3,6 +3,7 @@
 declare -r EXIT_SUCCESS="0"
 declare -r EXIT_ERROR="1"
 
+declare -r PENDING_COMPONENT_FW="/usr/bin/install-pending-fw.py"
 declare -r FW_UPGRADE_SCRIPT="/usr/bin/mlnx-fw-upgrade.sh"
 declare -r SYSFS_PWR_CYCLE="/sys/devices/platform/mlxplat/mlxreg-io/hwmon/hwmon*/pwr_cycle"
 
@@ -40,4 +41,6 @@ if [[ "${EXIT_CODE}" != "${EXIT_SUCCESS}" ]]; then
     fi
 fi
 
+${PENDING_COMPONENT_FW}
+
 SafePwrCycle

files/build_templates/sonic_debian_extension.j2

@@ -842,6 +842,7 @@ sudo cp $files_path/$ISSU_VERSION_FILE $FILESYSTEM_ROOT/etc/mlnx/issu-version
 sudo cp $files_path/$MLNX_FFB_SCRIPT $FILESYSTEM_ROOT/usr/bin/mlnx-ffb.sh
 sudo cp $files_path/$MLNX_ONIE_FW_UPDATE $FILESYSTEM_ROOT/usr/bin/$MLNX_ONIE_FW_UPDATE
 sudo cp $files_path/$MLNX_SSD_FW_UPDATE $FILESYSTEM_ROOT/usr/bin/$MLNX_SSD_FW_UPDATE
+sudo cp $files_path/$MLNX_INSTALL_PENDING_FW $FILESYSTEM_ROOT/usr/bin/$MLNX_INSTALL_PENDING_FW
 j2 platform/mellanox/mlnx-fw-upgrade.j2 | sudo tee $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
 sudo chmod 755 $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
 

platform/mellanox/install-pending-fw.dep

@@ -0,0 +1,10 @@
+# DPKG FRK
+
+DPATH := $($(MLNX_INSTALL_PENDING_FW)_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/install-pending-fw.mk $(PLATFORM_PATH)/install-pending-fw.dep
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES += $(addprefix $(DPATH),$(MLNX_INSTALL_PENDING_FW))
+
+$(MLNX_INSTALL_PENDING_FW)_CACHE_MODE := GIT_CONTENT_SHA
+$(MLNX_INSTALL_PENDING_FW)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(MLNX_INSTALL_PENDING_FW)_DEP_FILES := $(DEP_FILES)

platform/mellanox/install-pending-fw.mk

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2020-2021 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Firmware pending update checker and installer
+
+MLNX_INSTALL_PENDING_FW = install-pending-fw.py
+$(MLNX_INSTALL_PENDING_FW)_PATH = $(PLATFORM_PATH)/
+SONIC_COPY_FILES += $(MLNX_INSTALL_PENDING_FW)
+
+MLNX_FILES += $(MLNX_INSTALL_PENDING_FW)
+
+export MLNX_INSTALL_PENDING_FW

platform/mellanox/install-pending-fw.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from fwutil.lib import ComponentStatusProvider, PlatformComponentsParser
+from sonic_platform.component import ComponentCPLD, MPFAManager
+
+# Globals
+FW_STATUS_SCHEDULED = "scheduled"
+CPLD_FLAG = False
+
+# Init platform chassis helper classes
+csp = ComponentStatusProvider()
+pcp = PlatformComponentsParser(csp.is_modular_chassis())
+
+# Parse update status file
+update_status = csp.read_au_status_file_if_exists()
+
+if update_status is None:
+    exit(0)
+
+# Parse platform components file
+try:
+    pcp.parse_platform_components()
+except Exception as e:
+    print("Error parsing platform components. Firmware update failed: {}".format(str(e)))
+    print("System will reboot in 10 seconds please fix issue and run update command again.")
+    time.sleep(10)
+    exit(-1)
+
+# Iterate each component in the status file
+comp_install = []
+files = []
+
+for boot_type, components in update_status.items():
+    for comp in components:
+
+        # Skip if fw isn't scheduled for install at reboot
+        if comp["info"] != FW_STATUS_SCHEDULED: continue
+
+        # Get component object and target firmware file
+        key = comp["comp"]
+        comp_path = key.split("/")
+
+        if len(comp_path) == 3:
+            # Module component
+            _, parent_name, comp_name = comp_path
+            fw_file = pcp.module_component_map[parent_name][comp_name]["firmware"]
+            component = csp.module_component_map[parent_name][comp_name]
+        else: 
+            # Chassis component
+            parent_name, comp_name = comp_path
+            fw_file = pcp.chassis_component_map[parent_name][comp_name]["firmware"]
+            component = csp.chassis_component_map[parent_name][comp_name]
+
+        # Install firmware. If CPLD flag to be installed last due to force reboot during refresh
+        if type(component) == ComponentCPLD:
+            if CPLD_FLAG:
+                # Only need one refresh
+                continue
+            mpfa = MPFAManager(fw_file)
+            mpfa.extract()
+            if not mpfa.get_metadata().has_option('firmware', 'refresh'):
+                print("Failed to get CPLD refresh firmware. Skipping.")
+                continue
+            CPLD_FLAG = True
+            refresh_firmware = mpfa.get_metadata().get('firmware', 'refresh')
+            comp_install = comp_install + [component]
+            files = files + [os.path.join(mpfa.get_path(), refresh_firmware)]
+        else:
+            comp_install = [component] + comp_install
+            files = [fw_file] + files
+
+# Do install
+for i, c in enumerate(comp_install):
+    try:
+        if type(c) == ComponentCPLD:
+            c.install_firmware(files[i])
+        else:
+            c.install_firmware(files[i], allow_reboot=False)
+    except Exception as e:
+        print("Firmware install for {} FAILED with: {}".format(c.get_name(),e))
+

platform/mellanox/mlnx-onie-fw-update.sh

@@ -186,7 +186,12 @@ case "${cmd}" in
             rc=$?
             disable_onie_access
             if [[ ${rc} -eq 0 ]]; then
-                system_reboot
+                if [[ "${arg}" == "--no-reboot" ]]; then
+                    echo "INFO: ONIE firmware update successfully STAGED for install at NEXT reboot. Please reboot manually to complete installation."
+                    exit 0
+                else
+                    system_reboot
+                fi
             else
                 echo "ERROR: failed to enable ONIE firmware update mode"
                 exit ${rc}