[Build] Upgrade BFB build process to avoid default bfb and upgrade to…

… 3.8.5 ver (#25)

Signed-off-by: Vivek Reddy Karri <vkarri@nvidia.com>
Co-authored-by: Vivek Reddy Karri <vkarri@nvidia.com>

build_image.sh

@@ -294,7 +294,7 @@ elif [[ $IMAGE_TYPE = s2s && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then
     generate_s2s_installer_image_bluefield
 
 elif [[ $IMAGE_TYPE == bfb && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then
-    sudo --preserve-env /sonic/installer/bluefield/create_sonic_bfb --sonic_version $IMAGE_VERSION
+    sudo --preserve-env /sonic/installer/bluefield/create_sonic_bfb --kernel $KVERSION
     sudo chown $USER ./$OUTPUT_BFB_IMAGE
 
 elif [[ $IMAGE_TYPE == pxe && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then

installer/bluefield/install.sh

@@ -31,10 +31,6 @@ demo_volume_revision_label="SONiC-OS-${IMAGE_VERSION}"
 PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/mellanox/scripts"
 CHROOT_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-if [ -e /etc/bf.cfg ]; then
-	. /etc/bf.cfg
-fi
-
 rshimlog=`which bfrshlog 2> /dev/null`
 distro="SONiC"
 
@@ -47,6 +43,48 @@ log()
 	fi
 }
 
+#
+# Check auto configuration passed from boot-fifo
+#
+boot_fifo_path="/sys/bus/platform/devices/MLNXBF04:00/bootfifo"
+if [ -e "${boot_fifo_path}" ]; then
+	cfg_file=$(mktemp)
+	# Get 16KB assuming it's big enough to hold the config file.
+	dd if=${boot_fifo_path} of=${cfg_file} bs=4096 count=4
+
+	#
+	# Check the .xz signature {0xFD, '7', 'z', 'X', 'Z', 0x00} and extract the
+	# config file from it. Then start decompression in the background.
+	#
+	offset=$(strings -a -t d ${cfg_file} | grep -m 1 "7zXZ" | awk '{print $1}')
+	if [ -s "${cfg_file}" -a ."${offset}" != ."1" ]; then
+		log "INFO: Found bf.cfg"
+		cat ${cfg_file} | tr -d '\0' > /etc/bf.cfg
+	fi
+	rm -f $cfg_file
+fi
+
+if [ -e /etc/bf.cfg ]; then
+	. /etc/bf.cfg
+fi
+
+log_file=/sonic_install_log_file.log
+if [ "X${DEBUG}" == "Xyes" ]; then
+    touch $log_file
+fi
+ex(){
+    if [ "X${DEBUG}" == "Xyes" ]; then
+        # Execute the command and redirect the logs to the log file
+        echo "COMMAND: $@" >> $log_file
+        echo "STDOUT & STDERR:" >> $log_file
+        "$@" >> $log_file 2>&1
+        echo "EXT STATUS: $?" >> $log_file
+        echo -e "\n" >> $log_file
+    else
+        "$@" > /dev/null 2>&1
+    fi
+}
+
 log "INFO: $distro installation started"
 
 device=/dev/mmcblk0
@@ -58,26 +96,6 @@ while [ ! -b $device ]; do
     sleep 1
 done
 
-# Flash firmware from bfb to MMC device boot partitions (it alternates
-# between boot0 and boot1, that is why we call mlx-bootctl twice).
-
-# Update default.bfb
-bfb_location=/lib/firmware/mellanox/default.bfb
-
-if [ -f "$bfb_location" ]; then
-	/bin/rm -f /mnt/lib/firmware/mellanox/boot/default.bfb
-	mkdir -p /mnt/lib/firmware/mellanox/boot
-	cp $bfb_location /mnt/lib/firmware/mellanox/boot/default.bfb
-fi
-
-# Update eMMC boot partitions. Update via either capsule
-# path or default path (i.e., mlxbf-bootctl). This command
-# MUST be executed after 'update_boot' and 'install_grub',
-# otherwise the newly created boot option would be either
-# cleaned up or unset from default option.
-bfrec --policy dual
-sleep 3
-
 # Flash image
 bs=512
 reserved=34
@@ -111,36 +129,38 @@ EOF
 sync
 
 # Refresh partition table
-blockdev --rereadpt ${device} > /dev/null 2>&1
+ex blockdev --rereadpt ${device}
 
 if function_exists bfb_pre_install; then
 	bfb_pre_install
 fi
 
 # Generate some entropy
-mke2fs  ${device}p2 >> /dev/null
+ex mke2fs  ${device}p2 >> /dev/null
 
-mkdosfs ${device}p1 -n "system-boot"
-mkfs.ext4 -F ${device}p2 -L "SONiC-OS"
+ex mkdosfs ${device}p1 -n "system-boot"
+ex mkfs.ext4 -F ${device}p2 -L "SONiC-OS"
 
-fsck.vfat -a ${device}p1
+ex fsck.vfat -a ${device}p1
 
 mkdir -p /mnt
-mount -t ext4 -o defaults,rw ${device}p2 /mnt
+ex mount -t ext4 ${device}p2 /mnt
 mkdir -p /mnt/boot/efi
-mount -t vfat ${device}p1 /mnt/boot/efi
+ex mount -t vfat ${device}p1 /mnt/boot/efi
+
+if [ "X${DEBUG}" == "Xyes" ]; then
+    log "Extracting SONiC files"
+fi
 
-echo "Installing SONiC to /mnt/$image_dir"
 mkdir -p /mnt/$image_dir
 
 # Extract the INSTALLER_PAYLOAD to the $image_dir
 export EXTRACT_UNSAFE_SYMLINKS=1
-unzip -o /debian/$INSTALLER_PAYLOAD -x $FILESYSTEM_DOCKERFS -d /mnt/$image_dir
+ex unzip -o /debian/$INSTALLER_PAYLOAD -x $FILESYSTEM_DOCKERFS -d /mnt/$image_dir
 mkdir -p /mnt/$image_dir/$DOCKERFS_DIR
 unzip -op /debian/$INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz --warning=no-timestamp -f - -C /mnt/$image_dir/$DOCKERFS_DIR
 
 # Copy in the machine.conf file
-# TODO: Don't statically define bf_platform 
 cat <<EOF > /mnt/machine.conf
 bf_vendor=nvidia
 bf_machine=nvidia-bluefield
@@ -151,39 +171,15 @@ chmod a+r /mnt/machine.conf
 
 sync
 
-if (grep -qE "MemTotal:\s+16" /proc/meminfo > /dev/null 2>&1); then
-	echo "net.netfilter.nf_conntrack_max = 500000" > /mnt/usr/lib/sysctl.d/90-bluefield.conf
+if [ "X${DEBUG}" == "Xyes" ]; then
+    log "Installing GRUB"
 fi
 
-
-umount /mnt/boot/efi
-umount /mnt
-
-blockdev --rereadpt ${device} > /dev/null 2>&1
-
-fsck.vfat -a ${device}p1
-sync
-
-bfbootmgr --cleanall
-
-# Make it the boot partition
-mount -t efivarfs none /sys/firmware/efi/efivars
-mount ${device}p2 /mnt/
-mount ${device}p1 /mnt/boot/efi/
-
-if [ -x /usr/sbin/grub-install ]; then
-	grub-install \
-		--bootloader-id="SONiC-OS" \
-		--locale-directory=/mnt/usr/share/locale \
-		--efi-directory=/mnt/boot/efi/ \
-		--boot-directory=/mnt/ \
-		${device}p1
-else
-	if efibootmgr | grep buster; then
-		efibootmgr -b "$(efibootmgr | grep buster | cut -c 5-8)" -B
-	fi
-	efibootmgr -c -d "$device" -p 1 -L buster -l "\EFI\debian\grubaa64.efi"
-fi
+ex grub-install \
+    --bootloader-id="SONiC-OS" \
+    --efi-directory=/mnt/boot/efi/ \
+    --boot-directory=/mnt/ \
+    ${device}p1
 
 # Create a minimal grub.cfg that allows for:
 #   - configure the serial console
@@ -192,13 +188,12 @@ fi
 
 grub_cfg=$(mktemp)
 
-# Set a few GRUB_xxx environment variables that will be picked up and
-# used by the 50_onie_grub script.  This is similiar to what an OS
-# would specify in /etc/default/grub.
-#
-# GRUB_SERIAL_COMMAND
+# Modify GRUB_CMDLINE_LINUX from bf.cfg file if required
 # GRUB_CMDLINE_LINUX
-[ -r /debian/platform.conf ] && . /debian/platform.conf
+
+DEFAULT_GRUB_CMDLINE_LINUX="console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 quiet"
+GRUB_CMDLINE_LINUX=${GRUB_CMDLINE_LINUX:-"$DEFAULT_GRUB_CMDLINE_LINUX"}
+export GRUB_CMDLINE_LINUX
 
 # Add the logic to support grub-reboot and grub-set-default
 cat <<EOF >> $grub_cfg
@@ -228,7 +223,7 @@ else
 fi
 
 # Add common configuration, like the timeout
-cat <<EOF > $grub_cfg
+cat <<EOF >> $grub_cfg
 set timeout=5
 EOF
 
@@ -240,7 +235,7 @@ menuentry '$demo_grub_entry' {
         search --no-floppy --label --set=root SONiC-OS
         if [ x$grub_platform = xxen ]; then insmod xzio; insmod lzopio; fi
         echo 'Loading SONiC-OS Kernel'
-        linux   /$image_dir/boot/vmlinuz-5.10.0-8-2-arm64 root=$grub_cfg_root rw console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 fixrtc \
+        linux   /$image_dir/boot/vmlinuz-5.10.0-8-2-arm64 root=$grub_cfg_root rw $GRUB_CMDLINE_LINUX fixrtc \
                 loop=$image_dir/$FILESYSTEM_SQUASHFS loopfstype=squashfs                       \
                 systemd.unified_cgroup_hierarchy=0 \
                 apparmor=1 security=apparmor varlog_size=4096 systemd.unified_cgroup_hierarchy=0 \
@@ -251,22 +246,93 @@ menuentry '$demo_grub_entry' {
 EOF
 
 # Copy the grub.cfg onto the boot-directory as specified in the grub-install
-mkdir -p /mnt/grub && cp $grub_cfg /mnt/grub/grub.cfg
+ex mkdir -p /mnt/grub 
+ex cp $grub_cfg /mnt/grub/grub.cfg
 
 sync
 
+if [ "X${DEBUG}" == "Xyes" ]; then
+    log "GRUB CFG Updated"
+fi
+
 umount /mnt/boot/efi
 umount /mnt
-umount /sys/firmware/efi/efivars
+
+ex blockdev --rereadpt ${device}
+
+ex fsck.vfat -a ${device}p1
+sync
+
+if [ -e /lib/firmware/mellanox/boot/capsule/update.cap ]; then
+	ex bfrec --capsule /lib/firmware/mellanox/boot/capsule/update.cap
+fi
+
+ex bfbootmgr --cleanall
+
+# Make it the boot partition
+mounted_efivarfs=0
+if [ ! -d /sys/firmware/efi/efivars ]; then
+	ex mount -t efivarfs none /sys/firmware/efi/efivars
+	mounted_efivarfs=1
+fi
+
+if efibootmgr | grep SONiC-OS; then
+    ex efibootmgr -b "$(efibootmgr | grep SONiC-OS | cut -c 5-8)" -B
+fi
+ex efibootmgr -c -d "$device" -p 1 -L SONiC-OS -l "\EFI\SONiC-OS\grubaa64.efi"
+
+if [ "X${DEBUG}" == "Xyes" ]; then
+    log "EFIBootMgr Updated"
+fi
 
 BFCFG=`which bfcfg 2> /dev/null`
 if [ -n "$BFCFG" ]; then
-	$BFCFG
+	# Create PXE boot entries
+	if [ -e /etc/bf.cfg ]; then
+		mv /etc/bf.cfg /etc/bf.cfg.orig
+	fi
+
+	cat > /etc/bf.cfg << EOF
+BOOT0=DISK
+BOOT1=NET-NIC_P0-IPV4
+BOOT2=NET-NIC_P0-IPV6
+BOOT3=NET-NIC_P1-IPV4
+BOOT4=NET-NIC_P1-IPV6
+BOOT5=NET-OOB-IPV4
+BOOT6=NET-OOB-IPV6
+PXE_DHCP_CLASS_ID=$DHCP_CLASS_ID
+EOF
+
+	ex $BFCFG
+
+	# Restore the original bf.cfg
+	/bin/rm -f /etc/bf.cfg
+	if [ -e /etc/bf.cfg.orig ]; then
+		mv /etc/bf.cfg.orig /etc/bf.cfg
+	fi
+fi
+
+if [ $mounted_efivarfs -eq 1 ]; then
+	umount /sys/firmware/efi/efivars > /dev/null 2>&1
+fi
+
+if [ -n "$BFCFG" ]; then
+	ex $BFCFG
 fi
 
 if function_exists bfb_post_install; then
-	bfb_post_install
+	ex bfb_post_install
 fi
 
-log "INFO: Installation finished"
-log "INFO: Rebooting..."
+if [ "X${DEBUG}" == "Xyes" ]; then
+    # Copy the log file to Persistent Storage
+    mount -t ext4 ${device}p2 /mnt
+    mkdir -p /mnt/bfb-boot-logs/
+    dmesg > /mnt/bfb-boot-logs/installer_bfb_dmesg.log
+    mv $log_file /mnt/bfb-boot-logs/$log_file
+    umount /mnt
+    log "Rebooting..."
+    log "Installation finished"
+    # Wait for these messages to be pulled by the rshim service
+    sleep 3
+fi

platform/nvidia-bluefield/mlx-bootctl/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!Makefile

platform/nvidia-bluefield/mlx-bootctl/Makefile

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -ex
+
+MLX_BOOTCTL = mlx-bootctl-$(MLX_BOOTCTL_DRIVER_VERSION)
+MLX_BOOTCTL_TAR = ${MLX_BOOTCTL}.tar.gz
+
+$(addprefix $(DEST)/, $(MLX_BOOTCTL_DRIVER)): $(DEST)/% :
+	rm -rf ${MLX_BOOTCTL} ${MLX_BOOTCTL_TAR} *.buildinfo *.changes *.deb
+	wget -N "https://linux.mellanox.com/public/repo/bluefield/${BSD_VER}/extras/SOURCES/$(MLX_BOOTCTL_TAR)"
+	tar -z -f $(MLX_BOOTCTL_TAR) -x
+
+	pushd ${MLX_BOOTCTL}  
+
+	# Build the package
+	KVER=$(KVERSION) WITH_DKMS=0 dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+
+	mv $* $(DEST)/
+
+	popd

platform/nvidia-bluefield/recipes/bootimages.dep

@@ -2,5 +2,5 @@ DEP_FILES := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/$(RECIPE_DIR)/bootimage
 DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
 
 $(BOOTIMAGES)_CACHE_MODE := GIT_CONTENT_SHA
-$(BOOTIMAGES)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(BOOTIMAGES)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) $(BSD_VER) $(BSD_REV) $(BFB_IMG_TYPE)
 $(BOOTIMAGES)_DEP_FILES := $(DEP_FILES)

platform/nvidia-bluefield/recipes/bootimages.mk

@@ -14,19 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-BOOTIMAGES_VERSION = 3.8.0-11969
-BOOTIMAGES_BASE_PATH = $(PLATFORM_PATH)/sdk-src/sonic-bluefield-packages/bin/BlueField-$(BFINSTALL_VERSION)
+IMAGE_TYPE=dev
 
-BOOTIMAGES_BASE_URL =
+BOOTIMAGES_BASE_URL = https://linux.mellanox.com/public/repo/bluefield/3.8.5/bootimages/$(IMAGE_TYPE)
+BOOTIMAGES = mlxbf-bootimages_$(BSD_VER)-$(BSD_REV)_arm64.deb
 
-BOOTIMAGES = mlxbf-bootimages_$(BFINSTALL_VERSION)_arm64.deb
+# Required for building BFB image
+UPDATE_CAP = update.cap
 
-ifneq ($(BOOTIMAGES_BASE_URL), )
 $(BOOTIMAGES)_URL = $(BOOTIMAGES_BASE_URL)/$(BOOTIMAGES)
 SONIC_ONLINE_DEBS += $(BOOTIMAGES)
-else
-$(BOOTIMAGES)_PATH = $(BOOTIMAGES_BASE_PATH)
-SONIC_COPY_DEBS += $(BOOTIMAGES)
-endif
 
-export BOOTIMAGES
+$(UPDATE_CAP)_URL = $(BOOTIMAGES_BASE_URL)/$(UPDATE_CAP)
+SONIC_ONLINE_FILES += $(UPDATE_CAP)
+
+export BOOTIMAGES UPDATE_CAP

platform/nvidia-bluefield/recipes/installer-image.mk

@@ -43,11 +43,6 @@ $(SONIC_BF2_IMAGE_BASE)_INSTALLS += $(ETHTOOL) \
 $(SONIC_BF2_IMAGE_BASE)_DOCKERS = $(filter-out $(DOCKER_ORCHAGENT) $(DOCKER_PLATFORM_MONITOR),$(SONIC_INSTALL_DOCKER_IMAGES))
 
 
-# TODO:
-# INSTALLS: $(MLNX_BLUEFIELD_BUILD_SCRIPTS) $(MLXBF_BOOTCTL_DEB)
-# DEPENDS: $(MLNX_BLUEFIELD_BUILD_SCRIPTS)
-# Add these to PXE and BFB image once they are working and ready
-
 # A compressed archive which contains individual files required for PXE boot
 SONIC_BF2_IMAGE_PXE = $(SONIC_BF2_IMAGE_BASE).gz
 $(SONIC_BF2_IMAGE_PXE)_IMAGE_TYPE = pxe
@@ -74,6 +69,6 @@ $(SONIC_BF2_IMAGE_BFB)_INSTALLS += $($(SONIC_BF2_IMAGE_BASE)_INSTALLS)
 $(SONIC_BF2_IMAGE_BFB)_DEPENDS += $($(SONIC_BF2_IMAGE_BASE)_DEPENDS)
 $(SONIC_BF2_IMAGE_BFB)_DOCKERS += $($(SONIC_BF2_IMAGE_BASE)_DOCKERS)
 $(SONIC_BF2_IMAGE_BFB)_LAZY_INSTALLS += $($(SONIC_BF2_IMAGE_BASE)_LAZY_INSTALLS)
-$(SONIC_BF2_IMAGE_BFB)_FILES += $(BFINSTALL_FILE)
+$(SONIC_BF2_IMAGE_BFB)_FILES += $(UPDATE_CAP) $(MLX_BOOTCTL_DRIVER)
 
 SONIC_INSTALLERS += $(SONIC_BF2_IMAGE_PXE) $(SONIC_BF2_IMAGE_BIN) $(SONIC_BF2_IMAGE_BFB)

platform/nvidia-bluefield/recipes/mlx-bootctl.dep

@@ -0,0 +1,9 @@
+
+SPATH       := $($(MLX_BOOTCTL_DRIVER)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.mk $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(MLX_BOOTCTL_DRIVER)_CACHE_MODE  := GIT_CONTENT_SHA
+$(MLX_BOOTCTL_DRIVER)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(MLX_BOOTCTL_DRIVER)_DEP_FILES   := $(DEP_FILES)

platform/nvidia-bluefield/recipes/mlx-bootctl.mk

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+MLX_BOOTCTL_DRIVER_VERSION = 1.5
+MLX_BOOTCTL_DRIVER = mlx-bootctl.ko
+
+$(MLX_BOOTCTL_DRIVER)_SRC_PATH = $(PLATFORM_PATH)/mlx-bootctl
+$(MLX_BOOTCTL_DRIVER)_DEPENDS += $(LINUX_HEADERS) $(LINUX_HEADERS_COMMON)
+SONIC_MAKE_FILES += $(MLX_BOOTCTL_DRIVER)
+
+export MLX_BOOTCTL_DRIVER_VERSION
+export MLX_BOOTCTL_DRIVER

platform/nvidia-bluefield/rules.mk

@@ -18,8 +18,10 @@
 RECIPE_DIR = recipes
 
 # Bluefied Software Distribution Version
-BSD_VER=3.8.0
-export BSD_VER
+BSD_VER=3.8.5
+BSD_REV=12027
+BFB_IMG_TYPE=prod
+export BSD_VER BSD_REV BFB_IMG_TYPE
 
 
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/bfinstall.mk
@@ -30,6 +32,7 @@ include $(PLATFORM_PATH)/$(RECIPE_DIR)/i2c-mlxbf.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/bluefield-edac.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/ipmb-dev-int.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/ipmb-host.mk
+include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-trio.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlxbf-bootctl.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlxbf-livefish.mk

sonic-slave-bullseye/Dockerfile.j2

@@ -362,8 +362,15 @@ RUN apt-get update && apt-get install -y \
         libgirepository1.0-dev \
         libsystemd-dev \
         pkg-config \
+# For building bfb image
+        fdisk \
+        usbutils \
+        dosfstools \
+        efibootmgr \
+        grub-efi-arm64-bin \
+        grub2-common \
 # For socat build
-	net-tools \
+        net-tools \
 # For audisp-tacplus
         libauparse-dev \
         auditd