diff --git a/bfb-mk-demo.sh b/bfb-mk-demo.sh deleted file mode 100755 index 1e410a1619e2..000000000000 --- a/bfb-mk-demo.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -REPO="mellanox/bluefield" -TAG="bfb_builder_denian10-5.4-2.4.1.3-3.7.1.11866-1" - -CDIR="/root/sonic/conf" -SDIR="/root/sonic" -DDIR="/sonic/" -TARGET_MACHINE=nvidia-bluefield - -fail_with_cmd() -{ - echo $1 - exit 1 -} - -# Check if the image already exists -docker image inspect $REPO:$TAG &> /dev/null -if [ $? -ne 0 ]; then - echo "Image doesn't Exist, fetching $TAG from $REPO" - docker pull $REPO:$TAG || fail_with_cmd "docker pull failed" -fi - -. onie-image-arm64.conf - -SONIC_VERSION=$(cat $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml | grep build_version | cut -f2 -d" ") - -echo "SONIC_VERSION: $SONIC_VERSION" - -if [ ! -f "$ONIE_INSTALLER_PAYLOAD" ]; then - echo "SONIC Related Payload '$ONIE_INSTALLER_PAYLOAD' is not found at $(pwd) exiting" - exit 1 -fi - -docker run --privileged --init -e container=docker\ - -e "CDIR=$CDIR" -e "SDIR=$SDIR" -e "DDIR=$DDIR" \ - --mount type=bind,source="$(pwd)",target=/sonic \ - $REPO:$TAG \ - bash -c "mkdir -p $SDIR && - rm -rf $SDIR/* && - mkdir -p $CDIR && - cp /sonic/$ONIE_INSTALLER_PAYLOAD $CDIR && - cp /sonic/onie-image-arm64.conf $CDIR && - cp /sonic/installer/bluefield/create_sonic_bfb $SDIR && - cp /sonic/installer/bluefield/install.sh $SDIR && - cp /root/workspace/install.bfb $SDIR && - ls $SDIR && - /root/sonic/create_sonic_bfb -s $SONIC_VERSION" || fail_with_cmd "docker run failed" - -echo "Building BFB Finished" -exit 0 diff --git a/build_image.sh b/build_image.sh index cf0cd1b0009b..113ac8ef7efe 100755 --- a/build_image.sh +++ b/build_image.sh @@ -294,7 +294,7 @@ elif [[ $IMAGE_TYPE = s2s && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then generate_s2s_installer_image_bluefield elif [[ $IMAGE_TYPE == bfb && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then - sudo --preserve-env /sonic/installer/bluefield/create_sonic_bfb --sonic_version $IMAGE_VERSION + sudo --preserve-env /sonic/installer/bluefield/create_sonic_bfb --kernel $KVERSION sudo chown $USER ./$OUTPUT_BFB_IMAGE elif [[ $IMAGE_TYPE == pxe && $CONFIGURED_PLATFORM == nvidia-bluefield ]]; then diff --git a/installer/bluefield/create_sonic_bfb b/installer/bluefield/create_sonic_bfb index 757cbbfd7a17..302d103c2b37 100755 --- a/installer/bluefield/create_sonic_bfb +++ b/installer/bluefield/create_sonic_bfb @@ -17,61 +17,35 @@ # set -e +WDIR= IMAGE_VER= +KERNEL= +CHROOT_DIR= +INITRD= +VMLINUZ= +MODULES_DIR= +EFIVARS_MOD= + CDIR=/sonic/ -SDIR=/sonic/installer/bluefield -DDIR=/sonic/ -WDIR= -CAPSULE=/lib/firmware/mellanox/boot/capsule/MmcBootCap +SDIR=$CDIR/installer/bluefield +DDIR=$CDIR +CAPSULE=$CDIR/$FILES_PATH/update.cap +BOOTCTL_DRIVER=$CDIR/$FILES_PATH/mlx-bootctl.ko +TMFIFO_DRIVER=$CDIR/$FILES_PATH/mlxbf-tmfifo.ko TARGET_MACHINE=nvidia-bluefield +BFB="${BFB:-/lib/firmware/mellanox/boot/default.bfb}" usage() { cat << EOF Usage: `basename $0` [ OPTIONS ] OPTIONS: -i, --install_bfb Installation BFB to be used as a basis for the target BFB. Default: $BFB --s, --sonic_version SONiC version to be installed +-k, --kernel Kernel version for the SmartNIC. -v, --verbose Run script in verbose mode. Will print out each step of execution. -h, --help Display help EOF } -validate_config() { - if [[ $BFB == "" ]]; then - if [[ $BFINSTALL_FILE == "" ]]; then - echo "Error! Path to reference BFB is not specified! Please set BFINSTALL_FILE variable." - exit 1 - fi - - BFB=/sonic/$FILES_PATH/$BFINSTALL_FILE - if [ ! -f $BFB ]; then - echo "Error! Reference BFB does not exist!" - exit 1 - fi - fi - - if [ ! -n "$OUTPUT_BFB_IMAGE" ]; then - echo "OUTPUT_BFB_IMAGE name not defined. Exiting.." - exit 1 - fi - - if [ ! -f "$CDIR/$ONIE_INSTALLER_PAYLOAD" ]; then - echo "$ONIE_INSTALLER_PAYLOAD not found. Exiting.." - exit 1 - fi - - mkbfb=`which mlx-mkbfb` - if [ ! -x "${mkbfb}" ]; then - echo "Error! mlx-mkbfb is required to build BFB image" - exit 1 - fi - - if [ ! -e $CAPSULE ]; then - echo "ERROR: Capsule file $CAPSULE does not exist" - exit 1 - fi -} - parse_args() { while [[ $@ != "" ]]; do case $1 in @@ -83,9 +57,9 @@ parse_args() { shift BFB=$1 ;; - -s|--sonic_version) + -k|--kernel) shift - IMAGE_VER=$1 + KERNEL=$1 ;; *) usage @@ -96,6 +70,69 @@ parse_args() { done } +validate_config() { + if [[ ! -d "$CHROOT_DIR" ]]; then + echo "[create_sonic_bfb] Error! Path to CHROOT not found" + exit 1 + fi + + if [[ ! -f "$INITRD" ]]; then + echo "[create_sonic_bfb] Error! SONiC INITRD not found" + exit 1 + fi + + if [[ ! -f "$VMLINUZ" ]]; then + echo "[create_sonic_bfb] Error! SONiC VMLINUZ not found" + exit 1 + fi + + if [[ ! -d "$MODULES_DIR" ]]; then + echo "[create_sonic_bfb] Error! Path to Kernel Modules not found" + exit 1 + fi + + if [[ ! -n "$OUTPUT_BFB_IMAGE" ]]; then + echo "[create_sonic_bfb] Error! OUTPUT_BFB_IMAGE name not defined. Exiting.." + exit 1 + fi + + if [[ ! -f "$CDIR/$ONIE_INSTALLER_PAYLOAD" ]]; then + echo "$ONIE_INSTALLER_PAYLOAD not found. Exiting.." + exit 1 + fi + + mkbfb=`which mlx-mkbfb` + if [[ ! -x "${mkbfb}" ]]; then + echo "Error! mlx-mkbfb is required to build BFB image" + exit 1 + fi + + if [[ ! -f $CAPSULE ]]; then + echo "ERROR: Capsule file $CAPSULE does not exist" + exit 1 + fi + + if [[ ! -f $BOOTCTL_DRIVER ]]; then + echo "ERROR: Bootctl driver $BOOTCTL_DRIVER does not exist" + exit 1 + fi + + if [[ ! -f $TMFIFO_DRIVER ]]; then + echo "ERROR: Bootctl driver $TMFIFO_DRIVER does not exist" + exit 1 + fi + + if [[ ! -f $EFIVARS_MOD ]]; then + echo "ERROR: EFIVARS driver $EFIVARS_MOD does not exist" + exit 1 + fi + + if [[ ! -f $BFB ]]; then + echo "ERROR: Default BFB $BFB does not exist" + exit 1 + fi +} + cleanup_workdir() { rm -rf $WDIR } @@ -109,7 +146,8 @@ create_workdir() { fi } -initramfs_put_installer() { +add_sonic_to_initramfs() { + # Add the logic to put second stage installer into bfb initramfs mkdir -p debian cp ${SDIR}/install.sh temp_install.sh @@ -124,61 +162,191 @@ initramfs_put_installer() { # Copy the INSTALLER payload cp $CDIR/$ONIE_INSTALLER_PAYLOAD ./debian/ - cat > etc/init.d/install-debian << EOF -#!/bin/sh +cat > scripts/initrd-install << EOF +#!/bin/bash -echo -echo "=================================" | tee /dev/kmsg -echo "Installing SONiC. Please wait..." | tee /dev/kmsg -echo "=================================" | tee /dev/kmsg +printf_msg() +{ + echo "$@" | tee /dev/kmsg + return 0 +} + +depmod -a $KERNEL > /dev/null 2>&1 +insmod /usr/lib/modules/5.10.0-8-2-arm64/kernel/drivers/virtio/virtio.ko > /dev/null 2>&1 +insmod /usr/lib/modules/5.10.0-8-2-arm64/kernel/drivers/virtio/virtio_ring.ko > /dev/null 2>&1 +insmod /efivarfs.ko > /dev/null 2>&1 +insmod /mlx-bootctl.ko > /dev/null 2>&1 +insmod /mlxbf-tmfifo.ko > /dev/null 2>&1 -sleep 5 +printf_msg "=================================" +printf_msg "Installing SONiC. Please wait..." +printf_msg "=================================" -/bin/sh /debian/install.sh +/bin/bash /debian/install.sh if [ \$? -eq 0 ]; then - echo "===================================" | tee /dev/kmsg - echo "Installation finished. Rebooting..." | tee /dev/kmsg - echo "===================================" | tee /dev/kmsg - echo - sleep 3 - /sbin/shutdown -r 0 + printf_msg "===================================" + printf_msg "Installation finished. Rebooting..." + printf_msg "===================================" + printf_msg + reboot -f else - echo "========================" | tee /dev/kmsg - echo "Failed to install SONiC" | tee /dev/kmsg - echo "========================" | tee /dev/kmsg + printf_msg "========================" + printf_msg "Failed to install SONiC" + printf_msg "========================" fi EOF - chmod 755 etc/init.d/install-debian - ln -snf ../init.d/install-debian etc/rcS.d/S99install-debian + chmod +x scripts/initrd-install + sudo rm -f conf/conf.d/debian-core* + sudo sh -c 'echo "BOOT=initrd-install" > conf/conf.d/initrd_install' + sudo sh -c 'echo "ROOT=\"LABEL=writable\"" > conf/conf.d/default_root' } -create_initramfs() { - $mkbfb -x ${BFB} - local initramfs_template=$PWD/dump-initramfs-v0 - local sonic_installer_initramfs=$PWD/sonic-installer-initramfs - if [ ! -f $initramfs_template ]; then - echo "Error! BFB initramfs ('$initramfs_template') is not found!" - exit 1 +copy_bin() +{ + if [ -e $1 ]; then + bin=$1 + else + bin=`which $1 2> /dev/null` fi - local initramfs_dir=$WDIR/initramfs - mkdir -p $initramfs_dir - pushd $initramfs_dir >/dev/null - cpio -id < $initramfs_template + if [[ -z "$bin" ]]; then + echo "ERROR: Cannot find $1" + exit 1 + else + sudo mkdir -p .`dirname $bin` + if [ ! -e .${bin} ]; then + sudo cp -a $bin .${bin} + fi - initramfs_put_installer + # Copy dependencies + for lib in `ldd $bin 2> /dev/null | grep '=>' | awk '{print $3}'` + do + if [ -e .$lib ]; then + continue + fi + sudo mkdir -p .`dirname $lib` + sudo cp -a $lib .$lib + if [ -h $lib ]; then + tlib=`readlink -f $lib` + if [ ! -e .$tlib ]; then + sudo mkdir -p .`dirname $tlib` + sudo cp $tlib .$tlib + fi + fi + done + fi +} - find . | cpio -H newc -o | gzip -9 > $sonic_installer_initramfs +create_bfb_image() { - popd >/dev/null + pushd $WDIR - echo $sonic_installer_initramfs -} + # Copy the initrd into the work directory + initramfs=$(realpath $INITRD) + cp $initramfs $WDIR/dump-initramfs-v0 + initramfs=$WDIR/dump-initramfs-v0 + + case "$(file --brief --mime-type "$initramfs")" in + "application/x-lzma") + cat_initrd="lzcat" ;; + "application/x-lz4") + cat_initrd="lz4cat" ;; + *) + cat_initrd="zcat" ;; + esac + + echo "Rebuilding $initramfs" + + mkdir -p ${WDIR}/initramfs + pushd initramfs + $cat_initrd "$initramfs" | cpio -i + + # Remove tools coming with busybox + for tool in `dpkg -L mlxbf-scripts | grep bin/` \ + `dpkg -L grub2-common` \ + `dpkg -L dosfstools` \ + `dpkg -L e2fsprogs` \ + `dpkg -L fdisk` \ + `dpkg -L kmod | grep -v share` \ + `dpkg -L pciutils | grep -v share` \ + `dpkg -L usbutils | grep -v share` \ + `dpkg -L mft` \ + `dpkg -L mft-oem` \ + `dpkg -L kernel-mft-modules` \ + `dpkg -L tar` + do + if [ -d $tool ]; then + continue + fi + /bin/rm -f .${tool} + done + + for tool in `dpkg -L mlxbf-scripts | grep bin/` \ + `dpkg -L dosfstools | grep -v share` \ + `dpkg -L e2fsprogs | grep -v share` \ + `dpkg -L grub2-common` \ + `dpkg -L fdisk | grep -v share` \ + `dpkg -L kmod | grep -v share` \ + `dpkg -L pciutils | grep -v share` \ + `dpkg -L usbutils | grep -v share` \ + `dpkg -L mft` \ + `dpkg -L mft-oem` \ + `dpkg -L kernel-mft-modules` \ + `dpkg -L tar | grep -v share` \ + `dpkg -L grub-efi-arm64-bin` \ + xz efibootmgr gcc-10 bash getopt hexdump lspci perl \ + lsblk shutdown systemctl strings aarch64-linux-gnu-strings \ + mlxbf-bootctl id mkfs fsck + do + if [ -d $tool ]; then + continue + fi + copy_bin $tool + done + + mkdir -p usr/share/misc/ bin/ var/log/watchdog + cp $CHROOT_DIR/usr/share/misc/pci.ids usr/share/misc/ + cp $BOOTCTL_DRIVER . + cp $TMFIFO_DRIVER . + cp $EFIVARS_MOD . + + mkdir -p ./lib/firmware/mellanox/boot/ + cp /lib/firmware/mellanox/boot/default.bfb ./lib/firmware/mellanox/boot/default.bfb + cp -a /lib/firmware/mellanox/boot/capsule ./lib/firmware/mellanox/boot/ + cp $CAPSULE ./lib/firmware/mellanox/boot/capsule/ + mkdir -p mnt dev sys proc + + add_sonic_to_initramfs + + # Make initramfs with new debian + find . -print0 | sudo cpio --null -o --format=newc | gzip -9 > "$initramfs" + + popd + + # Make bfb + boot_args=$(mktemp) + boot_path=$(mktemp) + boot_desc=$(mktemp) + printf "console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 initrd=initramfs" > \ + "$boot_args" + + printf "VenHw(F019E406-8C9C-11E5-8797-001ACA00BFC4)/Image" > "$boot_path" + printf "Linux from rshim" > "$boot_desc" + vmlinuz=$WDIR/vmlinuz + cat $VMLINUZ > "$vmlinuz" + + $mkbfb --image "$vmlinuz" \ + --initramfs "$initramfs" \ + --capsule "$CAPSULE" \ + --boot-args "$boot_args" \ + --boot-path "$boot_path" \ + --boot-desc "$boot_desc" \ + ${BFB} ${DDIR}/${OUTPUT_BFB_IMAGE} -create_bfb_image() { - local initramfs=$1 - $mkbfb --initramfs $initramfs --capsule $CAPSULE ${BFB} ${DDIR}/${OUTPUT_BFB_IMAGE} echo "BFB is ready: $(readlink -f ${DDIR}/${OUTPUT_BFB_IMAGE})" + + rm -rf $boot_args $boot_path $boot_desc + popd } main() { @@ -187,15 +355,18 @@ main() { . $CDIR/onie-image-arm64.conf + CHROOT_DIR=$CDIR/$FILESYSTEM_ROOT + INITRD=$CDIR/$FILESYSTEM_ROOT/boot/initrd.img-$KERNEL + VMLINUZ=$CDIR/$FILESYSTEM_ROOT/boot/vmlinuz-$KERNEL + MODULES_DIR=$CDIR/$FILESYSTEM_ROOT/lib/modules/$KERNEL + EFIVARS_MOD=$MODULES_DIR/kernel/fs/efivarfs/efivarfs.ko + IMAGE_VER=$(cat $CHROOT_DIR/etc/sonic/sonic_version.yml | grep "build_version" | sed -e "s/build_version: //g;s/'//g") validate_config create_workdir echo "Work directory: $WDIR" - pushd $WDIR - local sonic_installer_initramfs=$(create_initramfs) - create_bfb_image $sonic_installer_initramfs - popd + create_bfb_image } main $@ diff --git a/installer/bluefield/install.sh b/installer/bluefield/install.sh index adc9453b17c0..bde89281ad0d 100755 --- a/installer/bluefield/install.sh +++ b/installer/bluefield/install.sh @@ -31,10 +31,6 @@ demo_volume_revision_label="SONiC-OS-${IMAGE_VERSION}" PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/mellanox/scripts" CHROOT_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -if [ -e /etc/bf.cfg ]; then - . /etc/bf.cfg -fi - rshimlog=`which bfrshlog 2> /dev/null` distro="SONiC" @@ -47,6 +43,48 @@ log() fi } +# +# Check auto configuration passed from boot-fifo +# +boot_fifo_path="/sys/bus/platform/devices/MLNXBF04:00/bootfifo" +if [ -e "${boot_fifo_path}" ]; then + cfg_file=$(mktemp) + # Get 16KB assuming it's big enough to hold the config file. + dd if=${boot_fifo_path} of=${cfg_file} bs=4096 count=4 + + # + # Check the .xz signature {0xFD, '7', 'z', 'X', 'Z', 0x00} and extract the + # config file from it. Then start decompression in the background. + # + offset=$(strings -a -t d ${cfg_file} | grep -m 1 "7zXZ" | awk '{print $1}') + if [ -s "${cfg_file}" -a ."${offset}" != ."1" ]; then + log "INFO: Found bf.cfg" + cat ${cfg_file} | tr -d '\0' > /etc/bf.cfg + fi + rm -f $cfg_file +fi + +if [ -e /etc/bf.cfg ]; then + . /etc/bf.cfg +fi + +log_file=/sonic_install_log_file.log +if [ "X${DEBUG}" == "Xyes" ]; then + touch $log_file +fi +ex(){ + if [ "X${DEBUG}" == "Xyes" ]; then + # Execute the command and redirect the logs to the log file + echo "COMMAND: $@" >> $log_file + echo "STDOUT & STDERR:" >> $log_file + "$@" >> $log_file 2>&1 + echo "EXT STATUS: $?" >> $log_file + echo -e "\n" >> $log_file + else + "$@" > /dev/null 2>&1 + fi +} + log "INFO: $distro installation started" device=/dev/mmcblk0 @@ -58,26 +96,6 @@ while [ ! -b $device ]; do sleep 1 done -# Flash firmware from bfb to MMC device boot partitions (it alternates -# between boot0 and boot1, that is why we call mlx-bootctl twice). - -# Update default.bfb -bfb_location=/lib/firmware/mellanox/default.bfb - -if [ -f "$bfb_location" ]; then - /bin/rm -f /mnt/lib/firmware/mellanox/boot/default.bfb - mkdir -p /mnt/lib/firmware/mellanox/boot - cp $bfb_location /mnt/lib/firmware/mellanox/boot/default.bfb -fi - -# Update eMMC boot partitions. Update via either capsule -# path or default path (i.e., mlxbf-bootctl). This command -# MUST be executed after 'update_boot' and 'install_grub', -# otherwise the newly created boot option would be either -# cleaned up or unset from default option. -bfrec --policy dual -sleep 3 - # Flash image bs=512 reserved=34 @@ -111,36 +129,38 @@ EOF sync # Refresh partition table -blockdev --rereadpt ${device} > /dev/null 2>&1 +ex blockdev --rereadpt ${device} if function_exists bfb_pre_install; then bfb_pre_install fi # Generate some entropy -mke2fs ${device}p2 >> /dev/null +ex mke2fs ${device}p2 >> /dev/null -mkdosfs ${device}p1 -n "system-boot" -mkfs.ext4 -F ${device}p2 -L "SONiC-OS" +ex mkdosfs ${device}p1 -n "system-boot" +ex mkfs.ext4 -F ${device}p2 -L "SONiC-OS" -fsck.vfat -a ${device}p1 +ex fsck.vfat -a ${device}p1 mkdir -p /mnt -mount -t ext4 -o defaults,rw ${device}p2 /mnt +ex mount -t ext4 ${device}p2 /mnt mkdir -p /mnt/boot/efi -mount -t vfat ${device}p1 /mnt/boot/efi +ex mount -t vfat ${device}p1 /mnt/boot/efi + +if [ "X${DEBUG}" == "Xyes" ]; then + log "Extracting SONiC files" +fi -echo "Installing SONiC to /mnt/$image_dir" mkdir -p /mnt/$image_dir # Extract the INSTALLER_PAYLOAD to the $image_dir export EXTRACT_UNSAFE_SYMLINKS=1 -unzip -o /debian/$INSTALLER_PAYLOAD -x $FILESYSTEM_DOCKERFS -d /mnt/$image_dir +ex unzip -o /debian/$INSTALLER_PAYLOAD -x $FILESYSTEM_DOCKERFS -d /mnt/$image_dir mkdir -p /mnt/$image_dir/$DOCKERFS_DIR unzip -op /debian/$INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz --warning=no-timestamp -f - -C /mnt/$image_dir/$DOCKERFS_DIR # Copy in the machine.conf file -# TODO: Don't statically define bf_platform cat < /mnt/machine.conf bf_vendor=nvidia bf_machine=nvidia-bluefield @@ -151,39 +171,15 @@ chmod a+r /mnt/machine.conf sync -if (grep -qE "MemTotal:\s+16" /proc/meminfo > /dev/null 2>&1); then - echo "net.netfilter.nf_conntrack_max = 500000" > /mnt/usr/lib/sysctl.d/90-bluefield.conf +if [ "X${DEBUG}" == "Xyes" ]; then + log "Installing GRUB" fi - -umount /mnt/boot/efi -umount /mnt - -blockdev --rereadpt ${device} > /dev/null 2>&1 - -fsck.vfat -a ${device}p1 -sync - -bfbootmgr --cleanall - -# Make it the boot partition -mount -t efivarfs none /sys/firmware/efi/efivars -mount ${device}p2 /mnt/ -mount ${device}p1 /mnt/boot/efi/ - -if [ -x /usr/sbin/grub-install ]; then - grub-install \ - --bootloader-id="SONiC-OS" \ - --locale-directory=/mnt/usr/share/locale \ - --efi-directory=/mnt/boot/efi/ \ - --boot-directory=/mnt/ \ - ${device}p1 -else - if efibootmgr | grep buster; then - efibootmgr -b "$(efibootmgr | grep buster | cut -c 5-8)" -B - fi - efibootmgr -c -d "$device" -p 1 -L buster -l "\EFI\debian\grubaa64.efi" -fi +ex grub-install \ + --bootloader-id="SONiC-OS" \ + --efi-directory=/mnt/boot/efi/ \ + --boot-directory=/mnt/ \ + ${device}p1 # Create a minimal grub.cfg that allows for: # - configure the serial console @@ -192,13 +188,12 @@ fi grub_cfg=$(mktemp) -# Set a few GRUB_xxx environment variables that will be picked up and -# used by the 50_onie_grub script. This is similiar to what an OS -# would specify in /etc/default/grub. -# -# GRUB_SERIAL_COMMAND +# Modify GRUB_CMDLINE_LINUX from bf.cfg file if required # GRUB_CMDLINE_LINUX -[ -r /debian/platform.conf ] && . /debian/platform.conf + +DEFAULT_GRUB_CMDLINE_LINUX="console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 quiet" +GRUB_CMDLINE_LINUX=${GRUB_CMDLINE_LINUX:-"$DEFAULT_GRUB_CMDLINE_LINUX"} +export GRUB_CMDLINE_LINUX # Add the logic to support grub-reboot and grub-set-default cat <> $grub_cfg @@ -228,7 +223,7 @@ else fi # Add common configuration, like the timeout -cat < $grub_cfg +cat <> $grub_cfg set timeout=5 EOF @@ -240,7 +235,7 @@ menuentry '$demo_grub_entry' { search --no-floppy --label --set=root SONiC-OS if [ x$grub_platform = xxen ]; then insmod xzio; insmod lzopio; fi echo 'Loading SONiC-OS Kernel' - linux /$image_dir/boot/vmlinuz-5.10.0-8-2-arm64 root=$grub_cfg_root rw console=ttyAMA1 console=hvc0 console=ttyAMA0 earlycon=pl011,0x01000000 earlycon=pl011,0x01800000 fixrtc \ + linux /$image_dir/boot/vmlinuz-5.10.0-8-2-arm64 root=$grub_cfg_root rw $GRUB_CMDLINE_LINUX fixrtc \ loop=$image_dir/$FILESYSTEM_SQUASHFS loopfstype=squashfs \ systemd.unified_cgroup_hierarchy=0 \ apparmor=1 security=apparmor varlog_size=4096 systemd.unified_cgroup_hierarchy=0 \ @@ -251,22 +246,93 @@ menuentry '$demo_grub_entry' { EOF # Copy the grub.cfg onto the boot-directory as specified in the grub-install -mkdir -p /mnt/grub && cp $grub_cfg /mnt/grub/grub.cfg +ex mkdir -p /mnt/grub +ex cp $grub_cfg /mnt/grub/grub.cfg sync +if [ "X${DEBUG}" == "Xyes" ]; then + log "GRUB CFG Updated" +fi + umount /mnt/boot/efi umount /mnt -umount /sys/firmware/efi/efivars + +ex blockdev --rereadpt ${device} + +ex fsck.vfat -a ${device}p1 +sync + +if [ -e /lib/firmware/mellanox/boot/capsule/update.cap ]; then + ex bfrec --capsule /lib/firmware/mellanox/boot/capsule/update.cap +fi + +ex bfbootmgr --cleanall + +# Make it the boot partition +mounted_efivarfs=0 +if [ ! -d /sys/firmware/efi/efivars ]; then + ex mount -t efivarfs none /sys/firmware/efi/efivars + mounted_efivarfs=1 +fi + +if efibootmgr | grep SONiC-OS; then + ex efibootmgr -b "$(efibootmgr | grep SONiC-OS | cut -c 5-8)" -B +fi +ex efibootmgr -c -d "$device" -p 1 -L SONiC-OS -l "\EFI\SONiC-OS\grubaa64.efi" + +if [ "X${DEBUG}" == "Xyes" ]; then + log "EFIBootMgr Updated" +fi BFCFG=`which bfcfg 2> /dev/null` if [ -n "$BFCFG" ]; then - $BFCFG + # Create PXE boot entries + if [ -e /etc/bf.cfg ]; then + mv /etc/bf.cfg /etc/bf.cfg.orig + fi + + cat > /etc/bf.cfg << EOF +BOOT0=DISK +BOOT1=NET-NIC_P0-IPV4 +BOOT2=NET-NIC_P0-IPV6 +BOOT3=NET-NIC_P1-IPV4 +BOOT4=NET-NIC_P1-IPV6 +BOOT5=NET-OOB-IPV4 +BOOT6=NET-OOB-IPV6 +PXE_DHCP_CLASS_ID=$DHCP_CLASS_ID +EOF + + ex $BFCFG + + # Restore the original bf.cfg + /bin/rm -f /etc/bf.cfg + if [ -e /etc/bf.cfg.orig ]; then + mv /etc/bf.cfg.orig /etc/bf.cfg + fi +fi + +if [ $mounted_efivarfs -eq 1 ]; then + umount /sys/firmware/efi/efivars > /dev/null 2>&1 +fi + +if [ -n "$BFCFG" ]; then + ex $BFCFG fi if function_exists bfb_post_install; then - bfb_post_install + ex bfb_post_install fi -log "INFO: Installation finished" -log "INFO: Rebooting..." +if [ "X${DEBUG}" == "Xyes" ]; then + # Copy the log file to Persistent Storage + mount -t ext4 ${device}p2 /mnt + mkdir -p /mnt/bfb-boot-logs/ + dmesg > /mnt/bfb-boot-logs/installer_bfb_dmesg.log + mv $log_file /mnt/bfb-boot-logs/$log_file + umount /mnt + log "Rebooting..." + log "Installation finished" + # Wait for these messages to be pulled by the rshim service + sleep 3 +fi diff --git a/platform/nvidia-bluefield/mlx-bootctl/.gitignore b/platform/nvidia-bluefield/mlx-bootctl/.gitignore new file mode 100644 index 000000000000..a0991ff4402b --- /dev/null +++ b/platform/nvidia-bluefield/mlx-bootctl/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!Makefile diff --git a/platform/nvidia-bluefield/mlx-bootctl/Makefile b/platform/nvidia-bluefield/mlx-bootctl/Makefile new file mode 100644 index 000000000000..0048b03c1e31 --- /dev/null +++ b/platform/nvidia-bluefield/mlx-bootctl/Makefile @@ -0,0 +1,37 @@ +# +# Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. +# Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -ex + +MLX_BOOTCTL = mlx-bootctl-$(MLX_BOOTCTL_DRIVER_VERSION) +MLX_BOOTCTL_TAR = ${MLX_BOOTCTL}.tar.gz + +$(addprefix $(DEST)/, $(MLX_BOOTCTL_DRIVER)): $(DEST)/% : + rm -rf ${MLX_BOOTCTL} ${MLX_BOOTCTL_TAR} *.buildinfo *.changes *.deb + wget -N "https://linux.mellanox.com/public/repo/bluefield/${BSD_VER}/extras/SOURCES/$(MLX_BOOTCTL_TAR)" + tar -z -f $(MLX_BOOTCTL_TAR) -x + + pushd ${MLX_BOOTCTL} + + # Build the package + KVER=$(KVERSION) WITH_DKMS=0 dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) + + mv $* $(DEST)/ + + popd diff --git a/platform/nvidia-bluefield/recipes/bootimages.dep b/platform/nvidia-bluefield/recipes/bootimages.dep index eda2c727d79c..69e4ee117286 100644 --- a/platform/nvidia-bluefield/recipes/bootimages.dep +++ b/platform/nvidia-bluefield/recipes/bootimages.dep @@ -2,5 +2,5 @@ DEP_FILES := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/$(RECIPE_DIR)/bootimage DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) $(BOOTIMAGES)_CACHE_MODE := GIT_CONTENT_SHA -$(BOOTIMAGES)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(BOOTIMAGES)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) $(BSD_VER) $(BSD_REV) $(BFB_IMG_TYPE) $(BOOTIMAGES)_DEP_FILES := $(DEP_FILES) diff --git a/platform/nvidia-bluefield/recipes/bootimages.mk b/platform/nvidia-bluefield/recipes/bootimages.mk index 14cb9fc5b79a..ac6b7bd3fe8e 100644 --- a/platform/nvidia-bluefield/recipes/bootimages.mk +++ b/platform/nvidia-bluefield/recipes/bootimages.mk @@ -14,19 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -BOOTIMAGES_VERSION = 3.8.0-11969 -BOOTIMAGES_BASE_PATH = $(PLATFORM_PATH)/sdk-src/sonic-bluefield-packages/bin/BlueField-$(BFINSTALL_VERSION) +IMAGE_TYPE=dev -BOOTIMAGES_BASE_URL = +BOOTIMAGES_BASE_URL = https://linux.mellanox.com/public/repo/bluefield/3.8.5/bootimages/$(IMAGE_TYPE) +BOOTIMAGES = mlxbf-bootimages_$(BSD_VER)-$(BSD_REV)_arm64.deb -BOOTIMAGES = mlxbf-bootimages_$(BFINSTALL_VERSION)_arm64.deb +# Required for building BFB image +UPDATE_CAP = update.cap -ifneq ($(BOOTIMAGES_BASE_URL), ) $(BOOTIMAGES)_URL = $(BOOTIMAGES_BASE_URL)/$(BOOTIMAGES) SONIC_ONLINE_DEBS += $(BOOTIMAGES) -else -$(BOOTIMAGES)_PATH = $(BOOTIMAGES_BASE_PATH) -SONIC_COPY_DEBS += $(BOOTIMAGES) -endif -export BOOTIMAGES \ No newline at end of file +$(UPDATE_CAP)_URL = $(BOOTIMAGES_BASE_URL)/$(UPDATE_CAP) +SONIC_ONLINE_FILES += $(UPDATE_CAP) + +export BOOTIMAGES UPDATE_CAP diff --git a/platform/nvidia-bluefield/recipes/installer-image.mk b/platform/nvidia-bluefield/recipes/installer-image.mk index 79a0175d3be2..69c00891110b 100644 --- a/platform/nvidia-bluefield/recipes/installer-image.mk +++ b/platform/nvidia-bluefield/recipes/installer-image.mk @@ -43,11 +43,6 @@ $(SONIC_BF2_IMAGE_BASE)_INSTALLS += $(ETHTOOL) \ $(SONIC_BF2_IMAGE_BASE)_DOCKERS = $(filter-out $(DOCKER_ORCHAGENT) $(DOCKER_PLATFORM_MONITOR),$(SONIC_INSTALL_DOCKER_IMAGES)) -# TODO: -# INSTALLS: $(MLNX_BLUEFIELD_BUILD_SCRIPTS) $(MLXBF_BOOTCTL_DEB) -# DEPENDS: $(MLNX_BLUEFIELD_BUILD_SCRIPTS) -# Add these to PXE and BFB image once they are working and ready - # A compressed archive which contains individual files required for PXE boot SONIC_BF2_IMAGE_PXE = $(SONIC_BF2_IMAGE_BASE).gz $(SONIC_BF2_IMAGE_PXE)_IMAGE_TYPE = pxe @@ -74,6 +69,6 @@ $(SONIC_BF2_IMAGE_BFB)_INSTALLS += $($(SONIC_BF2_IMAGE_BASE)_INSTALLS) $(SONIC_BF2_IMAGE_BFB)_DEPENDS += $($(SONIC_BF2_IMAGE_BASE)_DEPENDS) $(SONIC_BF2_IMAGE_BFB)_DOCKERS += $($(SONIC_BF2_IMAGE_BASE)_DOCKERS) $(SONIC_BF2_IMAGE_BFB)_LAZY_INSTALLS += $($(SONIC_BF2_IMAGE_BASE)_LAZY_INSTALLS) -$(SONIC_BF2_IMAGE_BFB)_FILES += $(BFINSTALL_FILE) +$(SONIC_BF2_IMAGE_BFB)_FILES += $(UPDATE_CAP) $(MLX_BOOTCTL_DRIVER) SONIC_INSTALLERS += $(SONIC_BF2_IMAGE_PXE) $(SONIC_BF2_IMAGE_BIN) $(SONIC_BF2_IMAGE_BFB) diff --git a/platform/nvidia-bluefield/recipes/mlx-bootctl.dep b/platform/nvidia-bluefield/recipes/mlx-bootctl.dep new file mode 100644 index 000000000000..8647f5c1c5f8 --- /dev/null +++ b/platform/nvidia-bluefield/recipes/mlx-bootctl.dep @@ -0,0 +1,9 @@ + +SPATH := $($(MLX_BOOTCTL_DRIVER)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.mk $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(MLX_BOOTCTL_DRIVER)_CACHE_MODE := GIT_CONTENT_SHA +$(MLX_BOOTCTL_DRIVER)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(MLX_BOOTCTL_DRIVER)_DEP_FILES := $(DEP_FILES) diff --git a/platform/nvidia-bluefield/recipes/mlx-bootctl.mk b/platform/nvidia-bluefield/recipes/mlx-bootctl.mk new file mode 100644 index 000000000000..89285553ddcb --- /dev/null +++ b/platform/nvidia-bluefield/recipes/mlx-bootctl.mk @@ -0,0 +1,26 @@ +# +# Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. +# Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MLX_BOOTCTL_DRIVER_VERSION = 1.5 +MLX_BOOTCTL_DRIVER = mlx-bootctl.ko + +$(MLX_BOOTCTL_DRIVER)_SRC_PATH = $(PLATFORM_PATH)/mlx-bootctl +$(MLX_BOOTCTL_DRIVER)_DEPENDS += $(LINUX_HEADERS) $(LINUX_HEADERS_COMMON) +SONIC_MAKE_FILES += $(MLX_BOOTCTL_DRIVER) + +export MLX_BOOTCTL_DRIVER_VERSION +export MLX_BOOTCTL_DRIVER diff --git a/platform/nvidia-bluefield/rules.mk b/platform/nvidia-bluefield/rules.mk index 83a358482c92..60e66a346ec9 100644 --- a/platform/nvidia-bluefield/rules.mk +++ b/platform/nvidia-bluefield/rules.mk @@ -18,8 +18,10 @@ RECIPE_DIR = recipes # Bluefied Software Distribution Version -BSD_VER=3.8.0 -export BSD_VER +BSD_VER=3.8.5 +BSD_REV=12027 +BFB_IMG_TYPE=prod +export BSD_VER BSD_REV BFB_IMG_TYPE include $(PLATFORM_PATH)/$(RECIPE_DIR)/bfinstall.mk @@ -30,6 +32,7 @@ include $(PLATFORM_PATH)/$(RECIPE_DIR)/i2c-mlxbf.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/bluefield-edac.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/ipmb-dev-int.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/ipmb-host.mk +include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-bootctl.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-trio.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlxbf-bootctl.mk include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlxbf-livefish.mk diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index 98e043c23f04..3b5e0f39a7d9 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -362,8 +362,15 @@ RUN apt-get update && apt-get install -y \ libgirepository1.0-dev \ libsystemd-dev \ pkg-config \ +# For building bfb image + fdisk \ + usbutils \ + dosfstools \ + efibootmgr \ + grub-efi-arm64-bin \ + grub2-common \ # For socat build - net-tools \ + net-tools \ # For audisp-tacplus libauparse-dev \ auditd