node-builder: add virtio-blk kata pod sandboxing and separate out tardev-snapshotter build

tools/osbuilder/node-builder/azure-linux/Makefile

@@ -89,3 +89,11 @@ deploy-confpods-package-tools:
 .PHONY: deploy-confpods-uvm
 deploy-confpods-uvm:
 	CONF_PODS=yes ./uvm_install.sh
+
+.PHONY: tardev
+tardev:
+	./package_tardev_build.sh
+
+.PHONY: deploy-tardev
+deploy-tardev:
+	./package_tardev_install.sh

tools/osbuilder/node-builder/azure-linux/common.sh

@@ -43,6 +43,8 @@ else
 	SHIM_CONFIG_INST_FILE_NAME="configuration.toml"
 	SHIM_DBG_CONFIG_FILE_NAME="configuration-clh-debug.toml"
 	SHIM_DBG_CONFIG_INST_FILE_NAME="${SHIM_DBG_CONFIG_FILE_NAME}"
+	SHIM_BLK_CONFIG_FILE_NAME="configuration-blk.toml"
+	SHIM_BLK_CONFIG_INST_FILE_NAME="${SHIM_BLK_CONFIG_FILE_NAME}"
 	DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin"
 	SHIM_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin"
 	SHIM_BINARY_NAME="containerd-shim-kata-v2"

tools/osbuilder/node-builder/azure-linux/package_build.sh

@@ -40,31 +40,10 @@ if [ "${OS_VERSION}" == "3.0" ]; then
 fi
 
 agent_make_flags="LIBC=gnu OPENSSL_NO_VENDOR=Y DESTDIR=${AGENT_INSTALL_DIR} BUILD_TYPE=${AGENT_BUILD_TYPE}"
-
-if [ "${CONF_PODS}" == "yes" ]; then
-	agent_make_flags+=" AGENT_POLICY=yes"
-fi
+agent_make_flags+=" AGENT_POLICY=yes"
 
 pushd "${repo_dir}"
 
-if [ "${CONF_PODS}" == "yes" ]; then
-
-	echo "Building utarfs binary"
-	pushd src/utarfs/
-	make all
-	popd
-
-	echo "Building kata-overlay binary"
-	pushd src/overlay/
-	make all
-	popd
-
-	echo "Building tardev-snapshotter service binary"
-	pushd src/tardev-snapshotter/
-	make all
-	popd
-fi
-
 echo "Building shim binary and configuration"
 pushd src/runtime/
 if [ "${CONF_PODS}" == "yes" ] || [ "${OS_VERSION}" == "3.0" ]; then

tools/osbuilder/node-builder/azure-linux/package_install.sh

@@ -14,7 +14,6 @@ CONF_PODS=${CONF_PODS:-no}
 PREFIX=${PREFIX:-}
 SHIM_REDEPLOY_CONFIG=${SHIM_REDEPLOY_CONFIG:-yes}
 SHIM_USE_DEBUG_CONFIG=${SHIM_USE_DEBUG_CONFIG:-no}
-START_SERVICES=${START_SERVICES:-yes}
 
 script_dir="$(dirname $(readlink -f $0))"
 repo_dir="${script_dir}/../../../../"
@@ -29,22 +28,6 @@ mkdir -p "${PREFIX}/${SHIM_CONFIG_PATH}"
 mkdir -p "${PREFIX}/${DEBUGGING_BINARIES_PATH}"
 mkdir -p "${PREFIX}/${SHIM_BINARIES_PATH}"
 
-if [ "${CONF_PODS}" == "yes" ]; then
-	echo "Installing tardev-snapshotter binaries and service file"
-	mkdir -p ${PREFIX}/usr/sbin
-	cp -a --backup=numbered src/utarfs/target/release/utarfs ${PREFIX}/usr/sbin/mount.tar
-	mkdir -p ${PREFIX}/usr/bin
-	cp -a --backup=numbered src/overlay/target/release/kata-overlay ${PREFIX}/usr/bin/
-	cp -a --backup=numbered src/tardev-snapshotter/target/release/tardev-snapshotter ${PREFIX}/usr/bin/
-	mkdir -p ${PREFIX}/usr/lib/systemd/system/
-	cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/
-
-	echo "Enabling and starting snapshotter service"
-	if [ "${START_SERVICES}" == "yes" ]; then
-		systemctl enable tardev-snapshotter && systemctl daemon-reload && systemctl restart tardev-snapshotter
-	fi
-fi
-
 echo "Installing diagnosability binaries (monitor, runtime, collect-data script)"
 cp -a --backup=numbered src/runtime/kata-monitor "${PREFIX}/${DEBUGGING_BINARIES_PATH}"
 cp -a --backup=numbered src/runtime/kata-runtime "${PREFIX}/${DEBUGGING_BINARIES_PATH}"
@@ -70,4 +53,9 @@ else
 	echo "Skipping installation of shim configuration"
 fi
 
+if [ "${CONF_PODS}" == "no" ]; then
+	echo "Installing virtio-blk based shim configuration"
+	cp -a --backup=numbered src/runtime/config/"${SHIM_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_BLK_CONFIG_INST_FILE_NAME}"
+	sed -i 's/shared_fs = "virtio-fs"/shared_fs = "none"/' "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_BLK_CONFIG_INST_FILE_NAME}"
+fi
 popd

tools/osbuilder/node-builder/azure-linux/package_tardev_build.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o pipefail
+set -o errtrace
+
+[ -n "$DEBUG" ] && set -x
+
+script_dir="$(dirname $(readlink -f $0))"
+repo_dir="${script_dir}/../../../../"
+
+common_file="common.sh"
+source "${common_file}"
+
+pushd "${repo_dir}"
+
+echo "Building utarfs binary"
+pushd src/utarfs/
+make all
+popd
+
+echo "Building kata-overlay binary"
+pushd src/overlay/
+make all
+popd
+
+echo "Building tardev-snapshotter service binary"
+pushd src/tardev-snapshotter/
+make all
+popd
+
+popd

tools/osbuilder/node-builder/azure-linux/package_tardev_install.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o pipefail
+set -o errtrace
+
+[ -n "$DEBUG" ] && set -x
+
+PREFIX=${PREFIX:-}
+START_SERVICES=${START_SERVICES:-yes}
+
+script_dir="$(dirname $(readlink -f $0))"
+repo_dir="${script_dir}/../../../../"
+
+common_file="common.sh"
+source "${common_file}"
+
+pushd "${repo_dir}"
+
+echo "Installing utarfs and kata-overlay binaries"
+mkdir -p ${PREFIX}/usr/sbin
+cp -a --backup=numbered src/utarfs/target/release/utarfs ${PREFIX}/usr/sbin/mount.tar
+mkdir -p ${PREFIX}/usr/bin
+cp -a --backup=numbered src/overlay/target/release/kata-overlay ${PREFIX}/usr/bin/
+mkdir -p ${PREFIX}/usr/lib/systemd/system/
+
+echo "Installing tardev-snapshotter binaries and service file"
+mkdir -p ${PREFIX}/usr/bin
+cp -a --backup=numbered src/tardev-snapshotter/target/release/tardev-snapshotter ${PREFIX}/usr/bin/
+mkdir -p ${PREFIX}/usr/lib/systemd/system/
+cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/
+
+if [ "${START_SERVICES}" == "yes" ]; then
+    echo "Enabling and starting snapshotter service"
+    systemctl enable tardev-snapshotter && systemctl daemon-reload && systemctl restart tardev-snapshotter
+fi
+
+popd

tools/osbuilder/node-builder/azure-linux/package_tools_install.sh

@@ -27,10 +27,10 @@ mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner"
 mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder"
 mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin"
 mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa"
 
 if [ "${CONF_PODS}" == "yes" ]; then
-	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa"
-	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs"
 	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux"
 fi
 
@@ -52,11 +52,13 @@ cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/u
 cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-containers.target "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/"
 cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-agent.service "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/"
 
+cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
+cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
+
+cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/"
+
 if [ "${CONF_PODS}" == "yes" ]; then
-	cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/"
 	cp -a --backup=numbered src/kata-opa/allow-set-policy.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/"
-	cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
-	cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
 	cp -a --backup=numbered tools/osbuilder/igvm-builder/igvm_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/"
 	cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/"
 	cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/"

tools/osbuilder/node-builder/azure-linux/uvm_build.sh

@@ -27,14 +27,15 @@ source "${common_file}"
 rootfs_make_flags="AGENT_SOURCE_BIN=${AGENT_INSTALL_DIR}/usr/bin/kata-agent OS_VERSION=${OS_VERSION}"
 
 if [ "${CONF_PODS}" == "yes" ]; then
-	rootfs_make_flags+=" AGENT_POLICY=yes CONF_GUEST=yes AGENT_POLICY_FILE=${agent_policy_file_abs}"
+	rootfs_make_flags+=" CONF_GUEST=yes AGENT_POLICY=yes AGENT_POLICY_FILE=${agent_policy_file_abs}"
+else
+	agent_policy_allow_all="${repo_dir}/src/kata-opa/allow-all.rego"
+	rootfs_make_flags+=" AGENT_POLICY=yes AGENT_POLICY_FILE=${agent_policy_file_allow_all}"
 fi
 
-if [ "${CONF_PODS}" == "yes" ]; then
-	set_uvm_kernel_vars
-	if [ -z "${UVM_KERNEL_HEADER_DIR}}" ]; then
-		exit 1
-	fi
+set_uvm_kernel_vars
+if [ -z "${UVM_KERNEL_HEADER_DIR}}" ]; then
+	exit 1
 fi
 
 pushd "${repo_dir}"
@@ -63,12 +64,13 @@ echo "Installing agent service files into rootfs"
 sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-containers.target ${ROOTFS_PATH}/usr/lib/systemd/system/kata-containers.target
 sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-agent.service ${ROOTFS_PATH}/usr/lib/systemd/system/kata-agent.service
 
+echo "Building tarfs kernel driver and installing into rootfs"
+pushd src/tarfs
+make KDIR=${UVM_KERNEL_HEADER_DIR}
+sudo make KDIR=${UVM_KERNEL_HEADER_DIR} KVER=${UVM_KERNEL_VERSION} INSTALL_MOD_PATH=${ROOTFS_PATH} install
+popd
+
 if [ "${CONF_PODS}" == "yes" ]; then
-	echo "Building tarfs kernel driver and installing into rootfs"
-	pushd src/tarfs
-	make KDIR=${UVM_KERNEL_HEADER_DIR}
-	sudo make KDIR=${UVM_KERNEL_HEADER_DIR} KVER=${UVM_KERNEL_VERSION} INSTALL_MOD_PATH=${ROOTFS_PATH} install
-	popd
 	echo "Building dm-verity protected image based on rootfs"
 	pushd tools/osbuilder
 	sudo -E PATH=${PATH} IMAGE_NAME=${LOCAL_IMAGE_NAME} make DISTRO=cbl-mariner MEASURED_ROOTFS=yes DM_VERITY_FORMAT=kernelinit image