From 851c863db2b810943cf39146232b686ed31ca179 Mon Sep 17 00:00:00 2001 From: Mitch Zhu Date: Wed, 19 Mar 2025 18:10:18 +0000 Subject: [PATCH] node-builder: add virtio-blk kata pod sandboxing and separate out tardev-snapshotter build This PR is to support new virtio-blk kata pod sandboxing runtime. It adds new runtime config, adjusts dependencies for kata pod sandboxing in general, and extract tardev-snapshotter build and install steps into separate scripts. Signed-off by: Mitch Zhu --- .../node-builder/azure-linux/Makefile | 8 ++++ .../node-builder/azure-linux/common.sh | 2 + .../node-builder/azure-linux/package_build.sh | 23 +--------- .../azure-linux/package_install.sh | 22 +++------- .../azure-linux/package_tardev_build.sh | 36 ++++++++++++++++ .../azure-linux/package_tardev_install.sh | 42 +++++++++++++++++++ .../azure-linux/package_tools_install.sh | 12 +++--- .../node-builder/azure-linux/uvm_build.sh | 24 ++++++----- 8 files changed, 114 insertions(+), 55 deletions(-) create mode 100755 tools/osbuilder/node-builder/azure-linux/package_tardev_build.sh create mode 100755 tools/osbuilder/node-builder/azure-linux/package_tardev_install.sh diff --git a/tools/osbuilder/node-builder/azure-linux/Makefile b/tools/osbuilder/node-builder/azure-linux/Makefile index 8f222677b5b2..9cbe53f58fb5 100644 --- a/tools/osbuilder/node-builder/azure-linux/Makefile +++ b/tools/osbuilder/node-builder/azure-linux/Makefile @@ -89,3 +89,11 @@ deploy-confpods-package-tools: .PHONY: deploy-confpods-uvm deploy-confpods-uvm: CONF_PODS=yes ./uvm_install.sh + +.PHONY: tardev +tardev: + ./package_tardev_build.sh + +.PHONY: deploy-tardev +deploy-tardev: + ./package_tardev_install.sh diff --git a/tools/osbuilder/node-builder/azure-linux/common.sh b/tools/osbuilder/node-builder/azure-linux/common.sh index 42104b0d8897..25cc0289cf7c 100755 --- a/tools/osbuilder/node-builder/azure-linux/common.sh +++ b/tools/osbuilder/node-builder/azure-linux/common.sh @@ -43,6 +43,8 @@ else SHIM_CONFIG_INST_FILE_NAME="configuration.toml" SHIM_DBG_CONFIG_FILE_NAME="configuration-clh-debug.toml" SHIM_DBG_CONFIG_INST_FILE_NAME="${SHIM_DBG_CONFIG_FILE_NAME}" + SHIM_BLK_CONFIG_FILE_NAME="configuration-blk.toml" + SHIM_BLK_CONFIG_INST_FILE_NAME="${SHIM_BLK_CONFIG_FILE_NAME}" DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin" SHIM_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin" SHIM_BINARY_NAME="containerd-shim-kata-v2" diff --git a/tools/osbuilder/node-builder/azure-linux/package_build.sh b/tools/osbuilder/node-builder/azure-linux/package_build.sh index 27ff96ddbb88..a4218d410d6e 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_build.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_build.sh @@ -40,31 +40,10 @@ if [ "${OS_VERSION}" == "3.0" ]; then fi agent_make_flags="LIBC=gnu OPENSSL_NO_VENDOR=Y DESTDIR=${AGENT_INSTALL_DIR} BUILD_TYPE=${AGENT_BUILD_TYPE}" - -if [ "${CONF_PODS}" == "yes" ]; then - agent_make_flags+=" AGENT_POLICY=yes" -fi +agent_make_flags+=" AGENT_POLICY=yes" pushd "${repo_dir}" -if [ "${CONF_PODS}" == "yes" ]; then - - echo "Building utarfs binary" - pushd src/utarfs/ - make all - popd - - echo "Building kata-overlay binary" - pushd src/overlay/ - make all - popd - - echo "Building tardev-snapshotter service binary" - pushd src/tardev-snapshotter/ - make all - popd -fi - echo "Building shim binary and configuration" pushd src/runtime/ if [ "${CONF_PODS}" == "yes" ] || [ "${OS_VERSION}" == "3.0" ]; then diff --git a/tools/osbuilder/node-builder/azure-linux/package_install.sh b/tools/osbuilder/node-builder/azure-linux/package_install.sh index 791cff5d92d2..ce99598cd133 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_install.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_install.sh @@ -14,7 +14,6 @@ CONF_PODS=${CONF_PODS:-no} PREFIX=${PREFIX:-} SHIM_REDEPLOY_CONFIG=${SHIM_REDEPLOY_CONFIG:-yes} SHIM_USE_DEBUG_CONFIG=${SHIM_USE_DEBUG_CONFIG:-no} -START_SERVICES=${START_SERVICES:-yes} script_dir="$(dirname $(readlink -f $0))" repo_dir="${script_dir}/../../../../" @@ -29,22 +28,6 @@ mkdir -p "${PREFIX}/${SHIM_CONFIG_PATH}" mkdir -p "${PREFIX}/${DEBUGGING_BINARIES_PATH}" mkdir -p "${PREFIX}/${SHIM_BINARIES_PATH}" -if [ "${CONF_PODS}" == "yes" ]; then - echo "Installing tardev-snapshotter binaries and service file" - mkdir -p ${PREFIX}/usr/sbin - cp -a --backup=numbered src/utarfs/target/release/utarfs ${PREFIX}/usr/sbin/mount.tar - mkdir -p ${PREFIX}/usr/bin - cp -a --backup=numbered src/overlay/target/release/kata-overlay ${PREFIX}/usr/bin/ - cp -a --backup=numbered src/tardev-snapshotter/target/release/tardev-snapshotter ${PREFIX}/usr/bin/ - mkdir -p ${PREFIX}/usr/lib/systemd/system/ - cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/ - - echo "Enabling and starting snapshotter service" - if [ "${START_SERVICES}" == "yes" ]; then - systemctl enable tardev-snapshotter && systemctl daemon-reload && systemctl restart tardev-snapshotter - fi -fi - echo "Installing diagnosability binaries (monitor, runtime, collect-data script)" cp -a --backup=numbered src/runtime/kata-monitor "${PREFIX}/${DEBUGGING_BINARIES_PATH}" cp -a --backup=numbered src/runtime/kata-runtime "${PREFIX}/${DEBUGGING_BINARIES_PATH}" @@ -70,4 +53,9 @@ else echo "Skipping installation of shim configuration" fi +if [ "${CONF_PODS}" == "no" ]; then + echo "Installing virtio-blk based shim configuration" + cp -a --backup=numbered src/runtime/config/"${SHIM_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_BLK_CONFIG_INST_FILE_NAME}" + sed -i 's/shared_fs = "virtio-fs"/shared_fs = "none"/' "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_BLK_CONFIG_INST_FILE_NAME}" +fi popd diff --git a/tools/osbuilder/node-builder/azure-linux/package_tardev_build.sh b/tools/osbuilder/node-builder/azure-linux/package_tardev_build.sh new file mode 100755 index 000000000000..5cf1523526b2 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/package_tardev_build.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +pushd "${repo_dir}" + +echo "Building utarfs binary" +pushd src/utarfs/ +make all +popd + +echo "Building kata-overlay binary" +pushd src/overlay/ +make all +popd + +echo "Building tardev-snapshotter service binary" +pushd src/tardev-snapshotter/ +make all +popd + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/package_tardev_install.sh b/tools/osbuilder/node-builder/azure-linux/package_tardev_install.sh new file mode 100755 index 000000000000..f418b7064f86 --- /dev/null +++ b/tools/osbuilder/node-builder/azure-linux/package_tardev_install.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o pipefail +set -o errtrace + +[ -n "$DEBUG" ] && set -x + +PREFIX=${PREFIX:-} +START_SERVICES=${START_SERVICES:-yes} + +script_dir="$(dirname $(readlink -f $0))" +repo_dir="${script_dir}/../../../../" + +common_file="common.sh" +source "${common_file}" + +pushd "${repo_dir}" + +echo "Installing utarfs and kata-overlay binaries" +mkdir -p ${PREFIX}/usr/sbin +cp -a --backup=numbered src/utarfs/target/release/utarfs ${PREFIX}/usr/sbin/mount.tar +mkdir -p ${PREFIX}/usr/bin +cp -a --backup=numbered src/overlay/target/release/kata-overlay ${PREFIX}/usr/bin/ +mkdir -p ${PREFIX}/usr/lib/systemd/system/ + +echo "Installing tardev-snapshotter binaries and service file" +mkdir -p ${PREFIX}/usr/bin +cp -a --backup=numbered src/tardev-snapshotter/target/release/tardev-snapshotter ${PREFIX}/usr/bin/ +mkdir -p ${PREFIX}/usr/lib/systemd/system/ +cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/ + +if [ "${START_SERVICES}" == "yes" ]; then + echo "Enabling and starting snapshotter service" + systemctl enable tardev-snapshotter && systemctl daemon-reload && systemctl restart tardev-snapshotter +fi + +popd diff --git a/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh index 8bf306bce1ac..404cddede965 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh @@ -27,10 +27,10 @@ mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner" mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder" mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin" mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs" +mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa" if [ "${CONF_PODS}" == "yes" ]; then - mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa" - mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs" mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux" fi @@ -52,11 +52,13 @@ cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/u cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-containers.target "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/" cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-agent.service "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/" +cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" +cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" + +cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/" + if [ "${CONF_PODS}" == "yes" ]; then - cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/" cp -a --backup=numbered src/kata-opa/allow-set-policy.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/" - cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" - cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/" cp -a --backup=numbered tools/osbuilder/igvm-builder/igvm_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/" cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/" cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/" diff --git a/tools/osbuilder/node-builder/azure-linux/uvm_build.sh b/tools/osbuilder/node-builder/azure-linux/uvm_build.sh index 23cab5d0f549..e1c43e2e1629 100755 --- a/tools/osbuilder/node-builder/azure-linux/uvm_build.sh +++ b/tools/osbuilder/node-builder/azure-linux/uvm_build.sh @@ -27,14 +27,15 @@ source "${common_file}" rootfs_make_flags="AGENT_SOURCE_BIN=${AGENT_INSTALL_DIR}/usr/bin/kata-agent OS_VERSION=${OS_VERSION}" if [ "${CONF_PODS}" == "yes" ]; then - rootfs_make_flags+=" AGENT_POLICY=yes CONF_GUEST=yes AGENT_POLICY_FILE=${agent_policy_file_abs}" + rootfs_make_flags+=" CONF_GUEST=yes AGENT_POLICY=yes AGENT_POLICY_FILE=${agent_policy_file_abs}" +else + agent_policy_allow_all="${repo_dir}/src/kata-opa/allow-all.rego" + rootfs_make_flags+=" AGENT_POLICY=yes AGENT_POLICY_FILE=${agent_policy_file_allow_all}" fi -if [ "${CONF_PODS}" == "yes" ]; then - set_uvm_kernel_vars - if [ -z "${UVM_KERNEL_HEADER_DIR}}" ]; then - exit 1 - fi +set_uvm_kernel_vars +if [ -z "${UVM_KERNEL_HEADER_DIR}}" ]; then + exit 1 fi pushd "${repo_dir}" @@ -63,12 +64,13 @@ echo "Installing agent service files into rootfs" sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-containers.target ${ROOTFS_PATH}/usr/lib/systemd/system/kata-containers.target sudo cp ${AGENT_INSTALL_DIR}/usr/lib/systemd/system/kata-agent.service ${ROOTFS_PATH}/usr/lib/systemd/system/kata-agent.service +echo "Building tarfs kernel driver and installing into rootfs" +pushd src/tarfs +make KDIR=${UVM_KERNEL_HEADER_DIR} +sudo make KDIR=${UVM_KERNEL_HEADER_DIR} KVER=${UVM_KERNEL_VERSION} INSTALL_MOD_PATH=${ROOTFS_PATH} install +popd + if [ "${CONF_PODS}" == "yes" ]; then - echo "Building tarfs kernel driver and installing into rootfs" - pushd src/tarfs - make KDIR=${UVM_KERNEL_HEADER_DIR} - sudo make KDIR=${UVM_KERNEL_HEADER_DIR} KVER=${UVM_KERNEL_VERSION} INSTALL_MOD_PATH=${ROOTFS_PATH} install - popd echo "Building dm-verity protected image based on rootfs" pushd tools/osbuilder sudo -E PATH=${PATH} IMAGE_NAME=${LOCAL_IMAGE_NAME} make DISTRO=cbl-mariner MEASURED_ROOTFS=yes DM_VERITY_FORMAT=kernelinit image