Add the ability to output vmdk via qemu-img

Dockerfile

@@ -152,14 +152,16 @@ ARG VERSION_ID
 ARG BUILD_ID
 ARG NOCACHE
 ARG VARIANT
-ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID}
+ARG IMAGE_FORMAT
+ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID} IMAGE_FORMAT=${IMAGE_FORMAT}
 WORKDIR /root
 
 USER root
 RUN --mount=target=/host \
     /host/tools/rpm2img \
       --package-dir=/local/rpms \
       --output-dir=/local/output \
+      --output-fmt=${IMAGE_FORMAT} \
     && echo ${NOCACHE}
 
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=

Makefile.toml

@@ -383,18 +383,11 @@ dependencies = ["build-variant"]
 script = [
 '''
 mkdir -p ${BUILDSYS_OUTPUT_DIR}/latest
-ln -snf ../${BUILDSYS_NAME_FULL}.img.lz4 \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}.img.lz4
-ln -snf ../${BUILDSYS_NAME_FULL}-data.img.lz4 \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-data.img.lz4
-ln -snf ../${BUILDSYS_NAME_FULL}-boot.ext4.lz4 \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-boot.ext4.lz4
-ln -snf ../${BUILDSYS_NAME_FULL}-root.ext4.lz4 \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-root.ext4.lz4
-ln -snf ../${BUILDSYS_NAME_FULL}-root.verity.lz4 \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-root.verity.lz4
-ln -snf ../${BUILDSYS_NAME_FULL}-migrations.tar \
-  ${BUILDSYS_OUTPUT_DIR}/latest/${BUILDSYS_NAME_VARIANT}-migrations.tar
+for artifact in ${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}*; do
+  file_name="${artifact##*/}"
+  link_name="${file_name/${BUILDSYS_NAME_FULL}/${BUILDSYS_NAME_VARIANT}}"
+  ln -snf "../${file_name}" "${BUILDSYS_OUTPUT_DIR}/latest/${link_name}"
+done
 '''
 ]
 

tools/buildsys/src/builder.rs

@@ -19,6 +19,8 @@ use std::path::{Path, PathBuf};
 use std::process::Output;
 use walkdir::{DirEntry, WalkDir};
 
+use crate::manifest::ImageFormat;
+
 /*
 There's a bug in BuildKit that can lead to a build failure during parallel
 `docker build` executions:
@@ -46,7 +48,7 @@ impl PackageBuilder {
         let arch = getenv("BUILDSYS_ARCH")?;
         let output_dir: PathBuf = getenv("BUILDSYS_PACKAGES_DIR")?.into();
 
-        // We do *not* want to rebuild most packages when the variant changes, becauses most aren't
+        // We do *not* want to rebuild most packages when the variant changes, because most aren't
         // affected; packages that care about variant should "echo cargo:rerun-if-env-changed=VAR"
         // themselves in the package's spec file.
         let var = "BUILDSYS_VARIANT";
@@ -82,7 +84,7 @@ pub(crate) struct VariantBuilder;
 
 impl VariantBuilder {
     /// Build a variant with the specified packages installed.
-    pub(crate) fn build(packages: &[String]) -> Result<Self> {
+    pub(crate) fn build(packages: &[String], image_format: Option<&ImageFormat>) -> Result<Self> {
         // We want PACKAGES to be a value that contains spaces, since that's
         // easier to work with in the shell than other forms of structured data.
         let packages = packages.join("|");
@@ -91,6 +93,10 @@ impl VariantBuilder {
         let version_image = getenv("BUILDSYS_VERSION_IMAGE")?;
         let version_build = getenv("BUILDSYS_VERSION_BUILD")?;
         let output_dir: PathBuf = getenv("BUILDSYS_OUTPUT_DIR")?.into();
+        let image_format = match image_format {
+            Some(ImageFormat::Raw) | None => String::from("raw"),
+            Some(ImageFormat::Vmdk) => String::from("vmdk"),
+        };
 
         // Always rebuild variants since they are located in a different workspace,
         // and don't directly track changes in the underlying packages.
@@ -101,12 +107,14 @@ impl VariantBuilder {
              --build-arg ARCH={arch} \
              --build-arg VARIANT={variant} \
              --build-arg VERSION_ID={version_image} \
-             --build-arg BUILD_ID={version_build}",
+             --build-arg BUILD_ID={version_build} \
+             --build-arg IMAGE_FORMAT={image_format}",
             packages = packages,
             arch = arch,
             variant = variant,
             version_image = version_image,
             version_build = version_build,
+            image_format = image_format,
         );
         let tag = format!(
             "buildsys-var-{variant}-{arch}",

tools/buildsys/src/main.rs

@@ -166,7 +166,8 @@ fn build_variant() -> Result<()> {
         ManifestInfo::new(manifest_dir.join(manifest_file)).context(error::ManifestParse)?;
 
     if let Some(packages) = manifest.included_packages() {
-        VariantBuilder::build(&packages).context(error::BuildAttempt)?;
+        let image_format = manifest.image_format();
+        VariantBuilder::build(&packages, image_format).context(error::BuildAttempt)?;
     } else {
         println!("cargo:warning=No included packages in manifest. Skipping variant build.");
     }

tools/buildsys/src/manifest.rs

@@ -110,6 +110,11 @@ impl ManifestInfo {
             .and_then(|b| b.included_packages.as_ref())
     }
 
+    /// Convenience method to return the image format override, if any.
+    pub(crate) fn image_format(&self) -> Option<&ImageFormat> {
+        self.build_variant().and_then(|b| b.image_format.as_ref())
+    }
+
     /// Helper methods to navigate the series of optional struct fields.
     fn build_package(&self) -> Option<&BuildPackage> {
         self.package
@@ -152,6 +157,14 @@ pub(crate) struct BuildPackage {
 #[serde(rename_all = "kebab-case")]
 pub(crate) struct BuildVariant {
     pub(crate) included_packages: Option<Vec<String>>,
+    pub(crate) image_format: Option<ImageFormat>,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "lowercase")]
+pub(crate) enum ImageFormat {
+    Raw,
+    Vmdk,
 }
 
 #[derive(Deserialize, Debug)]

tools/rpm2img

@@ -3,22 +3,35 @@
 set -eu -o pipefail
 shopt -qs failglob
 
+OUTPUT_FMT="raw"
+
 for opt in "$@"; do
    optarg="$(expr "${opt}" : '[^=]*=\(.*\)')"
    case "${opt}" in
       --package-dir=*) PACKAGE_DIR="${optarg}" ;;
       --output-dir=*) OUTPUT_DIR="${optarg}" ;;
+      --output-fmt=*) OUTPUT_FMT="${optarg}" ;;
    esac
 done
 
+case "${OUTPUT_FMT}" in
+   raw|vmdk) ;;
+   *)
+      echo "unexpected image output format '${OUTPUT_FMT}'" >&2
+      exit 1
+      ;;
+esac
+
 mkdir -p "${OUTPUT_DIR}"
 
 FILENAME_PREFIX="bottlerocket-${VARIANT}-${ARCH}-${VERSION_ID}-${BUILD_ID}"
-DISK_IMAGE_NAME="${FILENAME_PREFIX}.img.lz4"
+
+DISK_IMAGE_BASENAME="${FILENAME_PREFIX}"
+DATA_IMAGE_BASENAME="${FILENAME_PREFIX}-data"
+
 BOOT_IMAGE_NAME="${FILENAME_PREFIX}-boot.ext4.lz4"
 VERITY_IMAGE_NAME="${FILENAME_PREFIX}-root.verity.lz4"
 ROOT_IMAGE_NAME="${FILENAME_PREFIX}-root.ext4.lz4"
-DATA_IMAGE_NAME="${FILENAME_PREFIX}-data.img.lz4"
 
 DISK_IMAGE="$(mktemp)"
 BOOT_IMAGE="$(mktemp)"
@@ -90,43 +103,43 @@ rm -rf "${ROOT_MOUNT}"/var/lib "${ROOT_MOUNT}"/usr/share/licenses/*
 if [[ "${ARCH}" == "x86_64" ]]; then
   SYS_ROOT="x86_64-bottlerocket-linux-gnu/sys-root"
   # MBR and BIOS-BOOT
-  echo "(hd0) ${DISK_IMAGE}" > ${ROOT_MOUNT}/boot/grub/device.map
+  echo "(hd0) ${DISK_IMAGE}" > "${ROOT_MOUNT}/boot/grub/device.map"
   "${ROOT_MOUNT}/sbin/grub-bios-setup" \
      --directory="${ROOT_MOUNT}/boot/grub" \
      --device-map="${ROOT_MOUNT}/boot/grub/device.map" \
      --root="hd0" \
      --skip-fs-probe \
-     ${DISK_IMAGE}
+     "${DISK_IMAGE}"
 
   rm -vf "${ROOT_MOUNT}"/boot/grub/* "${ROOT_MOUNT}"/sbin/grub*
 else
   SYS_ROOT="aarch64-bottlerocket-linux-gnu/sys-root"
   # For aarch64 we need an EFI partition instead, formatted
   # FAT32 with the .efi binary at the correct path, eg /efi/boot.
   # grub-mkimage has put bootaa64.efi at /boot/efi/EFI/BOOT
-  mv ${ROOT_MOUNT}/boot/efi/* "${EFI_MOUNT}"
+  mv "${ROOT_MOUNT}/boot/efi"/* "${EFI_MOUNT}"
 
   # The 'recommended' size for the EFI partition is 100MB but our aarch64.efi
   # only takes up around 700KB, so this will suffice for now.
-  dd if=/dev/zero of=${EFI_IMAGE} bs=1M count=4
+  dd if=/dev/zero of="${EFI_IMAGE}" bs=1M count=4
   mkfs.vfat -I -S 512 "${EFI_IMAGE}" $((4*2048))
-  mmd -i ${EFI_IMAGE} ::/EFI
-  mmd -i ${EFI_IMAGE} ::/EFI/BOOT
-  mcopy -i ${EFI_IMAGE} ${EFI_MOUNT}/EFI/BOOT/bootaa64.efi ::/EFI/BOOT
+  mmd -i "${EFI_IMAGE}" ::/EFI
+  mmd -i "${EFI_IMAGE}" ::/EFI/BOOT
+  mcopy -i "${EFI_IMAGE}" "${EFI_MOUNT}/EFI/BOOT/bootaa64.efi" ::/EFI/BOOT
   dd if="${EFI_IMAGE}" of="${DISK_IMAGE}" conv=notrunc bs=1M seek=1
 
   # Create the grub directory which grub-bios-setup would have otherwise done.
-  mkdir -p ${ROOT_MOUNT}/boot/grub
+  mkdir -p "${ROOT_MOUNT}/boot/grub"
 fi
 
 # Now that we're done messing with /, move /boot out of it
 mv "${ROOT_MOUNT}/boot"/* "${BOOT_MOUNT}"
 
 # Set the Bottlerocket variant, version, and build-id
-echo "PRETTY_NAME=\"Bottlerocket OS ${VERSION_ID}\"" >> ${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release
-echo "VARIANT_ID=${VARIANT}" >> ${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release
-echo "VERSION_ID=${VERSION_ID}" >> ${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release
-echo "BUILD_ID=${BUILD_ID}" >> ${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release
+echo "PRETTY_NAME=\"Bottlerocket OS ${VERSION_ID}\"" >> "${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release"
+echo "VARIANT_ID=${VARIANT}" >> "${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release"
+echo "VERSION_ID=${VERSION_ID}" >> "${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release"
+echo "BUILD_ID=${BUILD_ID}" >> "${ROOT_MOUNT}/${SYS_ROOT}/usr/lib/os-release"
 
 # BOTTLEROCKET-ROOT-A
 mkdir -p "${ROOT_MOUNT}/lost+found"
@@ -159,7 +172,7 @@ veritysetup verify "${ROOT_IMAGE}" "${VERITY_IMAGE}" "${VERITY_ROOT_HASH}"
 dd if="${VERITY_IMAGE}" of="${DISK_IMAGE}" conv=notrunc bs=1M seek=965
 
 # write GRUB config
-cat <<EOF > ${BOOT_MOUNT}/grub/grub.cfg
+cat <<EOF > "${BOOT_MOUNT}/grub/grub.cfg"
 set default="0"
 set timeout="0"
 
@@ -206,14 +219,23 @@ dd if="${BOTTLEROCKET_DATA}" of="${DATA_IMAGE}" conv=notrunc bs=1M seek=1
 sgdisk -v "${DISK_IMAGE}"
 sgdisk -v "${DATA_IMAGE}"
 
-lz4 -vc "${DISK_IMAGE}" >"${OUTPUT_DIR}/${DISK_IMAGE_NAME}"
-lz4 -vc "${DATA_IMAGE}" >"${OUTPUT_DIR}/${DATA_IMAGE_NAME}"
+if [[ ${OUTPUT_FMT} == "raw" ]]; then
+  lz4 -vc "${DISK_IMAGE}" >"${OUTPUT_DIR}/${DISK_IMAGE_BASENAME}.img.lz4"
+  lz4 -vc "${DATA_IMAGE}" >"${OUTPUT_DIR}/${DATA_IMAGE_BASENAME}.img.lz4"
+  chown 1000:1000 "${OUTPUT_DIR}/${DISK_IMAGE_BASENAME}.img.lz4" \
+    "${OUTPUT_DIR}/${DATA_IMAGE_BASENAME}.img.lz4"
+elif [[ ${OUTPUT_FMT} == "vmdk" ]]; then
+  # Stream optimization is required for creating an Open Virtual Appliance (OVA)
+  qemu-img convert -f raw -O vmdk -o subformat=streamOptimized "${DISK_IMAGE}" "${OUTPUT_DIR}/${DISK_IMAGE_BASENAME}.vmdk"
+  qemu-img convert -f raw -O vmdk -o subformat=streamOptimized "${DATA_IMAGE}" "${OUTPUT_DIR}/${DATA_IMAGE_BASENAME}.vmdk"
+  chown 1000:1000 "${OUTPUT_DIR}/${DISK_IMAGE_BASENAME}.vmdk" \
+    "${OUTPUT_DIR}/${DATA_IMAGE_BASENAME}.vmdk"
+fi
+
 lz4 -9vc "${BOOT_IMAGE}" >"${OUTPUT_DIR}/${BOOT_IMAGE_NAME}"
 lz4 -9vc "${VERITY_IMAGE}" >"${OUTPUT_DIR}/${VERITY_IMAGE_NAME}"
 lz4 -9vc "${ROOT_IMAGE}" >"${OUTPUT_DIR}/${ROOT_IMAGE_NAME}"
-chown 1000:1000 "${OUTPUT_DIR}/${DISK_IMAGE_NAME}" \
-    "${OUTPUT_DIR}/${DATA_IMAGE_NAME}" \
-    "${OUTPUT_DIR}/${BOOT_IMAGE_NAME}" \
+chown 1000:1000 "${OUTPUT_DIR}/${BOOT_IMAGE_NAME}" \
     "${OUTPUT_DIR}/${VERITY_IMAGE_NAME}" \
     "${OUTPUT_DIR}/${ROOT_IMAGE_NAME}"