From 541908a2c65e597109ee61c36bb756eaa892499f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 13 Apr 2020 16:41:59 +0000 Subject: [PATCH 1/2] build: Identify target disk by wwn This is a followup to: https://github.com/coreos/coreos-assembler/pull/1342 which is part of: https://github.com/coreos/coreos-assembler/pull/1289 Our builds right now run qemu in a way that's highly sensitive to changes in device ordering. SCSI hardware supports a [world wide name](https://en.wikipedia.org/wiki/World_Wide_Name); change our qemu invocation to use `42` (in decimal) for that. Unfortunately because we're not using udev in our supermin VM right now we don't get the nice `/dev/disk/by-id` symlink to it. Instead just walk `/sys/block` manually. --- src/cmd-buildextend-metal | 12 ++++-------- src/create_disk.sh | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 162bdf81c8..f229faebbb 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -234,20 +234,16 @@ if [ -n "${ref_is_temp}" ]; then ref_arg=${commit} fi -target_drive=("-drive" "if=virtio,id=target,format=${image_format},file=${path}.tmp,cache=unsafe") +target_device_opt="scsi-disk,drive=target,wwn=0x2a" # we need 4096 block size for ECKD DASD and (obviously) metal4k if [[ $image_type == dasd || $image_type == metal4k ]]; then - device_type=virtio-blk - if [[ $image_type == dasd ]]; then - device_type=virtio-blk-ccw - fi - target_drive=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ - "-device" "${device_type},drive=target,physical_block_size=4096,logical_block_size=4096,scsi=off") + target_device_opt="${target_device_opt},physical_block_size=4096,logical_block_size=4096" fi +target_drive=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ + "-device" "virtio-scsi-${devtype},id=scsitarget" "-device" "${target_device_opt}") runvm "${target_drive[@]}" -- \ /usr/lib/coreos-assembler/create_disk.sh \ - --disk /dev/vda \ --buildid "${build}" \ --imgid "${img}" \ --grub-script /usr/lib/coreos-assembler/grub.cfg \ diff --git a/src/create_disk.sh b/src/create_disk.sh index 638d386975..66d6a52db4 100755 --- a/src/create_disk.sh +++ b/src/create_disk.sh @@ -42,6 +42,7 @@ run as part of 'coreos-assembler build'. EOC } +disk= rootfs_size="0" boot_verity=0 rootfs_type="xfs" @@ -78,7 +79,25 @@ udevtrig() { export PATH=$PATH:/sbin:/usr/sbin arch="$(uname -m)" -disk="${disk:?--disk must be defined}" + +if [ -z "${disk:-}" ]; then + # hex 0x2a = 42 in decimal; this is set in cmd-buildextend-metal. + # We use the WWN as an unambiguous way to identify our target disk, + # independent of other devices attached to the VM (caches, etc.) + wwn=000000000000002a + for dev in /sys/block/*; do + if grep -F -e "${wwn}" "${dev}/device/wwid" 2>/dev/null; then + disk="/dev/$(basename ${dev})" + break + fi + done + if [ -z "${disk:-}" ]; then + echo "failed to find disk with wwn ${wwn}" 1>&2 + bash + exit 1 + fi +fi + buildid="${buildid:?--buildid must be defined}" imgid="${imgid:?--imgid must be defined}" ostree="${ostree:?--ostree-repo must be defined}" From 6a78bdef6167e7012a7606d44e4f07baa3764c87 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 25 Mar 2020 19:48:23 +0000 Subject: [PATCH 2/2] cmdlib: Use kola qemuexec And this final commit gets rid of almost all the core qemu logic in cmdlib, so e.g. architecture specific changes only need to happen in one place. The only special casing left is `$devtype` and the terminal. We inherit the virtio-rand setup, CPU calculations etc. And now we can just use virtio for the root disk and the cache disk, which drops all the eyeball-glazing scsi incantations. (Well, of course arch changes also end up in libvirt which we use indirectly too, but at least *we* only have one place) --- src/cmdlib.sh | 58 ++++++--------------------------------------------- 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index eb06be5103..e741bdec35 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -45,34 +45,7 @@ case $arch in "s390x") DEFAULT_TERMINAL="ttysclp0"; devtype=ccw ;; *) fatal "Architecture $(arch) not supported" esac -export DEFAULT_TERMINAL - -if [ -x /usr/libexec/qemu-kvm ]; then - QEMU_KVM="/usr/libexec/qemu-kvm" -else - # Enable arch-specific options for qemu - case "$(arch)" in - "x86_64") QEMU_KVM="qemu-system-$(arch) -accel kvm" ;; - "aarch64") QEMU_KVM="qemu-system-$(arch) -accel kvm -M virt,gic-version=host" ;; - "ppc64le") QEMU_KVM="qemu-system-ppc64 -accel kvm" ;; - "s390x") QEMU_KVM="qemu-system-$(arch) -accel kvm -M s390-ccw-virtio" ;; - *) fatal "Architecture $(arch) not supported" - esac -fi - -if grep -q 'kubepods' /proc/1/cgroup; then -# only use 1 core on kubernetes since we can't determine how much we can actually use - QEMU_PROCS=1 -elif [ "$(nproc)" -gt 16 ]; then -# cap qemu smp at some reasonable level to not exceed limitation of some platforms - QEMU_PROCS=16 -else - QEMU_PROCS="$(nproc)" -fi -# cap smp and dissable VGA as it "eats" output from console on some platforms, ppc64le most notably -QEMU_KVM+=" -smp ${QEMU_PROCS} -vga none" -export QEMU_KVM -export QEMU_PROCS +export DEFAULT_TERMINAL devtype _privileged= has_privileges() { @@ -489,8 +462,7 @@ EOF cachedisk=() if [ -f "${workdir}/cache/cache2.qcow2" ]; then - cachedisk=("-drive" "if=none,id=drive-scsi0-0-0-1,discard=unmap,file=${workdir}/cache/cache2.qcow2" \ - "-device" "scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1") + cachedisk=("-drive" "if=virtio,discard=unmap,file=${workdir}/cache/cache2.qcow2,index=2") fi # support local dev cases where src/config is a symlink @@ -500,33 +472,15 @@ EOF srcvirtfs=("-virtfs" "local,id=source,path=${workdir}/src/config,security_model=none,mount_tag=source") fi - scsibus="bus=pci.0,addr=0x3" - arch_args= - case $arch in - "aarch64") - # 'pci' bus doesn't work on aarch64 - scsibus="bus=pcie.0,addr=0x3" - arch_args='-bios /usr/share/AAVMF/AAVMF_CODE.fd' - ;; - "s390x") scsibus="devno=fe.0.0003" ;; - esac - rm -f "${workdir}/tmp/rc" #shellcheck disable=SC2086 - ${QEMU_KVM} ${arch_args:-} \ - -nodefaults -nographic -m 2048 -no-reboot -cpu host \ - -kernel "${vmbuilddir}/kernel" \ - -initrd "${vmbuilddir}/initrd" \ - -netdev user,id=eth0,hostname=supermin \ - -device virtio-net-"${devtype}",netdev=eth0 \ - -device virtio-scsi-"${devtype}",id=scsi0,"${scsibus}" \ - -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-"${devtype}",rng=rng0 \ - -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${vmbuilddir}/root" \ - -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \ + kola qemuexec -m 2048 --auto-cpus -U -- -no-reboot -nodefaults -serial stdio \ + -kernel "${vmbuilddir}/kernel" -initrd "${vmbuilddir}/initrd" \ + -drive "if=virtio,format=raw,snapshot=on,file=${vmbuilddir}/root,index=1" \ "${cachedisk[@]}" \ -virtfs local,id=workdir,path="${workdir}",security_model=none,mount_tag=workdir \ - "${srcvirtfs[@]}" -serial stdio -append "root=/dev/sda console=${DEFAULT_TERMINAL} selinux=1 enforcing=0 autorelabel=1" \ + "${srcvirtfs[@]}" -append "root=/dev/vda console=${DEFAULT_TERMINAL} selinux=1 enforcing=0 autorelabel=1" \ "${qemu_args[@]}" if [ ! -f "${workdir}"/tmp/rc ]; then