qemu:Use QMP to set bootindex dynamically

mantle/go.mod

@@ -21,6 +21,8 @@ require (
 	github.com/coreos/ioprogress v0.0.0-20151023204047-4637e494fd9b
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
 	github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 // indirect
+	github.com/digitalocean/go-libvirt v0.0.0-20200320195706-d56fdc7b97e1 // indirect
+	github.com/digitalocean/go-qemu v0.0.0-20181112162955-dd7bb9c771b8
 	github.com/digitalocean/godo v1.33.0
 	github.com/dimchansky/utfbom v1.1.0 // indirect
 	github.com/gedex/inflector v0.0.0-20170307190818-16278e9db813

mantle/go.sum

@@ -73,6 +73,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/digitalocean/go-libvirt v0.0.0-20200320195706-d56fdc7b97e1/go.mod h1:UMlaMc4V1DeGbb53Bw12wwvepjpg/D8xhrdL0wfS6Hs=
+github.com/digitalocean/go-qemu v0.0.0-20181112162955-dd7bb9c771b8 h1:N7nH2py78LcMqYY3rZjjrsX6N7uCN7sjvaosgpXN9Ow=
+github.com/digitalocean/go-qemu v0.0.0-20181112162955-dd7bb9c771b8/go.mod h1:/YnlngP1PARC0SKAZx6kaAEMOp8bNTQGqS+Ka3MctNI=
 github.com/digitalocean/godo v1.33.0 h1:JNZ/0v/Wp//UAIh84YWZ/x5neB3V5lKgcCHzyqErMJQ=
 github.com/digitalocean/godo v1.33.0/go.mod h1:iJnN9rVu6K5LioLxLimlq0uRI+y/eAQjROUmeU/r0hY=
 github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=

mantle/platform/metal.go

@@ -291,7 +291,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
 		pxe.boottype = "pxe"
 		pxe.networkdevice = "virtio-net-ccw"
 		pxe.tftpipaddr = "10.0.2.2"
-		pxe.bootindex = "1"
+		pxe.bootindex = "2"
 	default:
 		return nil, fmt.Errorf("Unsupported arch %s" + system.RpmArch())
 	}

mantle/platform/qemu.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
+	"net"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -35,6 +36,7 @@ import (
 	"github.com/coreos/mantle/system"
 	"github.com/coreos/mantle/system/exec"
 	"github.com/coreos/mantle/util"
+	"github.com/digitalocean/go-qemu/qmp"
 	"github.com/pkg/errors"
 )
 
@@ -64,14 +66,31 @@ type Disk struct {
 type QemuInstance struct {
 	qemu       exec.Cmd
 	tmpConfig  string
-	swtpmTmpd  string
+	tempdir    string
 	swtpm      exec.Cmd
 	tmpFiles   []string
 	nbdServers []exec.Cmd
 
+	qmpChan     *net.Conn
 	journalPipe *os.File
 }
 
+type QOMBlkDev struct {
+	Return []struct {
+		Device     string `json:"device"`
+		DevicePath string `json:"qdev"`
+	} `json:"return"`
+}
+
+type QOMSetBootindexCmd struct {
+	Execute   string `json:"execute"`
+	Arguments struct {
+		Path     string `json:"path"`
+		Property string `json:"property"`
+		Value    int    `json:"value"`
+	} `json:"arguments"`
+}
+
 func (inst *QemuInstance) Pid() int {
 	return inst.qemu.Pid()
 }
@@ -214,14 +233,13 @@ func (inst *QemuInstance) Destroy() {
 		}
 	}
 	inst.qemu = nil
-	if inst.swtpmTmpd != "" {
-		if inst.swtpm != nil {
-			inst.swtpm.Kill() // Ignore errors
-		}
+	if inst.swtpm != nil {
+		inst.swtpm.Kill() // Ignore errors
 		inst.swtpm = nil
-		// And ensure it's cleaned up
-		if err := os.RemoveAll(inst.swtpmTmpd); err != nil {
-			plog.Errorf("Error removing swtpm dir: %v", err)
+	}
+	if inst.tempdir != "" {
+		if err := os.RemoveAll(inst.tempdir); err != nil {
+			plog.Errorf("Error removing qemu tempdir: %v", err)
 		}
 	}
 	for _, nbdServ := range inst.nbdServers {
@@ -664,11 +682,6 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error {
 	if disk.SectorSize != 0 {
 		diskOpts = append(diskOpts, fmt.Sprintf("physical_block_size=%[1]d,logical_block_size=%[1]d", disk.SectorSize))
 	}
-	// Primary disk gets bootindex 1, all other disks have unspecified
-	// bootindex, which means lower priority.
-	if primary {
-		diskOpts = append(diskOpts, "bootindex=1")
-	}
 
 	opts := ""
 	if len(diskOpts) > 0 {
@@ -687,9 +700,6 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error {
 		wwn := rand.Uint64()
 
 		for i := 0; i < 2; i++ {
-			if i == 1 {
-				opts = strings.Replace(opts, "bootindex=1", "bootindex=2", -1)
-			}
 			pId := fmt.Sprintf("mpath%d%d", builder.diskId, i)
 			scsiId := fmt.Sprintf("scsi_%s", pId)
 			builder.Append("-device", fmt.Sprintf("virtio-scsi-pci,id=%s", scsiId))
@@ -734,7 +744,7 @@ func (builder *QemuBuilder) AddInstallIso(path string) error {
 	// boot. On reboot when the system is installed, the primary disk is
 	// selected. This allows us to have "boot once" functionality on both UEFI
 	// and BIOS (`-boot once=d` OTOH doesn't work with OVMF).
-	builder.Append("-drive", "file="+path+",format=raw,if=none,readonly=on,id=installiso", "-device", "ide-cd,drive=installiso,bootindex=2")
+	builder.Append("-drive", "file="+path+",format=raw,if=none,readonly=on,id=installiso", "-device", "ide-cd,drive=installiso")
 	return nil
 }
 
@@ -912,6 +922,11 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 	// We never want a popup window
 	argv = append(argv, "-nographic")
 
+	inst.tempdir, err = ioutil.TempDir("", "mantle-qemu-tmp")
+	if err != nil {
+		return nil, err
+	}
+
 	// Handle Ignition
 	if builder.ConfigFile != "" && !builder.configInjected {
 		if builder.supportsFwCfg() {
@@ -938,16 +953,11 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 
 	// Handle Software TPM
 	if builder.Swtpm && builder.supportsSwtpm() {
-		inst.swtpmTmpd, err = ioutil.TempDir("", "kola-swtpm")
-		if err != nil {
-			return nil, err
-		}
-
-		swtpmSock := filepath.Join(inst.swtpmTmpd, "swtpm-sock")
+		swtpmSock := filepath.Join(inst.tempdir, "swtpm-sock")
 
 		inst.swtpm = exec.Command("swtpm", "socket", "--tpm2",
 			"--ctrl", fmt.Sprintf("type=unixio,path=%s", swtpmSock),
-			"--terminate", "--tpmstate", fmt.Sprintf("dir=%s", inst.swtpmTmpd))
+			"--terminate", "--tpmstate", fmt.Sprintf("dir=%s", inst.tempdir))
 		cmd := inst.swtpm.(*exec.ExecCmd)
 		// For now silence the swtpm stderr as it prints errors when
 		// disconnected, but that's normal.
@@ -969,9 +979,13 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 		case "ppc64le":
 			argv = append(argv, "-device", "tpm-spapr,tpmdev=tpm0")
 		}
-
 	}
 
+	qmpPath := filepath.Join(inst.tempdir, "qmp.sock")
+	qmpId := "mantle-qmp"
+	builder.Append("-chardev", fmt.Sprintf("socket,id=%s,path=%s,server,nowait", qmpId, qmpPath))
+	builder.Append("-mon", fmt.Sprintf("chardev=%s,mode=control", qmpId))
+
 	// Set up the virtio channel to get Ignition failures by default
 	journalPipeR, err := builder.VirtioChannelRead("com.coreos.ignition.journal")
 	inst.journalPipe = journalPipeR
@@ -1013,6 +1027,62 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 		return nil, err
 	}
 
+	// Use QMP to connect to the qemu monitor and set the bootindex dynamically
+	// This is helpful for changing the bootindex for subsequent booting and helps
+	// in cases like s390x netboot where the netdevice has to have a higher boot order
+	// when starting up, but later should boot from disk. Here we query for a list of all
+	// the block devices and set the bootindex based on the id
+	var monitor *qmp.SocketMonitor
+	if err := util.Retry(10, 1*time.Second, func() error {
+		monitor, err = qmp.NewSocketMonitor("unix", qmpPath, 2*time.Second)
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return nil, errors.Wrapf(err, "Connecting to qemu monitor")
+	}
+
+	monitor.Connect()
+	defer monitor.Disconnect()
+	qmpcmd := []byte(`{ "execute": "query-block" }`)
+	raw, err := monitor.Run(qmpcmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Running QMP command")
+	}
+
+	var blkdevs QOMBlkDev
+	if err = json.Unmarshal(raw, &blkdevs); err != nil {
+		return nil, errors.Wrapf(err, "De-serializing QMP output")
+	}
+
+	qomsetcmd := QOMSetBootindexCmd{}
+	qomsetcmd.Execute = "qom-set"
+	qomsetcmd.Arguments.Property = "bootindex"
+	for _, blkdev := range blkdevs.Return {
+		qomsetcmd.Arguments.Path = filepath.Clean(strings.Trim(blkdev.DevicePath, "virtio-backend"))
+		switch blkdev.Device {
+		case "d1", "mpath10":
+			qomsetcmd.Arguments.Value = 1
+			break
+		case "installiso", "mpath11":
+			qomsetcmd.Arguments.Value = 2
+			break
+		default:
+			break
+		}
+		if qomsetcmd.Arguments.Value != 0 {
+			rawcmd, err := json.Marshal(qomsetcmd)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Serializing QMP output")
+			}
+
+			if _, err = monitor.Run(rawcmd); err != nil {
+				return nil, errors.Wrapf(err, "Running QMP command")
+			}
+		}
+		qomsetcmd.Arguments.Value = 0
+	}
 	return &inst, nil
 }