Skip to content

mantle: Rework testiso/metal to use new qemu channel API #1307

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 1 commit into from
Apr 2, 2020
Merged

mantle: Rework testiso/metal to use new qemu channel API #1307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 87 additions & 58 deletions mantle/cmd/kola/testiso.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,17 @@
package main

import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"math"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/coreos/mantle/system"
"github.com/coreos/mantle/util"
"github.com/pkg/errors"

Expand Down Expand Up @@ -60,16 +64,18 @@ var (
console bool
)

var signalCompletionUnit = `[Unit]
Requires=dev-virtio\\x2dports-completion.device
var signalCompleteString = "coreos-installer-test-OK"
var signalCompletionUnit = fmt.Sprintf(`[Unit]
Requires=dev-virtio\\x2dports-testisocompletion.device
OnFailure=emergency.target
OnFailureJobMode=isolate
[Service]
Type=oneshot
ExecStart=/bin/sh -c '/usr/bin/echo coreos-installer-test-OK >/dev/virtio-ports/completion && systemctl poweroff'
RemainAfterExit=yes
ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff'
[Install]
RequiredBy=multi-user.target
`
`, signalCompleteString)

func init() {
cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image")
Expand All @@ -82,17 +88,40 @@ func init() {
root.AddCommand(cmdTestIso)
}

func newQemuBuilder() *platform.QemuBuilder {
builder := platform.NewMetalQemuBuilderDefault()
builder.Firmware = kola.QEMUOptions.Firmware
sectorSize := 0
if kola.QEMUOptions.Native4k {
sectorSize = 4096
}
builder.AddPrimaryDisk(&platform.Disk{
Size: "12G", // Arbitrary

SectorSize: sectorSize,
})

// https://github.com/coreos/fedora-coreos-tracker/issues/388
// https://github.com/coreos/fedora-coreos-docs/pull/46
builder.Memory = 4096
if system.RpmArch() == "s390x" {
// FIXME - determine why this is
builder.Memory = int(math.Max(float64(builder.Memory), 16384))
}

builder.InheritConsole = console

return builder
}

func runTestIso(cmd *cobra.Command, args []string) error {
if kola.CosaBuild == nil {
return fmt.Errorf("Must provide --cosa-build")
}

baseInst := platform.Install{
CosaBuild: kola.CosaBuild,

Console: console,
Firmware: kola.QEMUOptions.Firmware,
Native4k: kola.QEMUOptions.Native4k,
Native4k: kola.QEMUOptions.Native4k,
}

if instInsecure {
Expand All @@ -107,10 +136,6 @@ func runTestIso(cmd *cobra.Command, args []string) error {

completionfile := filepath.Join(tmpd, "completion.txt")

baseInst.QemuArgs = []string{
"-device", "virtio-serial", "-device", "virtserialport,chardev=completion,name=completion",
"-chardev", "file,id=completion,path=" + completionfile}

if kola.CosaBuild.Meta.BuildArtifacts.Metal == nil {
return fmt.Errorf("Build %s must have a `metal` artifact", kola.CosaBuild.Meta.OstreeVersion)
}
Expand All @@ -124,7 +149,7 @@ func runTestIso(cmd *cobra.Command, args []string) error {
inst := baseInst // Pretend this is Rust and I wrote .copy()
inst.LegacyInstaller = true

if err := testPXE(inst, completionfile); err != nil {
if err := testPXE(inst); err != nil {
return err
}
fmt.Printf("Successfully tested legacy installer for %s\n", kola.CosaBuild.Meta.OstreeVersion)
Expand All @@ -142,7 +167,7 @@ func runTestIso(cmd *cobra.Command, args []string) error {
ranTest = true
instPxe := baseInst // Pretend this is Rust and I wrote .copy()

if err := testPXE(instPxe, completionfile); err != nil {
if err := testPXE(instPxe); err != nil {
return err
}
printSuccess("PXE")
Expand All @@ -165,6 +190,35 @@ func runTestIso(cmd *cobra.Command, args []string) error {
return nil
}

func awaitCompletion(inst *platform.QemuInstance, qchan *os.File, expected []string) error {
errchan := make(chan error)
go func() {
if err := inst.Wait(); err != nil {
errchan <- err
}
time.Sleep(1 * time.Minute)
errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion")
}()
go func() {
r := bufio.NewReader(qchan)
for _, exp := range expected {
l, err := r.ReadString('\n')
if err != nil {
errchan <- errors.Wrapf(err, "reading from completion channel")
return
}
line := strings.TrimSpace(l)
if line != exp {
errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp)
return
}
}
// OK!
errchan <- nil
}()
return <-errchan
}

func printSuccess(mode string) {
metaltype := "metal"
if kola.QEMUOptions.Native4k {
Expand All @@ -173,9 +227,7 @@ func printSuccess(mode string) {
fmt.Printf("Successfully tested %s live installer for %s on %s (%s)\n", mode, kola.CosaBuild.Meta.OstreeVersion, kola.QEMUOptions.Firmware, metaltype)
}

func testPXE(inst platform.Install, completionfile string) error {
completionstamp := "coreos-installer-test-OK"

func testPXE(inst platform.Install) error {
config := ignv3types.Config{
Ignition: ignv3types.Ignition{
Version: "3.0.0",
Expand Down Expand Up @@ -209,47 +261,43 @@ func testPXE(inst platform.Install, completionfile string) error {
configStr = string(buf)
}

mach, err := inst.PXE(nil, configStr)
if err != nil {
return errors.Wrapf(err, "running PXE")
}
defer mach.Destroy()

err = mach.QemuInst.Wait()
inst.Builder = newQemuBuilder()
completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion")
if err != nil {
return err
}

err = exec.Command("grep", "-q", "-e", completionstamp, completionfile).Run()
if err != nil {
return fmt.Errorf("Failed to find %s in %s: %s", completionstamp, completionfile, err)
}

err = os.Remove(completionfile)
mach, err := inst.PXE(nil, configStr)
if err != nil {
return errors.Wrapf(err, "removing %s", completionfile)
return errors.Wrapf(err, "running PXE")
}
defer mach.Destroy()

return nil
return awaitCompletion(mach.QemuInst, completionChannel, []string{signalCompleteString})
}

func testLiveIso(inst platform.Install, completionfile string) error {
completionstamp := "coreos-installer-test-OK"
inst.Builder = newQemuBuilder()
completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion")
if err != nil {
return err
}

// We're just testing that executing our custom Ignition in the live
// path worked ok.
liveOKSignal := "live-test-OK"
var liveSignalOKUnit = `[Unit]
Requires=dev-virtio\\x2dports-completion.device
var liveSignalOKUnit = fmt.Sprintf(`[Unit]
Requires=dev-virtio\\x2dports-testisocompletion.device
OnFailure=emergency.target
OnFailureJobMode=isolate
Before=coreos-installer.service
[Service]
Type=oneshot
ExecStart=/bin/sh -c '/usr/bin/echo live-test-OK >/dev/virtio-ports/completion'
RemainAfterExit=yes
ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion'
[Install]
RequiredBy=multi-user.target
`
`, liveOKSignal)

liveConfig := ignv3types.Config{
Ignition: ignv3types.Ignition{
Expand Down Expand Up @@ -295,24 +343,5 @@ func testLiveIso(inst platform.Install, completionfile string) error {
}
defer mach.Destroy()

err = mach.QemuInst.Wait()
if err != nil {
return err
}

err = exec.Command("grep", "-q", "-e", liveOKSignal, completionfile).Run()
if err != nil {
return fmt.Errorf("Failed to find %s in %s: %s", liveOKSignal, completionfile, err)
}
err = exec.Command("grep", "-q", "-e", completionstamp, completionfile).Run()
if err != nil {
return fmt.Errorf("Failed to find %s in %s: %s", completionstamp, completionfile, err)
}

err = os.Remove(completionfile)
if err != nil {
return errors.Wrapf(err, "removing %s", completionfile)
}

return nil
return awaitCompletion(mach.QemuInst, completionChannel, []string{liveOKSignal, signalCompleteString})
}
68 changes: 20 additions & 48 deletions mantle/platform/metal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,26 @@ var (
}
)

type Install struct {
CosaBuild *sdk.LocalBuild

Firmware string
Native4k bool
Console bool
Insecure bool
QemuArgs []string
// NewMetalQemuBuilderDefault returns a QEMU builder instance with some
// defaults set up for bare metal.
func NewMetalQemuBuilderDefault() *QemuBuilder {
builder := NewBuilder()
// https://github.com/coreos/fedora-coreos-tracker/issues/388
// https://github.com/coreos/fedora-coreos-docs/pull/46
builder.Memory = 4096
if system.RpmArch() == "s390x" {
// FIXME - determine why this is
builder.Memory = int(math.Max(float64(builder.Memory), 16384))
}
return builder
}

type Install struct {
CosaBuild *sdk.LocalBuild
Builder *QemuBuilder
Insecure bool
LegacyInstaller bool
Native4k bool

// These are set by the install path
kargs []string
Expand Down Expand Up @@ -216,34 +226,6 @@ func setupMetalImage(builddir, metalimg, destdir string) (string, error) {
}
}

func newQemuBuilder(firmware string, console bool, native4k bool) *QemuBuilder {
builder := NewBuilder()
builder.Firmware = firmware

sectorSize := 0
if native4k {
sectorSize = 4096
}

builder.AddPrimaryDisk(&Disk{
Size: "12G", // Arbitrary

SectorSize: sectorSize,
})

// This applies just in the legacy case
builder.Memory = 1536
if system.RpmArch() == "s390x" {
// FIXME - determine why this is
builder.Memory = int(math.Max(float64(builder.Memory), 16384))
}

// For now, but in the future we should rely on log capture
builder.InheritConsole = console

return builder
}

func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
if kern.kernel == "" {
return nil, fmt.Errorf("Missing kernel artifact")
Expand All @@ -252,7 +234,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
return nil, fmt.Errorf("Missing initramfs artifact")
}

builder := newQemuBuilder(inst.Firmware, inst.Console, inst.Native4k)
builder := inst.Builder

tempdir, err := ioutil.TempDir("", "kola-testiso")
if err != nil {
Expand Down Expand Up @@ -452,7 +434,6 @@ func (t *installerRun) run() (*QemuInstance, error) {
usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9"
}
builder.Append("-netdev", usernetdev)
builder.Append(t.inst.QemuArgs...)

inst, err := builder.Exec()
if err != nil {
Expand All @@ -461,12 +442,6 @@ func (t *installerRun) run() (*QemuInstance, error) {
return inst, nil
}

func setBuilderLiveMemory(builder *QemuBuilder) {
// https://github.com/coreos/fedora-coreos-tracker/issues/388
// https://github.com/coreos/fedora-coreos-docs/pull/46
builder.Memory = int(math.Max(float64(builder.Memory), 4096))
}

func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, error) {
t, err := inst.setup(kern)
if err != nil {
Expand All @@ -476,7 +451,6 @@ func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine,

kargs := renderBaseKargs()
if !legacy {
setBuilderLiveMemory(t.builder)
kargs = append(kargs, liveKargs...)
}

Expand Down Expand Up @@ -666,10 +640,8 @@ WantedBy=multi-user.target
return nil, errors.Wrapf(err, "running coreos-installer iso embed")
}

qemubuilder := newQemuBuilder(inst.Firmware, inst.Console, inst.Native4k)
setBuilderLiveMemory(qemubuilder)
qemubuilder := inst.Builder
qemubuilder.AddInstallIso(isoEmbeddedPath)
qemubuilder.Append(inst.QemuArgs...)

qinst, err := qemubuilder.Exec()
if err != nil {
Expand Down
Loading