From d3f4d9dab6d4db456bda8d0282445373b0be3194 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 20 Mar 2020 01:36:14 +0000 Subject: [PATCH 1/2] metal/testiso: Also support installs via `coreos-installer iso embed` Confusingly, the `testiso` command was (is) actually testing PXE installs. It still does, but now the `platform/metal` infrastructure has a new API to do an install via an ISO (scripting embedding an Ignition config with `coreos-installer iso embed`). --- mantle/cmd/kola/testiso.go | 106 ++++++++++++++++- mantle/platform/metal.go | 229 ++++++++++++++++++++++++++++++++++++- mantle/platform/qemu.go | 6 + 3 files changed, 329 insertions(+), 12 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index fd08aff27b..1726291c3d 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -54,6 +54,7 @@ var ( legacy bool nolive bool + nopxe bool ) var signalCompletionUnit = `[Unit] @@ -70,7 +71,8 @@ RequiredBy=multi-user.target func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVarP(&legacy, "legacy", "K", false, "Test legacy installer") - cmdTestIso.Flags().BoolVarP(&nolive, "no-live", "L", false, "Skip testing live installer") + cmdTestIso.Flags().BoolVarP(&nolive, "no-live", "L", false, "Skip testing live installer (PXE and ISO)") + cmdTestIso.Flags().BoolVarP(&nopxe, "no-pxe", "P", false, "Skip testing live installer PXE") root.AddCommand(cmdTestIso) } @@ -130,14 +132,22 @@ func runTestIso(cmd *cobra.Command, args []string) error { if !foundLive { return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Name) } - ranTest = true - inst := baseInst // Pretend this is Rust and I wrote .copy() + if !nopxe { + ranTest = true + instPxe := baseInst // Pretend this is Rust and I wrote .copy() - if err := testPXE(inst, completionfile); err != nil { - return err + if err := testPXE(instPxe, completionfile); err != nil { + return err + } + fmt.Printf("Successfully tested PXE live installer for %s\n", kola.CosaBuild.OstreeVersion) } - fmt.Printf("Successfully tested live installer for %s\n", kola.CosaBuild.OstreeVersion) + ranTest = true + instIso := baseInst // Pretend this is Rust and I wrote .copy() + if err := testLiveIso(instIso, completionfile); err != nil { + return err + } + fmt.Printf("Successfully tested ISO live installer for %s\n", kola.CosaBuild.OstreeVersion) } if !ranTest { @@ -206,3 +216,87 @@ func testPXE(inst platform.Install, completionfile string) error { return nil } + +func testLiveIso(inst platform.Install, completionfile string) error { + completionstamp := "coreos-installer-test-OK" + + // We're just testing that executing our custom Ignition in the live + // path worked ok. + liveOKSignal := "live-test-OK" + var liveSignalOKUnit = `[Unit] + Requires=dev-virtio\\x2dports-completion.device + OnFailure=emergency.target + OnFailureJobMode=isolate + Before=coreos-installer.service + [Service] + Type=oneshot + ExecStart=/bin/sh -c '/usr/bin/echo live-test-OK >/dev/virtio-ports/completion' + [Install] + RequiredBy=multi-user.target + ` + + liveConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "live-signal-ok.service", + Contents: &liveSignalOKUnit, + Enabled: util.BoolToPtr(true), + }, + }, + }, + } + liveConfigBuf, err := json.Marshal(liveConfig) + if err != nil { + return err + } + + targetConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "coreos-test-installer.service", + Contents: &signalCompletionUnit, + Enabled: util.BoolToPtr(true), + }, + }, + }, + } + + targetIgnitionBuf, err := json.Marshal(targetConfig) + if err != nil { + return err + } + mach, err := inst.InstallViaISOEmbed(nil, string(liveConfigBuf), string(targetIgnitionBuf)) + if err != nil { + return errors.Wrapf(err, "running iso install") + } + defer mach.Destroy() + + err = mach.QemuInst.Wait() + if err != nil { + return err + } + + err = exec.Command("grep", "-q", "-e", liveOKSignal, completionfile).Run() + if err != nil { + return fmt.Errorf("Failed to find %s in %s: %s", liveOKSignal, completionfile, err) + } + err = exec.Command("grep", "-q", "-e", completionstamp, completionfile).Run() + if err != nil { + return fmt.Errorf("Failed to find %s in %s: %s", completionstamp, completionfile, err) + } + + err = os.Remove(completionfile) + if err != nil { + return errors.Wrapf(err, "removing %s", completionfile) + } + + return nil +} diff --git a/mantle/platform/metal.go b/mantle/platform/metal.go index 1607ebc958..8a80c7ae2b 100644 --- a/mantle/platform/metal.go +++ b/mantle/platform/metal.go @@ -15,7 +15,9 @@ package platform import ( + "encoding/json" "fmt" + "io" "io/ioutil" "math" "net" @@ -24,10 +26,40 @@ import ( "path/filepath" "strings" + v3 "github.com/coreos/ignition/v2/config/v3_0" + ignv3types "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/pkg/errors" + "github.com/vincent-petithory/dataurl" + "github.com/coreos/mantle/cosa" + "github.com/coreos/mantle/platform/conf" "github.com/coreos/mantle/system" "github.com/coreos/mantle/system/exec" - "github.com/pkg/errors" + "github.com/coreos/mantle/util" +) + +const ( + // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option + defaultQemuHostIPv4 = "10.0.2.2" + + targetDevice = "/dev/vda" + + // rebootUnit is a copy of the system one without the ConditionPathExists + rebootUnit = `[Unit] + Description=Reboot after CoreOS Installer + After=coreos-installer.service + Requires=coreos-installer.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + + [Service] + Type=simple + ExecStart=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target +` ) // TODO derive this from docs, or perhaps include kargs in cosa metadata? @@ -55,8 +87,9 @@ type Install struct { LegacyInstaller bool // These are set by the install path - kargs []string - ignition string + kargs []string + ignition string + liveIgnition string } type InstalledMachine struct { @@ -414,6 +447,12 @@ func (t *installerRun) run() (*QemuInstance, error) { return inst, nil } +func setBuilderLiveMemory(builder *QemuBuilder) { + // https://github.com/coreos/fedora-coreos-tracker/issues/388 + // https://github.com/coreos/fedora-coreos-docs/pull/46 + builder.Memory = int(math.Max(float64(builder.Memory), 4096)) +} + func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, error) { t, err := inst.setup(kern) if err != nil { @@ -423,9 +462,7 @@ func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, kargs := renderBaseKargs() if !legacy { - // https://github.com/coreos/fedora-coreos-tracker/issues/388 - // https://github.com/coreos/fedora-coreos-docs/pull/46 - t.builder.Memory = int(math.Max(float64(t.builder.Memory), 4096)) + setBuilderLiveMemory(t.builder) kargs = append(kargs, liveKargs...) } @@ -443,3 +480,183 @@ func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, tempdir: t.tempdir, }, nil } + +func generatePointerIgnitionString(target string) string { + p := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + Config: ignv3types.IgnitionConfig{ + Merge: []ignv3types.ConfigReference{ + ignv3types.ConfigReference{ + Source: &target, + }, + }, + }, + }, + } + + buf, err := json.Marshal(p) + if err != nil { + panic(err) + } + return string(buf) +} + +func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgniton, targetIgnition string) (*InstalledMachine, error) { + if inst.CosaBuild.BuildArtifacts.Metal == nil { + return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.OstreeVersion) + } + if inst.CosaBuild.BuildArtifacts.LiveIso == nil { + return nil, fmt.Errorf("Build %s must have a live ISO", inst.CosaBuild.Name) + } + + if len(inst.kargs) > 0 { + return nil, errors.New("injecting kargs is not supported yet, see https://github.com/coreos/coreos-installer/issues/164") + } + + inst.kargs = kargs + inst.ignition = targetIgnition + inst.liveIgnition = liveIgniton + + tempdir, err := ioutil.TempDir("", "mantle-metal") + if err != nil { + return nil, err + } + cleanupTempdir := true + defer func() { + if cleanupTempdir { + os.RemoveAll(tempdir) + } + }() + + if err := ioutil.WriteFile(filepath.Join(tempdir, "target.ign"), []byte(inst.ignition), 0644); err != nil { + return nil, err + } + + builddir := filepath.Dir(inst.CosaBuildDir) + srcisopath := filepath.Join(builddir, inst.CosaBuild.BuildArtifacts.LiveIso.Path) + metalimg := inst.CosaBuild.BuildArtifacts.Metal.Path + metalname, err := setupMetalImage(builddir, metalimg, tempdir) + if err != nil { + return nil, errors.Wrapf(err, "setting up metal image") + } + + providedLiveConfig, _, err := v3.Parse([]byte(inst.liveIgnition)) + if err != nil { + return nil, errors.Wrapf(err, "parsing provided live config") + } + + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tempdir))) + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + port := listener.Addr().(*net.TCPAddr).Port + // Yeah this leaks + go func() { + http.Serve(listener, mux) + }() + baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) + + insecureOpt := "" + if inst.Insecure { + insecureOpt = "--insecure" + } + pointerIgnitionPath := "/var/opt/pointer.ign" + installerUnit := fmt.Sprintf(` +[Unit] +After=network-online.target +Wants=network-online.target +[Service] +RemainAfterExit=yes +Type=oneshot +ExecStart=/usr/bin/coreos-installer install --image-url %s/%s --ignition %s %s %s +StandardOutput=kmsg+console +StandardError=kmsg+console +[Install] +WantedBy=multi-user.target +`, baseurl, metalname, pointerIgnitionPath, insecureOpt, targetDevice) + // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 + // when it arrives + pointerIgnitionStr := generatePointerIgnitionString(baseurl + "/target.ign") + pointerIgnitionEnc := dataurl.EncodeBytes([]byte(pointerIgnitionStr)) + mode := 0644 + rebootUnitP := string(rebootUnit) + installerConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "coreos-installer.service", + Contents: &installerUnit, + Enabled: util.BoolToPtr(true), + }, + { + Name: "coreos-installer-reboot.service", + Contents: &rebootUnitP, + Enabled: util.BoolToPtr(true), + }, + }, + }, + Storage: ignv3types.Storage{ + Files: []ignv3types.File{ + { + Node: ignv3types.Node{ + Path: pointerIgnitionPath, + }, + FileEmbedded1: ignv3types.FileEmbedded1{ + Contents: ignv3types.FileContents{ + Source: &pointerIgnitionEnc, + }, + Mode: &mode, + }, + }, + }, + }, + } + mergedConfig := v3.Merge(providedLiveConfig, installerConfig) + mergedConfig = v3.Merge(mergedConfig, conf.GetAutologin()) + + isoEmbeddedPath := filepath.Join(tempdir, "test.iso") + // TODO ensure this tempdir is underneath cosa tempdir so we can reliably reflink + if err := exec.Command("cp", "--reflink=auto", srcisopath, isoEmbeddedPath).Run(); err != nil { + return nil, errors.Wrapf(err, "copying iso") + } + instCmd := exec.Command("coreos-installer", "iso", "embed", isoEmbeddedPath) + instCmd.Stderr = os.Stderr + instCmdStdin, err := instCmd.StdinPipe() + if err != nil { + return nil, err + } + go func() { + mergedConfigSerialized, err := json.Marshal(mergedConfig) + if err != nil { + panic(err) + } + defer instCmdStdin.Close() + if _, err := io.WriteString(instCmdStdin, string(mergedConfigSerialized)); err != nil { + panic(err) + } + }() + if err := instCmd.Run(); err != nil { + return nil, errors.Wrapf(err, "running coreos-installer iso embed") + } + + qemubuilder := newQemuBuilder(inst.Firmware) + setBuilderLiveMemory(qemubuilder) + qemubuilder.AddInstallIso(isoEmbeddedPath) + qemubuilder.Append(inst.QemuArgs...) + + qinst, err := qemubuilder.Exec() + if err != nil { + return nil, err + } + cleanupTempdir = false // Transfer ownership + return &InstalledMachine{ + QemuInst: qinst, + tempdir: tempdir, + }, nil +} diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 273c329d70..07fea01f36 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -519,6 +519,12 @@ func (builder *QemuBuilder) AddDisk(disk *Disk) error { return builder.addDiskImpl(disk, false) } +// AddInstallIso adds an ISO image, configuring to boot from it once +func (builder *QemuBuilder) AddInstallIso(path string) error { + builder.Append("-boot", "once=d", "-cdrom", path) + return nil +} + func (builder *QemuBuilder) finalize() { if builder.finalized { return From 9cfd6be5b7bb6fe2b207dadfecde8a4baa0b0cb3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 23 Mar 2020 18:07:09 +0000 Subject: [PATCH 2/2] metal/testiso: Don't inherit console by default, log cp errors The console is useful for debugging but in the success path is way too verbose. What we really want to do is hook into the console the same way main kola tests do. That will come in the future. Also log errors from `cp` since that's happening in CI (probably `ENOSPC`); and try to fix that by setting `TMPDIR` in CI. --- .cci.jenkinsfile | 2 +- mantle/cmd/kola/testiso.go | 4 ++++ mantle/platform/metal.go | 13 ++++++++----- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index bc55986ce2..815ae1327c 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -50,7 +50,7 @@ pod(image: 'registry.fedoraproject.org/fedora:31', runAsUser: 0, kvm: true, memo } stage("Image tests") { - cosa_cmd("kola testiso -S") + shwrap("cd /srv && env TMPDIR=\$(pwd)/tmp/ cosa kola testiso -S") } // Needs to be last because it's destructive diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 1726291c3d..6e7d25230c 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -55,6 +55,8 @@ var ( legacy bool nolive bool nopxe bool + + console bool ) var signalCompletionUnit = `[Unit] @@ -73,6 +75,7 @@ func init() { cmdTestIso.Flags().BoolVarP(&legacy, "legacy", "K", false, "Test legacy installer") cmdTestIso.Flags().BoolVarP(&nolive, "no-live", "L", false, "Skip testing live installer (PXE and ISO)") cmdTestIso.Flags().BoolVarP(&nopxe, "no-pxe", "P", false, "Skip testing live installer PXE") + cmdTestIso.Flags().BoolVar(&console, "console", false, "Display qemu console to stdout") root.AddCommand(cmdTestIso) } @@ -86,6 +89,7 @@ func runTestIso(cmd *cobra.Command, args []string) error { CosaBuildDir: kola.Options.CosaBuild, CosaBuild: kola.CosaBuild, + Console: console, Firmware: kola.QEMUOptions.Firmware, } diff --git a/mantle/platform/metal.go b/mantle/platform/metal.go index 8a80c7ae2b..960926c4c0 100644 --- a/mantle/platform/metal.go +++ b/mantle/platform/metal.go @@ -81,6 +81,7 @@ type Install struct { CosaBuild *cosa.Build Firmware string + Console bool Insecure bool QemuArgs []string @@ -215,7 +216,7 @@ func setupMetalImage(builddir, metalimg, destdir string) (string, error) { } } -func newQemuBuilder(firmware string) *QemuBuilder { +func newQemuBuilder(firmware string, console bool) *QemuBuilder { builder := NewBuilder("", false) builder.Firmware = firmware builder.AddDisk(&Disk{ @@ -230,7 +231,7 @@ func newQemuBuilder(firmware string) *QemuBuilder { } // For now, but in the future we should rely on log capture - builder.InheritConsole = true + builder.InheritConsole = console return builder } @@ -243,7 +244,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { return nil, fmt.Errorf("Missing initramfs artifact") } - builder := newQemuBuilder(inst.Firmware) + builder := newQemuBuilder(inst.Firmware, inst.Console) tempdir, err := ioutil.TempDir("", "kola-testiso") if err != nil { @@ -622,7 +623,9 @@ WantedBy=multi-user.target isoEmbeddedPath := filepath.Join(tempdir, "test.iso") // TODO ensure this tempdir is underneath cosa tempdir so we can reliably reflink - if err := exec.Command("cp", "--reflink=auto", srcisopath, isoEmbeddedPath).Run(); err != nil { + cpcmd := exec.Command("cp", "--reflink=auto", srcisopath, isoEmbeddedPath) + cpcmd.Stderr = os.Stderr + if err := cpcmd.Run(); err != nil { return nil, errors.Wrapf(err, "copying iso") } instCmd := exec.Command("coreos-installer", "iso", "embed", isoEmbeddedPath) @@ -645,7 +648,7 @@ WantedBy=multi-user.target return nil, errors.Wrapf(err, "running coreos-installer iso embed") } - qemubuilder := newQemuBuilder(inst.Firmware) + qemubuilder := newQemuBuilder(inst.Firmware, inst.Console) setBuilderLiveMemory(qemubuilder) qemubuilder.AddInstallIso(isoEmbeddedPath) qemubuilder.Append(inst.QemuArgs...)