diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index bc55986ce2..815ae1327c 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -50,7 +50,7 @@ pod(image: 'registry.fedoraproject.org/fedora:31', runAsUser: 0, kvm: true, memo } stage("Image tests") { - cosa_cmd("kola testiso -S") + shwrap("cd /srv && env TMPDIR=\$(pwd)/tmp/ cosa kola testiso -S") } // Needs to be last because it's destructive diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index fd08aff27b..6e7d25230c 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -54,6 +54,9 @@ var ( legacy bool nolive bool + nopxe bool + + console bool ) var signalCompletionUnit = `[Unit] @@ -70,7 +73,9 @@ RequiredBy=multi-user.target func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVarP(&legacy, "legacy", "K", false, "Test legacy installer") - cmdTestIso.Flags().BoolVarP(&nolive, "no-live", "L", false, "Skip testing live installer") + cmdTestIso.Flags().BoolVarP(&nolive, "no-live", "L", false, "Skip testing live installer (PXE and ISO)") + cmdTestIso.Flags().BoolVarP(&nopxe, "no-pxe", "P", false, "Skip testing live installer PXE") + cmdTestIso.Flags().BoolVar(&console, "console", false, "Display qemu console to stdout") root.AddCommand(cmdTestIso) } @@ -84,6 +89,7 @@ func runTestIso(cmd *cobra.Command, args []string) error { CosaBuildDir: kola.Options.CosaBuild, CosaBuild: kola.CosaBuild, + Console: console, Firmware: kola.QEMUOptions.Firmware, } @@ -130,14 +136,22 @@ func runTestIso(cmd *cobra.Command, args []string) error { if !foundLive { return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Name) } - ranTest = true - inst := baseInst // Pretend this is Rust and I wrote .copy() + if !nopxe { + ranTest = true + instPxe := baseInst // Pretend this is Rust and I wrote .copy() - if err := testPXE(inst, completionfile); err != nil { - return err + if err := testPXE(instPxe, completionfile); err != nil { + return err + } + fmt.Printf("Successfully tested PXE live installer for %s\n", kola.CosaBuild.OstreeVersion) } - fmt.Printf("Successfully tested live installer for %s\n", kola.CosaBuild.OstreeVersion) + ranTest = true + instIso := baseInst // Pretend this is Rust and I wrote .copy() + if err := testLiveIso(instIso, completionfile); err != nil { + return err + } + fmt.Printf("Successfully tested ISO live installer for %s\n", kola.CosaBuild.OstreeVersion) } if !ranTest { @@ -206,3 +220,87 @@ func testPXE(inst platform.Install, completionfile string) error { return nil } + +func testLiveIso(inst platform.Install, completionfile string) error { + completionstamp := "coreos-installer-test-OK" + + // We're just testing that executing our custom Ignition in the live + // path worked ok. + liveOKSignal := "live-test-OK" + var liveSignalOKUnit = `[Unit] + Requires=dev-virtio\\x2dports-completion.device + OnFailure=emergency.target + OnFailureJobMode=isolate + Before=coreos-installer.service + [Service] + Type=oneshot + ExecStart=/bin/sh -c '/usr/bin/echo live-test-OK >/dev/virtio-ports/completion' + [Install] + RequiredBy=multi-user.target + ` + + liveConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "live-signal-ok.service", + Contents: &liveSignalOKUnit, + Enabled: util.BoolToPtr(true), + }, + }, + }, + } + liveConfigBuf, err := json.Marshal(liveConfig) + if err != nil { + return err + } + + targetConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "coreos-test-installer.service", + Contents: &signalCompletionUnit, + Enabled: util.BoolToPtr(true), + }, + }, + }, + } + + targetIgnitionBuf, err := json.Marshal(targetConfig) + if err != nil { + return err + } + mach, err := inst.InstallViaISOEmbed(nil, string(liveConfigBuf), string(targetIgnitionBuf)) + if err != nil { + return errors.Wrapf(err, "running iso install") + } + defer mach.Destroy() + + err = mach.QemuInst.Wait() + if err != nil { + return err + } + + err = exec.Command("grep", "-q", "-e", liveOKSignal, completionfile).Run() + if err != nil { + return fmt.Errorf("Failed to find %s in %s: %s", liveOKSignal, completionfile, err) + } + err = exec.Command("grep", "-q", "-e", completionstamp, completionfile).Run() + if err != nil { + return fmt.Errorf("Failed to find %s in %s: %s", completionstamp, completionfile, err) + } + + err = os.Remove(completionfile) + if err != nil { + return errors.Wrapf(err, "removing %s", completionfile) + } + + return nil +} diff --git a/mantle/platform/metal.go b/mantle/platform/metal.go index 1607ebc958..960926c4c0 100644 --- a/mantle/platform/metal.go +++ b/mantle/platform/metal.go @@ -15,7 +15,9 @@ package platform import ( + "encoding/json" "fmt" + "io" "io/ioutil" "math" "net" @@ -24,10 +26,40 @@ import ( "path/filepath" "strings" + v3 "github.com/coreos/ignition/v2/config/v3_0" + ignv3types "github.com/coreos/ignition/v2/config/v3_0/types" + "github.com/pkg/errors" + "github.com/vincent-petithory/dataurl" + "github.com/coreos/mantle/cosa" + "github.com/coreos/mantle/platform/conf" "github.com/coreos/mantle/system" "github.com/coreos/mantle/system/exec" - "github.com/pkg/errors" + "github.com/coreos/mantle/util" +) + +const ( + // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option + defaultQemuHostIPv4 = "10.0.2.2" + + targetDevice = "/dev/vda" + + // rebootUnit is a copy of the system one without the ConditionPathExists + rebootUnit = `[Unit] + Description=Reboot after CoreOS Installer + After=coreos-installer.service + Requires=coreos-installer.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + + [Service] + Type=simple + ExecStart=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target +` ) // TODO derive this from docs, or perhaps include kargs in cosa metadata? @@ -49,14 +81,16 @@ type Install struct { CosaBuild *cosa.Build Firmware string + Console bool Insecure bool QemuArgs []string LegacyInstaller bool // These are set by the install path - kargs []string - ignition string + kargs []string + ignition string + liveIgnition string } type InstalledMachine struct { @@ -182,7 +216,7 @@ func setupMetalImage(builddir, metalimg, destdir string) (string, error) { } } -func newQemuBuilder(firmware string) *QemuBuilder { +func newQemuBuilder(firmware string, console bool) *QemuBuilder { builder := NewBuilder("", false) builder.Firmware = firmware builder.AddDisk(&Disk{ @@ -197,7 +231,7 @@ func newQemuBuilder(firmware string) *QemuBuilder { } // For now, but in the future we should rely on log capture - builder.InheritConsole = true + builder.InheritConsole = console return builder } @@ -210,7 +244,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { return nil, fmt.Errorf("Missing initramfs artifact") } - builder := newQemuBuilder(inst.Firmware) + builder := newQemuBuilder(inst.Firmware, inst.Console) tempdir, err := ioutil.TempDir("", "kola-testiso") if err != nil { @@ -414,6 +448,12 @@ func (t *installerRun) run() (*QemuInstance, error) { return inst, nil } +func setBuilderLiveMemory(builder *QemuBuilder) { + // https://github.com/coreos/fedora-coreos-tracker/issues/388 + // https://github.com/coreos/fedora-coreos-docs/pull/46 + builder.Memory = int(math.Max(float64(builder.Memory), 4096)) +} + func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, error) { t, err := inst.setup(kern) if err != nil { @@ -423,9 +463,7 @@ func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, kargs := renderBaseKargs() if !legacy { - // https://github.com/coreos/fedora-coreos-tracker/issues/388 - // https://github.com/coreos/fedora-coreos-docs/pull/46 - t.builder.Memory = int(math.Max(float64(t.builder.Memory), 4096)) + setBuilderLiveMemory(t.builder) kargs = append(kargs, liveKargs...) } @@ -443,3 +481,185 @@ func (inst *Install) runPXE(kern *kernelSetup, legacy bool) (*InstalledMachine, tempdir: t.tempdir, }, nil } + +func generatePointerIgnitionString(target string) string { + p := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + Config: ignv3types.IgnitionConfig{ + Merge: []ignv3types.ConfigReference{ + ignv3types.ConfigReference{ + Source: &target, + }, + }, + }, + }, + } + + buf, err := json.Marshal(p) + if err != nil { + panic(err) + } + return string(buf) +} + +func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgniton, targetIgnition string) (*InstalledMachine, error) { + if inst.CosaBuild.BuildArtifacts.Metal == nil { + return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.OstreeVersion) + } + if inst.CosaBuild.BuildArtifacts.LiveIso == nil { + return nil, fmt.Errorf("Build %s must have a live ISO", inst.CosaBuild.Name) + } + + if len(inst.kargs) > 0 { + return nil, errors.New("injecting kargs is not supported yet, see https://github.com/coreos/coreos-installer/issues/164") + } + + inst.kargs = kargs + inst.ignition = targetIgnition + inst.liveIgnition = liveIgniton + + tempdir, err := ioutil.TempDir("", "mantle-metal") + if err != nil { + return nil, err + } + cleanupTempdir := true + defer func() { + if cleanupTempdir { + os.RemoveAll(tempdir) + } + }() + + if err := ioutil.WriteFile(filepath.Join(tempdir, "target.ign"), []byte(inst.ignition), 0644); err != nil { + return nil, err + } + + builddir := filepath.Dir(inst.CosaBuildDir) + srcisopath := filepath.Join(builddir, inst.CosaBuild.BuildArtifacts.LiveIso.Path) + metalimg := inst.CosaBuild.BuildArtifacts.Metal.Path + metalname, err := setupMetalImage(builddir, metalimg, tempdir) + if err != nil { + return nil, errors.Wrapf(err, "setting up metal image") + } + + providedLiveConfig, _, err := v3.Parse([]byte(inst.liveIgnition)) + if err != nil { + return nil, errors.Wrapf(err, "parsing provided live config") + } + + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tempdir))) + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + port := listener.Addr().(*net.TCPAddr).Port + // Yeah this leaks + go func() { + http.Serve(listener, mux) + }() + baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) + + insecureOpt := "" + if inst.Insecure { + insecureOpt = "--insecure" + } + pointerIgnitionPath := "/var/opt/pointer.ign" + installerUnit := fmt.Sprintf(` +[Unit] +After=network-online.target +Wants=network-online.target +[Service] +RemainAfterExit=yes +Type=oneshot +ExecStart=/usr/bin/coreos-installer install --image-url %s/%s --ignition %s %s %s +StandardOutput=kmsg+console +StandardError=kmsg+console +[Install] +WantedBy=multi-user.target +`, baseurl, metalname, pointerIgnitionPath, insecureOpt, targetDevice) + // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 + // when it arrives + pointerIgnitionStr := generatePointerIgnitionString(baseurl + "/target.ign") + pointerIgnitionEnc := dataurl.EncodeBytes([]byte(pointerIgnitionStr)) + mode := 0644 + rebootUnitP := string(rebootUnit) + installerConfig := ignv3types.Config{ + Ignition: ignv3types.Ignition{ + Version: "3.0.0", + }, + Systemd: ignv3types.Systemd{ + Units: []ignv3types.Unit{ + { + Name: "coreos-installer.service", + Contents: &installerUnit, + Enabled: util.BoolToPtr(true), + }, + { + Name: "coreos-installer-reboot.service", + Contents: &rebootUnitP, + Enabled: util.BoolToPtr(true), + }, + }, + }, + Storage: ignv3types.Storage{ + Files: []ignv3types.File{ + { + Node: ignv3types.Node{ + Path: pointerIgnitionPath, + }, + FileEmbedded1: ignv3types.FileEmbedded1{ + Contents: ignv3types.FileContents{ + Source: &pointerIgnitionEnc, + }, + Mode: &mode, + }, + }, + }, + }, + } + mergedConfig := v3.Merge(providedLiveConfig, installerConfig) + mergedConfig = v3.Merge(mergedConfig, conf.GetAutologin()) + + isoEmbeddedPath := filepath.Join(tempdir, "test.iso") + // TODO ensure this tempdir is underneath cosa tempdir so we can reliably reflink + cpcmd := exec.Command("cp", "--reflink=auto", srcisopath, isoEmbeddedPath) + cpcmd.Stderr = os.Stderr + if err := cpcmd.Run(); err != nil { + return nil, errors.Wrapf(err, "copying iso") + } + instCmd := exec.Command("coreos-installer", "iso", "embed", isoEmbeddedPath) + instCmd.Stderr = os.Stderr + instCmdStdin, err := instCmd.StdinPipe() + if err != nil { + return nil, err + } + go func() { + mergedConfigSerialized, err := json.Marshal(mergedConfig) + if err != nil { + panic(err) + } + defer instCmdStdin.Close() + if _, err := io.WriteString(instCmdStdin, string(mergedConfigSerialized)); err != nil { + panic(err) + } + }() + if err := instCmd.Run(); err != nil { + return nil, errors.Wrapf(err, "running coreos-installer iso embed") + } + + qemubuilder := newQemuBuilder(inst.Firmware, inst.Console) + setBuilderLiveMemory(qemubuilder) + qemubuilder.AddInstallIso(isoEmbeddedPath) + qemubuilder.Append(inst.QemuArgs...) + + qinst, err := qemubuilder.Exec() + if err != nil { + return nil, err + } + cleanupTempdir = false // Transfer ownership + return &InstalledMachine{ + QemuInst: qinst, + tempdir: tempdir, + }, nil +} diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 273c329d70..07fea01f36 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -519,6 +519,12 @@ func (builder *QemuBuilder) AddDisk(disk *Disk) error { return builder.addDiskImpl(disk, false) } +// AddInstallIso adds an ISO image, configuring to boot from it once +func (builder *QemuBuilder) AddInstallIso(path string) error { + builder.Append("-boot", "once=d", "-cdrom", path) + return nil +} + func (builder *QemuBuilder) finalize() { if builder.finalized { return