Skip to content

qemu: Support monitoring for failures in the initramfs #1290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 1 commit into from
Apr 3, 2020
Merged

qemu: Support monitoring for failures in the initramfs #1290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions mantle/cmd/kola/qemuexec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@ var (
bindro []string
bindrw []string

directIgnition bool
forceConfigInjection bool
directIgnition bool
forceConfigInjection bool
propagateInitramfsFailure bool
)

func init() {
Expand All @@ -72,6 +73,8 @@ func init() {
cmdQemuExec.Flags().StringArrayVar(&bindro, "bind-ro", nil, "Mount readonly via 9pfs a host directory (use --bind-ro=/path/to/host,/var/mnt/guest")
cmdQemuExec.Flags().StringArrayVar(&bindrw, "bind-rw", nil, "Same as above, but writable")
cmdQemuExec.Flags().BoolVarP(&forceConfigInjection, "inject-ignition", "", false, "Force injecting Ignition config using guestfs")
cmdQemuExec.Flags().BoolVar(&propagateInitramfsFailure, "propagate-initramfs-failure", false, "Error out if the system fails in the initramfs")

}

func renderFragments(config v3types.Config) (*v3types.Config, error) {
Expand Down Expand Up @@ -202,8 +205,13 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
}
defer inst.Destroy()

// Ignore errors
_ = inst.Wait()

return nil
if propagateInitramfsFailure {
err = inst.WaitAll()
if err != nil {
return err
}
return nil
} else {
return inst.Wait()
}
}
2 changes: 1 addition & 1 deletion mantle/cmd/kola/testiso.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ func runTestIso(cmd *cobra.Command, args []string) error {
func awaitCompletion(inst *platform.QemuInstance, qchan *os.File, expected []string) error {
errchan := make(chan error)
go func() {
if err := inst.Wait(); err != nil {
if err := inst.WaitAll(); err != nil {
errchan <- err
}
time.Sleep(1 * time.Minute)
Expand Down
105 changes: 105 additions & 0 deletions mantle/kola/tests/ignition/qemufailure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Copyright 2020 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ignition

import (
"fmt"
"os"
"time"

ignv3types "github.com/coreos/ignition/v2/config/v3_0/types"
"github.com/pkg/errors"

"github.com/coreos/mantle/kola"
"github.com/coreos/mantle/kola/cluster"
"github.com/coreos/mantle/kola/register"
"github.com/coreos/mantle/platform"
"github.com/coreos/mantle/util"
)

func init() {
register.RegisterTest(&register.Test{
Name: "coreos.ignition.failure",
Run: runIgnitionFailure,
ClusterSize: 0,
Platforms: []string{"qemu-unpriv"},
})
}

func runIgnitionFailure(c cluster.TestCluster) {
if err := ignitionFailure(c); err != nil {
c.Fatal(err.Error())
}
}

func ignitionFailure(c cluster.TestCluster) error {
// TODO remove this once the feature lands
c.H.Skip("Needs https://github.com/coreos/ignition-dracut/pull/146")
// We can't create files in / due to the immutable bit OSTree creates, so
// this is a convenient way to test Ignition failure.
failConfig := ignv3types.Config{
Ignition: ignv3types.Ignition{
Version: "3.0.0",
},
Storage: ignv3types.Storage{
Files: []ignv3types.File{
{
Node: ignv3types.Node{
Path: "/notwritable.txt",
},
FileEmbedded1: ignv3types.FileEmbedded1{
Contents: ignv3types.FileContents{
Source: util.StrToPtr("data:,hello%20world%0A"),
},
Mode: util.IntToPtr(420),
},
},
},
},
}
builder := platform.NewBuilder()
builder.SetConfig(failConfig, kola.Options.IgnitionVersion == "v2")
builder.AddPrimaryDisk(&platform.Disk{
BackingFile: kola.QEMUOptions.DiskImage,
})
builder.Memory = 1024
inst, err := builder.Exec()
if err != nil {
return err
}
defer inst.Destroy()
errchan := make(chan error)
go func() {
err := inst.WaitAll()
if err == nil {
err = fmt.Errorf("Ignition unexpectedly succeeded")
} else if err == platform.ErrInitramfsEmergency {
// The expected case
err = nil
} else {
err = errors.Wrapf(err, "expected initramfs emergency.target error")
}
errchan <- err
}()
go func() {
time.Sleep(2 * time.Minute)
proc := os.Process{
Pid: inst.Pid(),
}
proc.Kill()
errchan <- errors.New("timed out waiting for initramfs error")
}()
return <-errchan
}
4 changes: 4 additions & 0 deletions mantle/platform/machine/aws/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ func (am *machine) SSH(cmd string) ([]byte, []byte, error) {
return am.cluster.SSH(am, cmd)
}

func (am *machine) IgnitionError() error {
return nil
}

func (am *machine) Reboot() error {
return platform.RebootMachine(am, am.journal)
}
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/azure/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ func (am *machine) SSH(cmd string) ([]byte, []byte, error) {
return am.cluster.SSH(am, cmd)
}

func (am *machine) IgnitionError() error {
return nil
}

// Re-fetch the Public & Private IP address for the event that it's changed during the reboot
func (am *machine) refetchIPs() error {
var err error
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/do/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ func (dm *machine) SSH(cmd string) ([]byte, []byte, error) {
return dm.cluster.SSH(dm, cmd)
}

func (dm *machine) IgnitionError() error {
return nil
}

func (dm *machine) Reboot() error {
return platform.RebootMachine(dm, dm.journal)
}
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/esx/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ func (em *machine) SSH(cmd string) ([]byte, []byte, error) {
return em.cluster.SSH(em, cmd)
}

func (em *machine) IgnitionError() error {
return nil
}

func (em *machine) Reboot() error {
return platform.RebootMachine(em, em.journal)
}
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/gcloud/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ func (gm *machine) SSH(cmd string) ([]byte, []byte, error) {
return gm.gc.SSH(gm, cmd)
}

func (gm *machine) IgnitionError() error {
return nil
}

func (gm *machine) Reboot() error {
return platform.RebootMachine(gm, gm.journal)
}
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/openstack/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ func (om *machine) SSH(cmd string) ([]byte, []byte, error) {
return om.cluster.SSH(om, cmd)
}

func (om *machine) IgnitionError() error {
return nil
}

func (om *machine) Reboot() error {
return platform.RebootMachine(om, om.journal)
}
Expand Down
4 changes: 4 additions & 0 deletions mantle/platform/machine/packet/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ func (pm *machine) SSH(cmd string) ([]byte, []byte, error) {
return pm.cluster.SSH(pm, cmd)
}

func (pm *machine) IgnitionError() error {
return nil
}

func (pm *machine) Reboot() error {
return platform.RebootMachine(pm, pm.journal)
}
Expand Down
9 changes: 9 additions & 0 deletions mantle/platform/machine/unprivqemu/machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,15 @@ func (m *machine) SSH(cmd string) ([]byte, []byte, error) {
return m.qc.SSH(m, cmd)
}

func (m *machine) IgnitionError() error {
_, err := m.inst.WaitIgnitionError()
if err != nil {
return err
}
// TODO render buf
return platform.ErrInitramfsEmergency
}

func (m *machine) Reboot() error {
return platform.RebootMachine(m, m.journal)
}
Expand Down
3 changes: 3 additions & 0 deletions mantle/platform/platform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type Machine interface {
// ID returns the plaform-specific machine identifier.
ID() string

// IgnitionError returns an error if the machine failed in Ignition
IgnitionError() error

// IP returns the machine's public IP.
IP() string

Expand Down
78 changes: 78 additions & 0 deletions mantle/platform/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package platform

import (
"bufio"
"encoding/json"
"fmt"
"io"
Expand All @@ -37,6 +38,10 @@ import (
"github.com/pkg/errors"
)

var (
ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs")
)

type MachineOptions struct {
AdditionalDisks []Disk
}
Expand All @@ -63,6 +68,8 @@ type QemuInstance struct {
swtpm exec.Cmd
tmpFiles []string
nbdServers []exec.Cmd

journalPipe *os.File
}

func (inst *QemuInstance) Pid() int {
Expand Down Expand Up @@ -127,6 +134,7 @@ func (inst *QemuInstance) SSHAddress() (string, error) {
return "", fmt.Errorf("didn't find an address")
}

// Wait for the qemu process to exit
func (inst *QemuInstance) Wait() error {
if inst.qemu != nil {
err := inst.qemu.Wait()
Expand All @@ -136,10 +144,69 @@ func (inst *QemuInstance) Wait() error {
return nil
}

// WaitIgnitionError will only return if the instance
// failed inside the initramfs. The resulting string will
// be a newline-delimited stream of JSON strings, as returned
// by `journalctl -o json`.
func (inst *QemuInstance) WaitIgnitionError() (string, error) {
b := bufio.NewReaderSize(inst.journalPipe, 64768)
var r strings.Builder
iscorrupted := false
_, err := b.Peek(1)
if err != nil {
if err == io.EOF {
return "", nil
}
return "", errors.Wrapf(err, "Reading from journal")
}
for {
line, prefix, err := b.ReadLine()
if err != nil {
return r.String(), errors.Wrapf(err, "Reading from journal channel")
}
if prefix {
iscorrupted = true
}
if len(line) == 0 || string(line) == "{}" {
break
}
r.Write(line)
}
if iscorrupted {
return r.String(), fmt.Errorf("journal was truncated due to overly long line")
}
return r.String(), nil
}

// WaitAll wraps the process exit as well as WaitIgnitionError,
// returning an error if either fail.
func (inst *QemuInstance) WaitAll() error {
Copy link
Member

@jlebon jlebon Apr 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, do we actually need this new interface? Can we have it part of Wait() directly and have that return ErrInitramfsEmergency if Ignition failed? That way we automatically get this check everywhere we use and will use Wait().

Copy link
Member Author

@cgwalters cgwalters Apr 3, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Funny you ask. I actually did it that way first. And then I went to go test out a FCOS build failing in the initramfs (which btw if you want to conveniently test, use e.g.
cosa run --kargs ignition.config.url=blah://) and qemu started, printed an error about the initramfs failure and then exited...so I couldn't debug it.

After that I made it a separate interface, because for that reason we don't always want to die if there's a failure in the initramfs. (Now, theoretically the caller could distinguish and explicitly call Wait() again but that's racy because there's already another goroutine doing that...)

c := make(chan error)
go func() {
buf, err := inst.WaitIgnitionError()
if err != nil {
c <- err
} else {
// TODO parse buf and try to nicely render something
if buf != "" {
c <- ErrInitramfsEmergency
}
}
}()
go func() {
c <- inst.Wait()
}()
return <-c
}

func (inst *QemuInstance) Destroy() {
for _, fN := range inst.tmpFiles {
os.RemoveAll(fN)
}
if inst.journalPipe != nil {
inst.journalPipe.Close()
inst.journalPipe = nil
}
// check if qemu is dead before trying to kill it
if inst.qemu != nil {
if err := inst.qemu.Kill(); err != nil {
Expand Down Expand Up @@ -885,6 +952,17 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
"-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0")
}

// Set up the virtio channel to get Ignition failures by default
journalPipeR, journalPipeW, err := os.Pipe()
if err != nil {
return nil, errors.Wrapf(err, "creating journal pipe")
}
inst.journalPipe = journalPipeR
argv = append(argv, "-device", "virtio-serial")
// https://www.redhat.com/archives/libvir-list/2015-December/msg00305.html
argv = append(argv, "-chardev", fmt.Sprintf("file,id=ignition-dracut,path=%s,append=on", builder.AddFd(journalPipeW)))
argv = append(argv, "-device", "virtserialport,chardev=ignition-dracut,name=com.coreos.ignition.journal")

fdnum := 3 // first additional file starts at position 3
for i, _ := range builder.fds {
fdset := i + 1 // Start at 1
Expand Down
Loading