From 41f2d0fc62fff17b31d03f1db49bd1711509eb03 Mon Sep 17 00:00:00 2001 From: Michael Nguyen Date: Wed, 22 Apr 2020 13:04:56 -0400 Subject: [PATCH 1/2] platform/qemu: Enable forwarding multiple ports One of the limitations of unprivileged qemu is there is no machine to machine networking. Host port forwarding allows the host machine to communicate to the guest machine through the local host address. It also allows the guest machine to communicate with the host machine through the gateway address. With two guest machines, we can bridge the two guests through the host machine with port forwarding. For example, Host A can run a webserver on port 8080 and Host B can access Host A's webserver through its gateway (host machine) address http://10.0.2.2:8080. This is useful for creating test services (such as tang) for use by the test machine. --- mantle/cmd/kola/qemuexec.go | 5 +- mantle/platform/machine/unprivqemu/cluster.go | 10 +- mantle/platform/qemu.go | 114 ++++++++---------- 3 files changed, 63 insertions(+), 66 deletions(-) diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 6c66edd701..d0a2cd6d28 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -216,7 +216,10 @@ func runQemuExec(cmd *cobra.Command, args []string) error { builder.Processors = -1 } if usernet { - builder.EnableUsermodeNetworking(22) + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.EnableUsermodeNetworking(h) } builder.InheritConsole = true builder.Append(args...) diff --git a/mantle/platform/machine/unprivqemu/cluster.go b/mantle/platform/machine/unprivqemu/cluster.go index 77a99946fe..faee2c2413 100644 --- a/mantle/platform/machine/unprivqemu/cluster.go +++ b/mantle/platform/machine/unprivqemu/cluster.go @@ -123,7 +123,15 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, errors.Wrapf(err, "adding additional disk") } } - builder.EnableUsermodeNetworking(22) + + if len(options.HostForwardPorts) > 0 { + builder.EnableUsermodeNetworking(options.HostForwardPorts) + } else { + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.EnableUsermodeNetworking(h) + } inst, err := builder.Exec() if err != nil { diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index e0b30ed9e9..584b562a16 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -20,10 +20,9 @@ import ( "io" "io/ioutil" "math/rand" + "net" "os" "path/filepath" - "regexp" - "strconv" "strings" "syscall" "time" @@ -33,7 +32,6 @@ import ( v3types "github.com/coreos/ignition/v2/config/v3_0/types" "github.com/coreos/mantle/system" "github.com/coreos/mantle/system/exec" - "github.com/coreos/mantle/util" "github.com/pkg/errors" ) @@ -41,8 +39,15 @@ var ( ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") ) +type HostForwardPort struct { + Service string + HostPort int + GuestPort int +} + type MachineOptions struct { - AdditionalDisks []Disk + AdditionalDisks []Disk + HostForwardPorts []HostForwardPort } type Disk struct { @@ -61,11 +66,12 @@ type Disk struct { } type QemuInstance struct { - qemu exec.Cmd - tmpConfig string - tempdir string - swtpm exec.Cmd - nbdServers []exec.Cmd + qemu exec.Cmd + tmpConfig string + tempdir string + swtpm exec.Cmd + nbdServers []exec.Cmd + hostForwardedPorts []HostForwardPort journalPipe *os.File } @@ -74,59 +80,11 @@ func (inst *QemuInstance) Pid() int { return inst.qemu.Pid() } -// parse /proc/net/tcp to determine the port selected by QEMU +// Get the IP address with the forwarded port func (inst *QemuInstance) SSHAddress() (string, error) { - pid := fmt.Sprintf("%d", inst.Pid()) - data, err := ioutil.ReadFile("/proc/net/tcp") - if err != nil { - return "", errors.Wrap(err, "reading /proc/net/tcp") - } - - for _, line := range strings.Split(string(data), "\n")[1:] { - fields := strings.Fields(line) - if len(fields) < 10 { - // at least 10 fields are neeeded for the local & remote address and the inode - continue - } - localAddress := fields[1] - remoteAddress := fields[2] - inode := fields[9] - - var isLocalPat *regexp.Regexp - if util.HostEndianness == util.LITTLE { - isLocalPat = regexp.MustCompile("0100007F:[[:xdigit:]]{4}") - } else { - isLocalPat = regexp.MustCompile("7F000001:[[:xdigit:]]{4}") - } - - if !isLocalPat.MatchString(localAddress) || remoteAddress != "00000000:0000" { - continue - } - - dir := fmt.Sprintf("/proc/%s/fd/", pid) - fds, err := ioutil.ReadDir(dir) - if err != nil { - return "", fmt.Errorf("listing %s: %v", dir, err) - } - - for _, f := range fds { - link, err := os.Readlink(filepath.Join(dir, f.Name())) - if err != nil { - continue - } - socketPattern := regexp.MustCompile("socket:\\[([0-9]+)\\]") - match := socketPattern.FindStringSubmatch(link) - if len(match) > 1 { - if inode == match[1] { - // this entry belongs to the QEMU pid, parse the port and return the address - portHex := strings.Split(localAddress, ":")[1] - port, err := strconv.ParseInt(portHex, 16, 32) - if err != nil { - return "", errors.Wrapf(err, "decoding port %q", portHex) - } - return fmt.Sprintf("127.0.0.1:%d", port), nil - } - } + for _, fwdPorts := range inst.hostForwardedPorts { + if fwdPorts.Service == "ssh" { + return fmt.Sprintf("127.0.0.1:%d", fwdPorts.HostPort), nil } } return "", fmt.Errorf("didn't find an address") @@ -275,6 +233,9 @@ type QemuBuilder struct { ignitionSet bool ignitionRendered bool + UsermodeNetworking bool + requestedHostForwardPorts []HostForwardPort + finalized bool diskId uint disks []*Disk @@ -375,10 +336,26 @@ func (builder *QemuBuilder) ConsoleToFile(path string) { builder.Append("-display", "none", "-chardev", "file,id=log,path="+path, "-serial", "chardev:log") } -func (builder *QemuBuilder) EnableUsermodeNetworking(forwardedPort uint) { +func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) { + builder.UsermodeNetworking = true + builder.requestedHostForwardPorts = h +} + +func (builder *QemuBuilder) usermodeNetworkingAssignPorts() error { netdev := "user,id=eth0" - if forwardedPort != 0 { - netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:0-:%d", forwardedPort) + for i := range builder.requestedHostForwardPorts { + address := fmt.Sprintf(":%d", builder.requestedHostForwardPorts[i].HostPort) + // Possible race condition between getting the port here and using it + // with qemu -- trade off for simpler port management + l, err := net.Listen("tcp", address) + if err != nil { + return err + } + l.Close() + builder.requestedHostForwardPorts[i].HostPort = l.Addr().(*net.TCPAddr).Port + netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:%d-:%d", + builder.requestedHostForwardPorts[i].HostPort, + builder.requestedHostForwardPorts[i].GuestPort) } if builder.Hostname != "" { @@ -386,6 +363,7 @@ func (builder *QemuBuilder) EnableUsermodeNetworking(forwardedPort uint) { } builder.Append("-netdev", netdev, "-device", virtio("net", "netdev=eth0")) + return nil } // Mount9p sets up a mount point from the host to guest. To be replaced @@ -986,6 +964,14 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } } + // Handle Usermode Networking + if builder.UsermodeNetworking { + if err := builder.usermodeNetworkingAssignPorts(); err != nil { + return nil, err + } + inst.hostForwardedPorts = builder.requestedHostForwardPorts + } + // Handle Software TPM if builder.Swtpm && builder.supportsSwtpm() { swtpmSock := filepath.Join(builder.tempdir, "swtpm-sock") From 07cf06a350379fa15f418af3250519f4ec2d8e75 Mon Sep 17 00:00:00 2001 From: Michael Nguyen Date: Mon, 27 Apr 2020 15:19:54 -0400 Subject: [PATCH 2/2] kola/tests/luks: Use separate machine as Tang server With multiple host port forwarding enabled and machine to machine communication possible, Tang is no longer needed in coreos-assembler. The Tang server is created as a container running inside a machine as a part of the test. --- build.sh | 6 - mantle/kola/tests/rhcos/luks.go | 229 +++++++++++--------------------- mantle/platform/platform.go | 3 +- src/deps.txt | 3 - src/tang/tangdw | 2 - 5 files changed, 80 insertions(+), 163 deletions(-) delete mode 100755 src/tang/tangdw diff --git a/build.sh b/build.sh index a867284126..2332987ef9 100755 --- a/build.sh +++ b/build.sh @@ -16,7 +16,6 @@ if [ $# -gt 1 ]; then echo " configure_yum_repos" echo " install_rpms" echo " make_and_makeinstall" - echo " install_tang" exit 1 fi @@ -75,10 +74,6 @@ install_rpms() { yum clean all } -install_tang() { - install -m 0755 -T "$srcdir"/src/tang/tangdw /usr/libexec/tangdw -} - make_and_makeinstall() { make && make install } @@ -123,5 +118,4 @@ else write_archive_info make_and_makeinstall configure_user - install_tang fi diff --git a/mantle/kola/tests/rhcos/luks.go b/mantle/kola/tests/rhcos/luks.go index 86120b85a9..485d2b3d8e 100644 --- a/mantle/kola/tests/rhcos/luks.go +++ b/mantle/kola/tests/rhcos/luks.go @@ -1,37 +1,19 @@ package rhcos import ( - "bytes" "encoding/base64" "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "os/user" "regexp" - "strconv" - "sync" - "syscall" "time" "github.com/coreos/mantle/kola/cluster" "github.com/coreos/mantle/kola/register" + "github.com/coreos/mantle/platform" "github.com/coreos/mantle/platform/conf" + "github.com/coreos/mantle/platform/machine/unprivqemu" "github.com/coreos/mantle/util" ) -var runOnce sync.Once - -const ( - dbDir = "/tmp/tang/db" - cacheDir = "/tmp/tang/cache" - xinetdConfFile = "/tmp/tang/xinetd.conf" - xinetdPidFile = "/tmp/tang/pid" - tangLogFile = "/tmp/tang/tang.log" - xinetdLogFile = "/tmp/tang/xinetd.log" -) - func init() { register.RegisterTest(®ister.Test{ Run: luksTPMTest, @@ -68,7 +50,6 @@ func init() { Name: `rhcos.luks.tang`, Flags: []register.Flag{}, Distros: []string{"rhcos"}, - Platforms: []string{"qemu-unpriv"}, ExcludeArchitectures: []string{"s390x", "ppc64le"}, // no TPM support for s390x, ppc64le in qemu Tags: []string{"luks", "tang"}, }) @@ -94,143 +75,88 @@ func init() { }) } -func setupTangKeys(c cluster.TestCluster) { - runOnce.Do(func() { - user, err := user.Current() - if err != nil { - c.Fatalf("Unable to get current user: %v", err) - } +func setupTangMachine(c cluster.TestCluster) (string, string) { + var m platform.Machine + var err error + var thumbprint []byte + var tangAddress string - // xinetd requires the service to be in /etc/services which Tang is not - // included. Use the webcache service (port 8080) and run as the - // current user so we can run unprivileged - xinetdConf := fmt.Sprintf(`defaults -{ - log_type = SYSLOG daemon info - log_on_failure = HOST - log_on_success = PID HOST DURATION EXIT - - cps = 50 10 - instances = 50 - per_source = 10 - - v6only = no - bind = 0.0.0.0 - groups = yes - umask = 002 -} + options := platform.MachineOptions{ + HostForwardPorts: []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + {Service: "tang", HostPort: 0, GuestPort: 80}, + }, + } -service webcache -{ - server_args = %s /dev/null - server = /usr/libexec/tangdw - socket_type = stream - protocol = tcp - only_from = 10.0.2.15 127.0.0.1 - user = %s - wait = no -}`, cacheDir, user.Username) - if err := os.MkdirAll(cacheDir, 0755); err != nil { - c.Fatalf("Unable to create %s: %v", err) + ignition := conf.Ignition(`{ + "ignition": { + "version": "2.2.0" } + }`) - f, err := os.Open(cacheDir) - if err != nil { - c.Fatalf("Unable to open %s: %v", cacheDir, err) - } - // This is a simple check that the directory is empty - _, err = f.Readdir(1) - if err == io.EOF { - if err := os.MkdirAll(dbDir, 0755); err != nil { - c.Fatalf("Unable to create %s: %v", err) - } - err := ioutil.WriteFile(xinetdConfFile, []byte(xinetdConf), 0644) - if err != nil { - c.Fatalf("Unable to write xinetd configuration file: %v", err) - } - keygen := exec.Command("/usr/libexec/tangd-keygen", dbDir) - if err := keygen.Run(); err != nil { - c.Fatalf("Unable to generate Tang keys: %v", err) + switch pc := c.Cluster.(type) { + // These cases have to be separated because when put together to the same case statement + // the golang compiler no longer checks that the individual types in the case have the + // NewMachineWithOptions function, but rather whether platform.Cluster does which fails + case *unprivqemu.Cluster: + m, err = pc.NewMachineWithOptions(ignition, options, true) + for _, hfp := range options.HostForwardPorts { + if hfp.Service == "tang" { + tangAddress = fmt.Sprintf("10.0.2.2:%d", hfp.HostPort) } - update := exec.Command("/usr/libexec/tangd-update", dbDir, cacheDir) - if err := update.Run(); err != nil { - c.Fatalf("Unable to update Tang DB: %v", err) - } - } - }) -} - -func getEncodedTangPin(c cluster.TestCluster) string { - return b64Encode(getTangPin(c)) -} - -func startTang(c cluster.TestCluster) int { - if _, err := os.Stat(xinetdPidFile); err == nil { - if os.Remove(xinetdPidFile); err != nil { - c.Fatalf("Unable to delete %s: %v", xinetdPidFile, err) } + default: + m, err = pc.NewMachine(ignition) + tangAddress = fmt.Sprintf("%s:80", m.PrivateIP()) + } + if err != nil { + c.Fatal(err) } - cmd := exec.Command("/usr/sbin/xinetd", "-f", xinetdConfFile, "-pidfile", xinetdPidFile, "-filelog", xinetdLogFile) - // Give the xinetd child process a separate process group ID so it can be - // killed independently from the test - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - if err := cmd.Run(); err != nil { - c.Fatalf("Unable to start Tang: %v ", err) + // TODO: move container image to centralized namespace + // container source: https://github.com/mike-nguyen/tang-docker-container/ + containerID, _, err := m.SSH("sudo podman run -d -p 80:80 docker.io/mnguyenrh/tangd") + if err != nil { + c.Fatalf("Unable to start Tang container: %v", err) } - // xinetd detatches itself and os.Process.Pid reports incorrect pid - // Use pid file instead - if err := util.Retry(10, 2*time.Second, func() error { - _, err := os.Stat(xinetdPidFile) + // Wait a little bit for the container to start + if err := util.Retry(10, time.Second, func() error { + cmd := fmt.Sprintf("sudo podman exec %s /usr/bin/tang-show-keys", string(containerID)) + thumbprint, _, err = m.SSH(cmd) if err != nil { return err } + if string(thumbprint) == "" { + return fmt.Errorf("tang-show-keys returns nothing") + } return nil }); err != nil { - c.Fatalf("Cannot find pid file %v", err) - } - output, err := ioutil.ReadFile(xinetdPidFile) - if err != nil { - c.Fatalf("Unable to get pid %v", err) - } - - p := bytes.Trim(output, "\n") - pid, err := strconv.Atoi(string(p)) - if err != nil { - c.Fatalf("Unable to convert pid to integer: %v", err) + c.Fatalf("Unable to retrieve Tang keys: %v", err) } - return pid + return tangAddress, string(thumbprint) } -func stopTang(c cluster.TestCluster, pid int) { - // kill with a negative pid is killing the process group - if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil { - c.Fatalf("Unable to stop xinetd: %v", err) - } +func getEncodedTangPin(c cluster.TestCluster, address string, thumbprint string) string { + return b64Encode(getTangPin(c, address, thumbprint)) } -func getTangPin(c cluster.TestCluster) string { - tangThumbprint, err := exec.Command("tang-show-keys", "8080").Output() - if err != nil { - c.Fatalf("Unable to retrieve Tang thumbprint: %v", err) - } - +func getTangPin(c cluster.TestCluster, address string, thumbprint string) string { return fmt.Sprintf(`{ - "url": "http://10.0.2.2:8080", + "url": "http://%s", "thp": "%s" -}`, tangThumbprint) +}`, address, string(thumbprint)) } // Generates a SSS clevis pin with TPM2 and a valid/invalid Tang config -func getEncodedSSSPin(c cluster.TestCluster, num int, tang bool) string { - tangPin := getTangPin(c) +func getEncodedSSSPin(c cluster.TestCluster, num int, tang bool, address string, thumbprint string) string { + tangPin := getTangPin(c, address, thumbprint) if !tang { tangPin = fmt.Sprintf(`{ - "url": "http://10.0.2.2:8080", + "url": "http://%s", "thp": "INVALIDTHUMBPRINT" -}`) +}`, address) } return b64Encode(fmt.Sprintf(`{ @@ -266,8 +192,7 @@ func mustNotMatch(c cluster.TestCluster, r string, output []byte) { } } -func luksSanityTest(c cluster.TestCluster, pin string) { - m := c.Machines()[0] +func luksSanityTest(c cluster.TestCluster, m platform.Machine, pin string) { luksDump := c.MustSSH(m, "sudo cryptsetup luksDump /dev/disk/by-partlabel/luks_root") // Yes, some hacky regexps. There is luksDump --debug-json but we'd have to massage the JSON // out of other debug output and it's not clear to me it's going to be more stable. @@ -289,16 +214,14 @@ func luksSanityTest(c cluster.TestCluster, pin string) { // Verify that the rootfs is encrypted with the TPM func luksTPMTest(c cluster.TestCluster) { - luksSanityTest(c, "tpm2") + luksSanityTest(c, c.Machines()[0], "tpm2") } // Verify that the rootfs is encrypted with Tang func luksTangTest(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedTangPin := getEncodedTangPin(c) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedTangPin := getEncodedTangPin(c, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -315,17 +238,17 @@ func luksTangTest(c cluster.TestCluster) { ] } }`, encodedTangPin))) - luksSanityTest(c, "tang") - + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "tang") } // Verify that the rootfs is encrypted with SSS with t=1 func luksSSST1Test(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedSSST1Pin := getEncodedSSSPin(c, 1, false) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedSSST1Pin := getEncodedSSSPin(c, 1, false, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -342,16 +265,17 @@ func luksSSST1Test(c cluster.TestCluster) { ] } }`, encodedSSST1Pin))) - luksSanityTest(c, "sss") + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "sss") } // Verify that the rootfs is encrypted with SSS with t=2 func luksSSST2Test(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedSSST2Pin := getEncodedSSSPin(c, 2, true) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedSSST2Pin := getEncodedSSSPin(c, 2, true, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -368,5 +292,8 @@ func luksSSST2Test(c cluster.TestCluster) { ] } }`, encodedSSST2Pin))) - luksSanityTest(c, "sss") + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "sss") } diff --git a/mantle/platform/platform.go b/mantle/platform/platform.go index aca335fc3a..af0bf717f4 100644 --- a/mantle/platform/platform.go +++ b/mantle/platform/platform.go @@ -35,7 +35,8 @@ import ( ) const ( - sshRetries = 30 + // Encryption takes a long time--retry more before failing + sshRetries = 60 sshTimeout = 10 * time.Second ) diff --git a/src/deps.txt b/src/deps.txt index 4cff7521d9..8c2a894d5c 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -58,9 +58,6 @@ python3-flake8 python3-pytest python3-pytest-cov pylint # For cmd-virt-install python3-libvirt -# For encryption tests -tang xinetd - # Support for Koji uploads. krb5-libs krb5-workstation koji-utils python3-koji python3-koji-cli-plugins diff --git a/src/tang/tangdw b/src/tang/tangdw deleted file mode 100755 index 3ddff975df..0000000000 --- a/src/tang/tangdw +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -/usr/libexec/tangd $1 2>>$2