diff --git a/build.sh b/build.sh index a867284126..2332987ef9 100755 --- a/build.sh +++ b/build.sh @@ -16,7 +16,6 @@ if [ $# -gt 1 ]; then echo " configure_yum_repos" echo " install_rpms" echo " make_and_makeinstall" - echo " install_tang" exit 1 fi @@ -75,10 +74,6 @@ install_rpms() { yum clean all } -install_tang() { - install -m 0755 -T "$srcdir"/src/tang/tangdw /usr/libexec/tangdw -} - make_and_makeinstall() { make && make install } @@ -123,5 +118,4 @@ else write_archive_info make_and_makeinstall configure_user - install_tang fi diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 6c66edd701..d0a2cd6d28 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -216,7 +216,10 @@ func runQemuExec(cmd *cobra.Command, args []string) error { builder.Processors = -1 } if usernet { - builder.EnableUsermodeNetworking(22) + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.EnableUsermodeNetworking(h) } builder.InheritConsole = true builder.Append(args...) diff --git a/mantle/kola/tests/rhcos/luks.go b/mantle/kola/tests/rhcos/luks.go index 86120b85a9..485d2b3d8e 100644 --- a/mantle/kola/tests/rhcos/luks.go +++ b/mantle/kola/tests/rhcos/luks.go @@ -1,37 +1,19 @@ package rhcos import ( - "bytes" "encoding/base64" "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "os/user" "regexp" - "strconv" - "sync" - "syscall" "time" "github.com/coreos/mantle/kola/cluster" "github.com/coreos/mantle/kola/register" + "github.com/coreos/mantle/platform" "github.com/coreos/mantle/platform/conf" + "github.com/coreos/mantle/platform/machine/unprivqemu" "github.com/coreos/mantle/util" ) -var runOnce sync.Once - -const ( - dbDir = "/tmp/tang/db" - cacheDir = "/tmp/tang/cache" - xinetdConfFile = "/tmp/tang/xinetd.conf" - xinetdPidFile = "/tmp/tang/pid" - tangLogFile = "/tmp/tang/tang.log" - xinetdLogFile = "/tmp/tang/xinetd.log" -) - func init() { register.RegisterTest(®ister.Test{ Run: luksTPMTest, @@ -68,7 +50,6 @@ func init() { Name: `rhcos.luks.tang`, Flags: []register.Flag{}, Distros: []string{"rhcos"}, - Platforms: []string{"qemu-unpriv"}, ExcludeArchitectures: []string{"s390x", "ppc64le"}, // no TPM support for s390x, ppc64le in qemu Tags: []string{"luks", "tang"}, }) @@ -94,143 +75,88 @@ func init() { }) } -func setupTangKeys(c cluster.TestCluster) { - runOnce.Do(func() { - user, err := user.Current() - if err != nil { - c.Fatalf("Unable to get current user: %v", err) - } +func setupTangMachine(c cluster.TestCluster) (string, string) { + var m platform.Machine + var err error + var thumbprint []byte + var tangAddress string - // xinetd requires the service to be in /etc/services which Tang is not - // included. Use the webcache service (port 8080) and run as the - // current user so we can run unprivileged - xinetdConf := fmt.Sprintf(`defaults -{ - log_type = SYSLOG daemon info - log_on_failure = HOST - log_on_success = PID HOST DURATION EXIT - - cps = 50 10 - instances = 50 - per_source = 10 - - v6only = no - bind = 0.0.0.0 - groups = yes - umask = 002 -} + options := platform.MachineOptions{ + HostForwardPorts: []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + {Service: "tang", HostPort: 0, GuestPort: 80}, + }, + } -service webcache -{ - server_args = %s /dev/null - server = /usr/libexec/tangdw - socket_type = stream - protocol = tcp - only_from = 10.0.2.15 127.0.0.1 - user = %s - wait = no -}`, cacheDir, user.Username) - if err := os.MkdirAll(cacheDir, 0755); err != nil { - c.Fatalf("Unable to create %s: %v", err) + ignition := conf.Ignition(`{ + "ignition": { + "version": "2.2.0" } + }`) - f, err := os.Open(cacheDir) - if err != nil { - c.Fatalf("Unable to open %s: %v", cacheDir, err) - } - // This is a simple check that the directory is empty - _, err = f.Readdir(1) - if err == io.EOF { - if err := os.MkdirAll(dbDir, 0755); err != nil { - c.Fatalf("Unable to create %s: %v", err) - } - err := ioutil.WriteFile(xinetdConfFile, []byte(xinetdConf), 0644) - if err != nil { - c.Fatalf("Unable to write xinetd configuration file: %v", err) - } - keygen := exec.Command("/usr/libexec/tangd-keygen", dbDir) - if err := keygen.Run(); err != nil { - c.Fatalf("Unable to generate Tang keys: %v", err) + switch pc := c.Cluster.(type) { + // These cases have to be separated because when put together to the same case statement + // the golang compiler no longer checks that the individual types in the case have the + // NewMachineWithOptions function, but rather whether platform.Cluster does which fails + case *unprivqemu.Cluster: + m, err = pc.NewMachineWithOptions(ignition, options, true) + for _, hfp := range options.HostForwardPorts { + if hfp.Service == "tang" { + tangAddress = fmt.Sprintf("10.0.2.2:%d", hfp.HostPort) } - update := exec.Command("/usr/libexec/tangd-update", dbDir, cacheDir) - if err := update.Run(); err != nil { - c.Fatalf("Unable to update Tang DB: %v", err) - } - } - }) -} - -func getEncodedTangPin(c cluster.TestCluster) string { - return b64Encode(getTangPin(c)) -} - -func startTang(c cluster.TestCluster) int { - if _, err := os.Stat(xinetdPidFile); err == nil { - if os.Remove(xinetdPidFile); err != nil { - c.Fatalf("Unable to delete %s: %v", xinetdPidFile, err) } + default: + m, err = pc.NewMachine(ignition) + tangAddress = fmt.Sprintf("%s:80", m.PrivateIP()) + } + if err != nil { + c.Fatal(err) } - cmd := exec.Command("/usr/sbin/xinetd", "-f", xinetdConfFile, "-pidfile", xinetdPidFile, "-filelog", xinetdLogFile) - // Give the xinetd child process a separate process group ID so it can be - // killed independently from the test - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - if err := cmd.Run(); err != nil { - c.Fatalf("Unable to start Tang: %v ", err) + // TODO: move container image to centralized namespace + // container source: https://github.com/mike-nguyen/tang-docker-container/ + containerID, _, err := m.SSH("sudo podman run -d -p 80:80 docker.io/mnguyenrh/tangd") + if err != nil { + c.Fatalf("Unable to start Tang container: %v", err) } - // xinetd detatches itself and os.Process.Pid reports incorrect pid - // Use pid file instead - if err := util.Retry(10, 2*time.Second, func() error { - _, err := os.Stat(xinetdPidFile) + // Wait a little bit for the container to start + if err := util.Retry(10, time.Second, func() error { + cmd := fmt.Sprintf("sudo podman exec %s /usr/bin/tang-show-keys", string(containerID)) + thumbprint, _, err = m.SSH(cmd) if err != nil { return err } + if string(thumbprint) == "" { + return fmt.Errorf("tang-show-keys returns nothing") + } return nil }); err != nil { - c.Fatalf("Cannot find pid file %v", err) - } - output, err := ioutil.ReadFile(xinetdPidFile) - if err != nil { - c.Fatalf("Unable to get pid %v", err) - } - - p := bytes.Trim(output, "\n") - pid, err := strconv.Atoi(string(p)) - if err != nil { - c.Fatalf("Unable to convert pid to integer: %v", err) + c.Fatalf("Unable to retrieve Tang keys: %v", err) } - return pid + return tangAddress, string(thumbprint) } -func stopTang(c cluster.TestCluster, pid int) { - // kill with a negative pid is killing the process group - if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil { - c.Fatalf("Unable to stop xinetd: %v", err) - } +func getEncodedTangPin(c cluster.TestCluster, address string, thumbprint string) string { + return b64Encode(getTangPin(c, address, thumbprint)) } -func getTangPin(c cluster.TestCluster) string { - tangThumbprint, err := exec.Command("tang-show-keys", "8080").Output() - if err != nil { - c.Fatalf("Unable to retrieve Tang thumbprint: %v", err) - } - +func getTangPin(c cluster.TestCluster, address string, thumbprint string) string { return fmt.Sprintf(`{ - "url": "http://10.0.2.2:8080", + "url": "http://%s", "thp": "%s" -}`, tangThumbprint) +}`, address, string(thumbprint)) } // Generates a SSS clevis pin with TPM2 and a valid/invalid Tang config -func getEncodedSSSPin(c cluster.TestCluster, num int, tang bool) string { - tangPin := getTangPin(c) +func getEncodedSSSPin(c cluster.TestCluster, num int, tang bool, address string, thumbprint string) string { + tangPin := getTangPin(c, address, thumbprint) if !tang { tangPin = fmt.Sprintf(`{ - "url": "http://10.0.2.2:8080", + "url": "http://%s", "thp": "INVALIDTHUMBPRINT" -}`) +}`, address) } return b64Encode(fmt.Sprintf(`{ @@ -266,8 +192,7 @@ func mustNotMatch(c cluster.TestCluster, r string, output []byte) { } } -func luksSanityTest(c cluster.TestCluster, pin string) { - m := c.Machines()[0] +func luksSanityTest(c cluster.TestCluster, m platform.Machine, pin string) { luksDump := c.MustSSH(m, "sudo cryptsetup luksDump /dev/disk/by-partlabel/luks_root") // Yes, some hacky regexps. There is luksDump --debug-json but we'd have to massage the JSON // out of other debug output and it's not clear to me it's going to be more stable. @@ -289,16 +214,14 @@ func luksSanityTest(c cluster.TestCluster, pin string) { // Verify that the rootfs is encrypted with the TPM func luksTPMTest(c cluster.TestCluster) { - luksSanityTest(c, "tpm2") + luksSanityTest(c, c.Machines()[0], "tpm2") } // Verify that the rootfs is encrypted with Tang func luksTangTest(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedTangPin := getEncodedTangPin(c) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedTangPin := getEncodedTangPin(c, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -315,17 +238,17 @@ func luksTangTest(c cluster.TestCluster) { ] } }`, encodedTangPin))) - luksSanityTest(c, "tang") - + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "tang") } // Verify that the rootfs is encrypted with SSS with t=1 func luksSSST1Test(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedSSST1Pin := getEncodedSSSPin(c, 1, false) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedSSST1Pin := getEncodedSSSPin(c, 1, false, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -342,16 +265,17 @@ func luksSSST1Test(c cluster.TestCluster) { ] } }`, encodedSSST1Pin))) - luksSanityTest(c, "sss") + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "sss") } // Verify that the rootfs is encrypted with SSS with t=2 func luksSSST2Test(c cluster.TestCluster) { - setupTangKeys(c) - pid := startTang(c) - defer stopTang(c, pid) - encodedSSST2Pin := getEncodedSSSPin(c, 2, true) - c.NewMachine(conf.Ignition(fmt.Sprintf(`{ + address, thumbprint := setupTangMachine(c) + encodedSSST2Pin := getEncodedSSSPin(c, 2, true, address, thumbprint) + m, err := c.NewMachine(conf.Ignition(fmt.Sprintf(`{ "ignition": { "version": "2.2.0" }, @@ -368,5 +292,8 @@ func luksSSST2Test(c cluster.TestCluster) { ] } }`, encodedSSST2Pin))) - luksSanityTest(c, "sss") + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + luksSanityTest(c, m, "sss") } diff --git a/mantle/platform/machine/unprivqemu/cluster.go b/mantle/platform/machine/unprivqemu/cluster.go index 77a99946fe..faee2c2413 100644 --- a/mantle/platform/machine/unprivqemu/cluster.go +++ b/mantle/platform/machine/unprivqemu/cluster.go @@ -123,7 +123,15 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, errors.Wrapf(err, "adding additional disk") } } - builder.EnableUsermodeNetworking(22) + + if len(options.HostForwardPorts) > 0 { + builder.EnableUsermodeNetworking(options.HostForwardPorts) + } else { + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.EnableUsermodeNetworking(h) + } inst, err := builder.Exec() if err != nil { diff --git a/mantle/platform/platform.go b/mantle/platform/platform.go index aca335fc3a..af0bf717f4 100644 --- a/mantle/platform/platform.go +++ b/mantle/platform/platform.go @@ -35,7 +35,8 @@ import ( ) const ( - sshRetries = 30 + // Encryption takes a long time--retry more before failing + sshRetries = 60 sshTimeout = 10 * time.Second ) diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index e0b30ed9e9..584b562a16 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -20,10 +20,9 @@ import ( "io" "io/ioutil" "math/rand" + "net" "os" "path/filepath" - "regexp" - "strconv" "strings" "syscall" "time" @@ -33,7 +32,6 @@ import ( v3types "github.com/coreos/ignition/v2/config/v3_0/types" "github.com/coreos/mantle/system" "github.com/coreos/mantle/system/exec" - "github.com/coreos/mantle/util" "github.com/pkg/errors" ) @@ -41,8 +39,15 @@ var ( ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") ) +type HostForwardPort struct { + Service string + HostPort int + GuestPort int +} + type MachineOptions struct { - AdditionalDisks []Disk + AdditionalDisks []Disk + HostForwardPorts []HostForwardPort } type Disk struct { @@ -61,11 +66,12 @@ type Disk struct { } type QemuInstance struct { - qemu exec.Cmd - tmpConfig string - tempdir string - swtpm exec.Cmd - nbdServers []exec.Cmd + qemu exec.Cmd + tmpConfig string + tempdir string + swtpm exec.Cmd + nbdServers []exec.Cmd + hostForwardedPorts []HostForwardPort journalPipe *os.File } @@ -74,59 +80,11 @@ func (inst *QemuInstance) Pid() int { return inst.qemu.Pid() } -// parse /proc/net/tcp to determine the port selected by QEMU +// Get the IP address with the forwarded port func (inst *QemuInstance) SSHAddress() (string, error) { - pid := fmt.Sprintf("%d", inst.Pid()) - data, err := ioutil.ReadFile("/proc/net/tcp") - if err != nil { - return "", errors.Wrap(err, "reading /proc/net/tcp") - } - - for _, line := range strings.Split(string(data), "\n")[1:] { - fields := strings.Fields(line) - if len(fields) < 10 { - // at least 10 fields are neeeded for the local & remote address and the inode - continue - } - localAddress := fields[1] - remoteAddress := fields[2] - inode := fields[9] - - var isLocalPat *regexp.Regexp - if util.HostEndianness == util.LITTLE { - isLocalPat = regexp.MustCompile("0100007F:[[:xdigit:]]{4}") - } else { - isLocalPat = regexp.MustCompile("7F000001:[[:xdigit:]]{4}") - } - - if !isLocalPat.MatchString(localAddress) || remoteAddress != "00000000:0000" { - continue - } - - dir := fmt.Sprintf("/proc/%s/fd/", pid) - fds, err := ioutil.ReadDir(dir) - if err != nil { - return "", fmt.Errorf("listing %s: %v", dir, err) - } - - for _, f := range fds { - link, err := os.Readlink(filepath.Join(dir, f.Name())) - if err != nil { - continue - } - socketPattern := regexp.MustCompile("socket:\\[([0-9]+)\\]") - match := socketPattern.FindStringSubmatch(link) - if len(match) > 1 { - if inode == match[1] { - // this entry belongs to the QEMU pid, parse the port and return the address - portHex := strings.Split(localAddress, ":")[1] - port, err := strconv.ParseInt(portHex, 16, 32) - if err != nil { - return "", errors.Wrapf(err, "decoding port %q", portHex) - } - return fmt.Sprintf("127.0.0.1:%d", port), nil - } - } + for _, fwdPorts := range inst.hostForwardedPorts { + if fwdPorts.Service == "ssh" { + return fmt.Sprintf("127.0.0.1:%d", fwdPorts.HostPort), nil } } return "", fmt.Errorf("didn't find an address") @@ -275,6 +233,9 @@ type QemuBuilder struct { ignitionSet bool ignitionRendered bool + UsermodeNetworking bool + requestedHostForwardPorts []HostForwardPort + finalized bool diskId uint disks []*Disk @@ -375,10 +336,26 @@ func (builder *QemuBuilder) ConsoleToFile(path string) { builder.Append("-display", "none", "-chardev", "file,id=log,path="+path, "-serial", "chardev:log") } -func (builder *QemuBuilder) EnableUsermodeNetworking(forwardedPort uint) { +func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) { + builder.UsermodeNetworking = true + builder.requestedHostForwardPorts = h +} + +func (builder *QemuBuilder) usermodeNetworkingAssignPorts() error { netdev := "user,id=eth0" - if forwardedPort != 0 { - netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:0-:%d", forwardedPort) + for i := range builder.requestedHostForwardPorts { + address := fmt.Sprintf(":%d", builder.requestedHostForwardPorts[i].HostPort) + // Possible race condition between getting the port here and using it + // with qemu -- trade off for simpler port management + l, err := net.Listen("tcp", address) + if err != nil { + return err + } + l.Close() + builder.requestedHostForwardPorts[i].HostPort = l.Addr().(*net.TCPAddr).Port + netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:%d-:%d", + builder.requestedHostForwardPorts[i].HostPort, + builder.requestedHostForwardPorts[i].GuestPort) } if builder.Hostname != "" { @@ -386,6 +363,7 @@ func (builder *QemuBuilder) EnableUsermodeNetworking(forwardedPort uint) { } builder.Append("-netdev", netdev, "-device", virtio("net", "netdev=eth0")) + return nil } // Mount9p sets up a mount point from the host to guest. To be replaced @@ -986,6 +964,14 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } } + // Handle Usermode Networking + if builder.UsermodeNetworking { + if err := builder.usermodeNetworkingAssignPorts(); err != nil { + return nil, err + } + inst.hostForwardedPorts = builder.requestedHostForwardPorts + } + // Handle Software TPM if builder.Swtpm && builder.supportsSwtpm() { swtpmSock := filepath.Join(builder.tempdir, "swtpm-sock") diff --git a/src/deps.txt b/src/deps.txt index 4cff7521d9..8c2a894d5c 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -58,9 +58,6 @@ python3-flake8 python3-pytest python3-pytest-cov pylint # For cmd-virt-install python3-libvirt -# For encryption tests -tang xinetd - # Support for Koji uploads. krb5-libs krb5-workstation koji-utils python3-koji python3-koji-cli-plugins diff --git a/src/tang/tangdw b/src/tang/tangdw deleted file mode 100755 index 3ddff975df..0000000000 --- a/src/tang/tangdw +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -/usr/libexec/tangd $1 2>>$2