env/linux-x86-vmx: add new Debian host that's like Container-Optimize…

…d OS + vmx

This adds scripts to create a new builder host image that acts like
Container-Optimized OS (has docker, runs konlet on startup) but with a
Debian 9 kernel + userspace that permits KVM for nested
virtualization.

Updates golang/go#15581 (solaris)
Updates golang/go#23060 (dragonfly)
Updates golang/go#30262 (riscv)
Updates golang/go#30267 (fuchsia)
Updates golang/go#23824 (android)

Change-Id: Ib1d3a250556703856083c222be2a70c4e8d91884
Reviewed-on: https://go-review.googlesource.com/c/163301
Reviewed-by: Dmitri Shuralyov <[email protected]>

buildlet/gce.go

@@ -146,16 +146,7 @@ func StartNewVM(creds *google.Credentials, buildEnv *buildenv.Environment, instN
 	if hconf.IsContainer() {
 		if hconf.NestedVirt {
 			minCPU = "Intel Haswell" // documented minimum from https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances
-			// TODO: use some variant of cosImage that finds our local
-			// forked copy of cos-stable with the VMX license added. For
-			// now, I just manually once ran:
-			//   gcloud compute images create cos-stable-72-11316-136-0-vmx --source-image=cos-stable-72-11316-136-0 --source-image-project=cos-cloud --licenses=https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx
-			// And we'll use that version for now. Perhaps when Nested
-			// Virtualization reaches GA it'll just become a boolean we
-			// can set in our compute.Instance creation request and this
-			// license opt-in mechanism will be unnecessary.
-			const cosVMXImage = "cros-kvm-2"
-			srcImage = "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + cosVMXImage
+			srcImage = "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + hconf.ContainerVMImage()
 		} else {
 			var err error
 			srcImage, err = cosImage(ctx, computeService)

cmd/debugnewvm/debugnewvm.go

@@ -79,9 +79,9 @@ func main() {
 			hconf.VMImage = img
 		}
 	}
-	vmImageSummary := hconf.VMImage
+	vmImageSummary := fmt.Sprintf("%q", hconf.VMImage)
 	if hconf.IsContainer() {
-		vmImageSummary = fmt.Sprintf("cos-stable, running %v", hconf.ContainerImage)
+		vmImageSummary = fmt.Sprintf("%q, running container %q", hconf.ContainerVMImage(), hconf.ContainerImage)
 	}
 
 	env = buildenv.FromFlags()
@@ -96,7 +96,7 @@ func main() {
 
 	name := fmt.Sprintf("debug-temp-%d", time.Now().Unix())
 
-	log.Printf("Creating %s (with VM image %q)", name, vmImageSummary)
+	log.Printf("Creating %s (with VM image %s)", name, vmImageSummary)
 	bc, err := buildlet.StartNewVM(creds, env, name, *hostType, buildlet.VMOpts{
 		OnInstanceRequested: func() { log.Printf("instance requested") },
 		OnInstanceCreated: func() {

dashboard/builders.go

@@ -585,7 +585,8 @@ type HostConfig struct {
 	HermeticReverse bool // whether reverse buildlet has fresh env per conn
 
 	// Container image options, if ContainerImage != "":
-	NestedVirt bool // container requires VMX nested virtualization
+	NestedVirt    bool   // container requires VMX nested virtualization
+	KonletVMImage string // optional VM image (containing konlet) to use instead of default
 
 	// Optional base env. GOROOT_BOOTSTRAP should go here if the buildlet
 	// has Go 1.4+ baked in somewhere.
@@ -1050,6 +1051,17 @@ func (c *HostConfig) PoolName() string {
 	panic("unknown builder type")
 }
 
+// ContainerVMImage returns the base VM name (not the fully qualified
+// URL resource name of the VM) that starts the konlet program that
+// pulls & runs a container. This method is only applicable when
+// c.IsContainer() is true.
+func (c *HostConfig) ContainerVMImage() string {
+	if c.KonletVMImage != "" {
+		return c.KonletVMImage
+	}
+	return "debian-stretch-vmx"
+}
+
 // IsHermetic reports whether this host config gets a fresh
 // environment (including /usr, /var, etc) for each execution. This is
 // true for VMs, GKE, and reverse buildlets running their containers

env/linux-x86-vmx/README.md

@@ -0,0 +1,6 @@
+# linux-x86-vmx
+
+These scripts create a GCE VM image that acts like Container-Optimized
+Linux but uses a Debian 9 (Stretch) kernel + userspace instead. We do
+this because Debian 9 includes CONFIG_KVM for nested virtualization,
+whereas that's not compiled in for Container-Optimized Linux.

env/linux-x86-vmx/create.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+# Copyright 2019 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This creates the debian-stretch-vmx buildlet VM that's
+# like the Container-Optimized OS but using Debian Stretch
+# instead of the Chromium OS, and with nested virtualization
+# enabled.
+
+set -e
+set -x
+
+ZONE=us-central1-f
+TARGET_IMAGE=debian-stretch-vmx
+
+TMP_DISK=dev-debian-vmx-tmpdisk
+TMP_IMG=dev-debian-vmx-image
+TMP_VM=dev-debian-vmx
+
+# Create disk, forking Debian 9 (Stretch).
+gcloud compute disks delete $TMP_DISK --zone=$ZONE --quiet || true
+gcloud compute disks create $TMP_DISK \
+       --zone=$ZONE \
+       --size=20GB \
+       --image-project=debian-cloud \
+       --image-family debian-9
+
+# Create image based on that disk, with the nested virtualization
+# opt-in flag ("license").
+gcloud compute images delete $TMP_IMG --quiet || true
+gcloud compute images create \
+       $TMP_IMG \
+       --source-disk=$TMP_DISK \
+       --source-disk-zone=$ZONE \
+       --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
+
+# No longer need that temp disk:
+gcloud compute disks delete $TMP_DISK --zone=$ZONE --quiet
+
+# Create the VM
+gcloud compute instances delete --zone=$ZONE $TMP_VM --quiet || true
+gcloud compute instances create \
+       $TMP_VM \
+       --zone=$ZONE \
+       --image=$TMP_IMG \
+       --min-cpu-platform "Intel Haswell"
+
+INTERNAL_IP=$(gcloud --format="value(networkInterfaces[0].networkIP)" compute instances list --filter="name=('$TMP_VM')")
+EXTERNAL_IP=$(gcloud --format="value(networkInterfaces[0].accessConfigs[0].natIP)" compute instances list --filter="name=('$TMP_VM')")
+echo "external IP: $EXTERNAL_IP, internal IP: $INTERNAL_IP"
+
+echo "Waiting for SSH port to be available..."
+while ! nc -w 2 -z $INTERNAL_IP 22; do
+    sleep 1
+done
+
+echo "SSH is up. Copying prep-vm.sh script to VM..."
+
+# gcloud compute scp lacks an --internal-ip flag, even though gcloud
+# compute ssh has it. Annoying. Workaround:
+gcloud compute scp --dry-run --zone=$ZONE prep-vm.sh bradfitz@$TMP_VM: | perl -npe "s/$EXTERNAL_IP/$INTERNAL_IP/" | sh
+
+# And prep the machine.
+gcloud compute ssh $TMP_VM --zone=$ZONE --internal-ip -- sudo bash ./prep-vm.sh
+
+echo "Done prepping machine; shutting down"
+
+# Shut it down so it's a stable source to snapshot from.
+gcloud compute instances stop $TMP_VM --zone=$ZONE
+
+# Now make the new image from our instance's disk.
+gcloud compute images delete $TARGET_IMAGE --quiet || true
+gcloud compute images create $TARGET_IMAGE --source-disk=$TMP_VM --source-disk-zone=$ZONE
+
+echo "Done."

env/linux-x86-vmx/prep-vm.sh

@@ -0,0 +1,34 @@
+#!/bin/false
+# Copyright 2019 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This runs on the Debian Stretch template VM to turn it into the
+# buildlet image we want. This isn't for running on the developer's
+# host machine.
+
+set -e
+set -x
+
+apt-get update
+apt-get install --yes apt-transport-https ca-certificates curl gnupg2 software-properties-common
+curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
+apt-get update
+apt-get install --yes docker-ce docker-ce-cli containerd.io
+
+git clone https://github.com/GoogleCloudPlatform/konlet.git
+mkdir -p /usr/share/google
+install konlet/scripts/get_metadata_value /usr/share/google
+mkdir -p /usr/share/gce-containers
+install konlet/scripts/konlet-startup /usr/share/gce-containers/konlet-startup
+install konlet/scripts/konlet-startup.service /etc/systemd/system
+systemctl enable /etc/systemd/system/konlet-startup.service
+systemctl start konlet-startup
+
+# Pre-pull some common images/layers to speed up future boots:
+gcloud auth configure-docker --quiet
+docker pull gcr.io/symbolic-datum-552/linux-x86-stretch:latest
+docker pull gcr.io/gce-containers/konlet:v.0.9-latest
+
+apt-get dist-upgrade --yes