env/dragonfly-amd64: add scripts for building GCE VM image

Now that Dragonfly runs on GCE, we can do that and retire the
one very slow reverse builder we are using today.

For golang/go#23060.

Change-Id: I2bd8c8be6735212ba6a8023327864b79dea08cf3
Reviewed-on: https://go-review.googlesource.com/c/build/+/419081
Auto-Submit: Russ Cox <[email protected]>
Run-TryBot: Russ Cox <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Heschi Kreinick <[email protected]>

env/dragonfly-amd64/.gitignore

@@ -0,0 +1,5 @@
+*.iso
+*.iso.bz2
+disk-*
+disk.*
+isotmp

env/dragonfly-amd64/Makefile

@@ -0,0 +1,110 @@
+# See README for an overview of how to use this Makefile.
+
+# Database of ISO names and MD5s. Variable name suffix is short version (622 for 6.2.2, see V below)
+ISO_600=dfly-x86_64-6.0.0_REL.iso
+MD5_600=252fc700803dbadce1de928432373bbb
+
+ISO_622=dfly-x86_64-6.2.2_REL.iso
+MD5_622=b391a29c8e9c1cc33f2441ff1f0d98a0
+
+# Default version when invoking make. Can override on command line: make V=600
+V=622
+
+ISO=$(ISO_$V)
+MD5=$(MD5_$V)
+
+default: disk-$V.tar.gz
+
+clean:
+	rm -f *.iso *.iso.bz2 phase*.iso isotmp disk-*.phase* disk-*.tar.gz disk.raw disk.tmp
+
+# Download image from DragonflyBSD page.
+$(ISO):
+	curl -o $@.bz2 https://mirror-master.dragonflybsd.org/iso-images/$@.bz2
+	echo "$(MD5)  $@.bz2" | md5sum -c -
+	bunzip2 $@.bz2
+
+# If Dragonfly boots with a CD containing pfi.conf that says pfi_script=name.sh,
+# then it runs name.sh at the end of the standard startup process.
+# We use this to run a script inside a QEMU image.
+# phase1.iso is a CD containing phase1.sh, and so on.
+%.iso: %.sh
+	rm -rf isotmp
+	mkdir isotmp
+	cp $^ isotmp
+	echo pfi_script=$*.sh >>isotmp/pfi.conf
+	chmod +x isotmp/$*.sh
+	rm -f $@
+	genisoimage -r -o $@ isotmp/
+	rm -rf isotmp
+
+# Disk image creation is broken into phases so that later phases
+# can be modified and rerun without repeating the earlier phases,
+# which can take a long time.
+
+# Phase 1 sets up a basic DragonflyBSD install on the disk.
+# The installer is CD0, the PFI script is CD1, and the installer is also CD2.
+# CD2 is used by phase1.sh to get a clean FS image with nothing mounted on it.
+disk-$V.phase1: $(ISO) phase1.iso
+	qemu-img create -f qcow2 $@ 16G
+	(sleep 1; echo 9; echo set console=comconsole; echo boot) | \
+	qemu-system-x86_64 \
+		-display none -serial stdio \
+		-m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \
+		-drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \
+		-drive file=$(ISO),media=cdrom \
+		-drive file=phase1.iso,media=cdrom \
+		-drive file=$(ISO),media=cdrom
+
+# Phase 2 updates the pkg database and installing curl, git, and so on.
+disk-$V.phase2: disk-$V.phase1 phase2.iso
+	cp $< $@
+	qemu-system-x86_64 \
+		-display none -serial stdio \
+		-m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \
+		-drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \
+		-drive file=phase2.iso,media=cdrom
+
+# Phase 3 customizes the image to run the buildlet.
+phase3.iso: buildlet
+disk-$V.phase3: disk-$V.phase2 phase3.iso
+	cp $< $@
+	qemu-system-x86_64 \
+		-display none -serial stdio \
+		-m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \
+		-drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \
+		-drive file=phase3.iso,media=cdrom
+
+# For testing and exploration
+run: disk-$V.phase3
+	cp $< disk.tmp
+	qemu-system-x86_64 \
+		-display none -serial stdio \
+		-m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \
+		-drive file=disk.tmp,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0
+
+# Convert QEMU image back to raw for Google Cloud.
+disk-$V.tar.gz: disk-$V.phase3
+	rm -f disk.raw
+	qemu-img convert -f qcow2 -O raw -t none -T none $< disk.raw
+	tar -Sczf $@ disk.raw
+	rm -f disk.raw
+
+# Upload and create prod disk image.
+upload-prod: disk-$V.tar.gz
+	@echo "Run 'gcloud auth application-default login' if you get credential errors."
+	go run ../../cmd/upload --verbose --file=$< --public go-builder-data/dragonfly-amd64-$V.tar.gz
+	gcloud compute --project symbolic-datum-552 images create dragonfly-amd64-$V --source-uri gs://go-builder-data/dragonfly-amd64-$V.tar.gz
+
+# Delete prod disk image.
+delete-prod:
+	gcloud compute --project symbolic-datum-552 images delete dragonfly-amd64-$V
+
+# Upload and create staging disk image.
+upload-staging: disk-$V.tar.gz
+	go run ../../cmd/upload --verbose --file=$< --public dev-go-builder-data/dragonfly-amd64-$V.tar.gz
+	gcloud compute --project go-dashboard-dev images create dragonfly-amd64-$V --source-uri gs://dev-go-builder-data/dragonfly-amd64-$V.tar.gz
+
+# Delete staging disk image.
+delete-staging:
+	gcloud compute --project go-dashboard-dev images delete dragonfly-amd64-$V

env/dragonfly-amd64/README

@@ -0,0 +1,28 @@
+The Makefile in this directory creates a Google Compute Engine VM image to run the Go
+DragonflyBSD builder, booting up to run the buildlet.
+
+make must be run on a Linux box with qemu and a few additional packages:
+
+	apt-get install qemu-system-x86 qemu-utils expect genisoimage
+
+To override the default Dragonfly version, set V= on the make command line:
+
+	make V=600
+
+To add new Dragonfly versions, add lines to the "Database" in the Makefile
+and perhaps change the V= line to update the default.
+
+	make upload-prod
+
+builds and uploads the image. To replace an existing image, use
+
+	make delete-prod upload-prod
+
+s/prod/staging/ to test in the staging cluster instead.
+
+The VM needs to be run with the GCE metadata attribute "buildlet-binary-url" set to a URL
+of the DragonflyBSD buildlet (cross-compiled, typically).
+
+	buildlet-binary-url == http://storage.googleapis.com/go-builder-data/buildlet.dragonfly-amd64
+
+The buildlet rc script is lightly modified from ../freebsd-amd64.

env/dragonfly-amd64/buildlet

@@ -0,0 +1,79 @@
+#!/bin/sh
+# PROVIDE: buildlet
+# REQUIRE: NETWORKING DAEMON
+# BEFORE: LOGIN
+
+. /etc/rc.subr
+
+name=buildlet
+rcvar=buildlet_enable
+
+procname=/buildlet
+pidfile="/var/run/${name}.pid"
+command=/usr/sbin/daemon
+command_args="-r -fc -p ${pidfile}"
+command_args="${command_args} ${procname}"
+start_precmd="${name}_prestart"
+stop_cmd=":"
+
+load_rc_config $name
+: ${buildlet_enable:="NO"}
+
+buildlet_resetnet()
+{
+	warn "buildlet resetting network"
+	ifconfig vtnet0 down
+	sleep 1
+	ifconfig vtnet0 up
+	sleep 1
+	dhclient vtnet0
+	sleep 1
+}
+
+buildlet_fixnet()
+{
+	# Some fraction of the time, the VM comes up unable to deliver UDP packets.
+	# If we detect this situation, by host -W 3 failing to look up metadata.google.internal,
+	# then reset the network interface (ifup/ifdown/dhclient).
+	# Once is almost always enough, so 10 attempts should drive the failure rate to zero.
+	for i in 0 1 2 3 4 5 6 7 8 9; do
+		if /usr/local/bin/host -W 3 metadata.google.internal; then
+			return 0
+		fi
+		buildlet_resetnet
+	done
+	return 1
+}
+
+buildlet_prestart()
+{
+	local buildlet_url
+
+	info $(netstat -rn)
+	info $(cat /etc/resolv.conf)
+
+	if ! buildlet_fixnet; then
+		warn "cannot fix network"
+		poweroff
+		return 1
+	fi
+
+	buildlet_url=$(/usr/local/bin/curl -s -H "Metadata-Flavor: Google" \
+		http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url)
+
+	if [ "$buildlet_url" = "" ]; then
+		warn "cannot find buildlet url"
+		poweroff
+		return 1
+	fi
+
+	if ! /usr/local/bin/curl -o /buildlet "${buildlet_url}"; then
+		warn "failed to download buildlet from ${buildlet_url}"
+		poweroff
+		return 1
+	fi
+
+	chmod a+x /buildlet
+}
+
+run_rc_command "$1"

env/dragonfly-amd64/phase1.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+
+# Phase 1 of the DragonflyBSD installation: boot from installer CD and install on empty disk.
+
+set -ex
+
+echo >&2 phase1.sh starting
+
+mkdir -p /root/gopherinstall
+cp -a /mnt/* /root/gopherinstall
+cd /root/gopherinstall
+
+echo >&2 install.sh running
+
+# Following Manual Installation section of https://www.dragonflybsd.org/docs/handbook/Installation/#index3h1
+
+fdisk -IB /dev/da0
+disklabel -r -w -B /dev/da0s1 auto
+disklabel da0s1 >label
+echo '
+a: 1G 0 4.2BSD
+d: * * HAMMER2
+' >>label
+disklabel -R -r da0s1 label
+
+newfs /dev/da0s1a
+newfs_hammer2 -L ROOT /dev/da0s1d
+mount /dev/da0s1d /mnt
+mkdir /mnt/boot /mnt/mnt
+mount /dev/da0s1a /mnt/boot
+
+# mount clean file system image from second copy of CD on /mnt/mnt for copying.
+# the install instructions use cpdup / /mnt,
+# but using this second copy of the CD avoids all the tmpfs
+# that are mounted on top of /, as well as our local modifications (like the gopherinstall user).
+mount_cd9660 /dev/cd2 /mnt/mnt
+cpdup /mnt/mnt/boot /mnt/boot
+cpdup /mnt/mnt /mnt
+
+cat >/mnt/etc/rc.conf <<'EOF'
+ifconfig_vtnet0='DHCP mtu 1460'
+EOF
+
+cat >/mnt/boot/loader.conf <<'EOF'
+vfs.root.mountfrom=hammer2:da0s1d
+console=comconsole
+EOF
+
+cat >/mnt/etc/fstab <<'EOF'
+da0s1a    /boot           ufs      rw              1 1
+da0s1d    /               hammer2  rw              1 1
+EOF
+
+umount /mnt/mnt
+umount /mnt/boot
+
+echo 'DONE WITH PHASE 1.'
+sync
+poweroff
+sleep 86400

env/dragonfly-amd64/phase2.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Phase 2 of the DragonflyBSD installation: update pkg database.
+
+set -ex
+
+echo >&2 phase2.sh starting
+
+# Make pfi look for CD again when booting for phase3.
+# Normally /etc/pfi.conf is left behind and stops future checks.
+# Edit /etc/rc.d/pfi to remove /etc/pfi.conf each time it starts running.
+echo '/REQUIRE/a
+rm -f /etc/pfi.conf
+.
+w
+q' | ed /etc/rc.d/pfi
+
+# pfi startup does not have full path that a root login does.
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/pkg/sbin:/usr/pkg/bin:/root/bin
+
+# Update pkg database and install extras we need.
+pkg install -y bash curl git gdb
+
+echo 'DONE WITH PHASE 2.'
+sync
+poweroff
+sleep 86400

env/dragonfly-amd64/phase3.sh

@@ -0,0 +1,47 @@
+#!/usr/local/bin/bash
+
+# Phase 3 of the DragonflyBSD installation: apply buildlet customizations.
+
+set -ex
+set -o pipefail
+
+echo >&2 phase3.sh starting
+rm -f /etc/pfi.conf
+
+# pfi startup does not have full path that a root login does.
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/pkg/sbin:/usr/pkg/bin:/root/bin
+
+# Add a gopher user.
+pw useradd gopher -g 0 -c 'Gopher Gopherson' -s /bin/sh
+
+# Disable keyboard console that won't exist on Google Cloud,
+# to silence errors about trying to run getty on them.
+# Serial console will remain enabled.
+perl -pi -e 's/^ttyv/# ttyv/' /etc/ttys
+
+# Set up buildlet service.
+cp /mnt/buildlet /etc/rc.d/buildlet
+chmod +x /etc/rc.d/buildlet
+
+# Update rc.conf to run buildlet.
+cat >>/etc/rc.conf <<'EOF'
+hostname="buildlet"
+rc_info="YES"
+rc_startmsgs="YES"
+sshd_enable="YES"
+buildlet_enable="YES"
+buildlet_env="PATH=/bin:/sbin:/usr/bin:/usr/local/bin"
+EOF
+
+# Update loader.conf to disable checksum offload.
+cat >>/boot/loader.conf <<'EOF'
+hw.vtnet.csum_disable="1"
+EOF
+
+# Generate ssh keys if needed.
+service sshd keygen
+
+echo 'DONE WITH PHASE 3.'
+sync
+poweroff
+sleep 86400

env/freebsd-amd64/buildlet

@@ -22,17 +22,14 @@ load_rc_config $name
 buildlet_prestart()
 {
 	local buildlet_url
-	local retval
 
 	info $(netstat -rn)
 	info $(cat /etc/resolv.conf)
 
 	buildlet_url=$(/usr/local/bin/curl -s -H "Metadata-Flavor: Google" \
 		http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url)
 
-	/usr/local/bin/curl -s -o /buildlet "${buildlet_url}" >/dev/null 2>&1
-	retval=$?
-	if [ $retval -ne 0 ]; then
+	if ! /usr/local/bin/curl -o /buildlet "${buildlet_url}"; then
 		warn "failed to download buildlet from ${buildlet_url}"
 		poweroff
 		return 1