devops: Update release process for Github

.github/workflows/main.yml

@@ -1,13 +1,15 @@
-name: Test/Release
+name: Test-Release
 on: [push]
 
 jobs:
-  test_maybe_release:
-    name: Test (then release if new version)
+  tests:
+    name: "Tests (webextension, interfacer: unit, tty, http-server)"
     runs-on: ubuntu-latest
     env:
       GOPATH: ${{ github.workspace }}
       GOBIN: ${{ github.workspace }}/bin
+    outputs:
+      is_new_version: ${{ steps.check_versions.outputs.is_new_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -16,15 +18,15 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18.x
+          go-version-file: 'interfacer/go.mod'
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version-file: '.nvmrc'
       - name: Install Firefox
         uses: browser-actions/setup-firefox@latest
         with:
-          firefox-version: 102.0.1 # TODO: Use same version in Dockerfile
+          firefox-version: 102.0.1
       - run: firefox --version
 
       # Web extension tests
@@ -33,14 +35,13 @@ jobs:
       - name: Web extension tests
         run: npm test
         working-directory: ./webext
-      - run: npm run build_webextension
-        working-directory: ./webext
 
       # Interfacer tests
-      - name: Build
-        run: go build ./cmd/browsh
-        working-directory: ./interfacer
-
+      - name: Interfacer tests setup
+        run: |
+          touch interfacer/src/browsh/browsh.xpi
+          cd webext
+          npm run build:dev
       - name: Unit tests
         run: go test -v $(find src/browsh -name '*.go' | grep -v windows)
         working-directory: ./interfacer
@@ -51,25 +52,72 @@ jobs:
         if: ${{ failure() }}
         run: cat ./interfacer/test/tty/debug.log || echo "No log file"
       - name: HTTP Server tests
-        run: go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3
-        working-directory: ./interfacer
+        uses: nick-fields/retry@v2
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 15
+          command: |
+            cd interfacer
+            go test test/http-server/*.go -v -ginkgo.slowSpecThreshold=30 -ginkgo.flakeAttempts=3
       - name: HTTP Server debug log
         if: ${{ failure() }}
         run: cat ./interfacer/test/http-server/debug.log || echo "No log file"
 
-      # Release
       - name: Check for new version
         id: check_versions
         run: ./ctl.sh github_actions_output_version_status
-      - name: Release
-        if: contains(steps.check_versions.outputs.is_new_version, 'true')
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: tests
+    if: github.ref == 'refs/heads/master' && contains(needs.tests.outputs.is_new_version, 'true')
+    env:
+      GOPATH: ${{ github.workspace }}
+      GOBIN: ${{ github.workspace }}/bin
+      MDN_KEY: ${{ secrets.MDN_KEY }}
+      DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Setup Deploy Keys
+        uses: webfactory/[email protected]
+        with:
+          # Note that these private keys depend on having an ssh-keygen'd comment with the
+          # Git remote URL. This is because Github Actions use the *first* matching private
+          # key and fails if it doesn't match. webfactory/ssh-agent
+          ssh-private-key: |
+            ${{ secrets.HOMEBREW_DEPLOY_KEY }}
+            ${{ secrets.WWW_DEPLOY_KEY }}
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: 'interfacer/go.mod'
+      - run: npm ci
+        working-directory: ./webext
+      - name: Binary Release
         run: ./ctl.sh release
+      - name: Push new tag
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          tags: true
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: tombh
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
       - name: Docker Release
         run: ./ctl.sh docker_release
+      - name: Update Homebrew Tap
+        run: ./ctl.sh update_homebrew_tap_with_new_version
       - name: Update Browsh Website
         run: ./ctl.sh update_browsh_website_with_new_version

.gitignore

@@ -12,3 +12,14 @@ webext/node_modules
 webext/dist/*
 dist
 *.xpi
+
+# This is because of an odd permissions quirk on Github Actions. I can't seem to find a
+# way to delete these files in CI, so let's just ignore them. Otherwise Goreleaser complains
+# about a dirty working tree.
+/pkg
+/bin
+
+# Goreleaser needs to upload the webextension as an extra file in the release. But it doesn't
+# like Git to be in a dirty state. Also Goreleaser is run at PWD ./interfacer, so we can't
+# reference the webextension with something like ../webext/...
+interfacer/browsh-*.xpi

Dockerfile

@@ -24,21 +24,19 @@ ADD .github .github
 ADD scripts scripts
 ADD ctl.sh .
 
-# Install Golang
+# Install Golang and Browsh
 ENV GOROOT=/go
 ENV GOPATH=/go-home
 ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
-RUN /build/ctl.sh install_golang
-
-# Install firefox
-RUN /build/ctl.sh install_firefox
-
-# Build Browsh
 ENV BASE=$GOPATH/src/browsh/interfacer
-WORKDIR $BASE
 ADD interfacer $BASE
+WORKDIR $BASE
+RUN /build/ctl.sh install_golang $BASE
 RUN /build/ctl.sh build_browsh_binary $BASE
 
+# Install firefox
+RUN /build/ctl.sh install_firefox
+
 
 ###########################
 # Actual final Docker image

README.md

@@ -35,8 +35,8 @@ is running somewhere else on mains electricity.
 
 ## Installation
 
-Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~7MB).
-You will need to have [Firefox 63](https://www.mozilla.org/en-US/firefox/new/) (or higher) already installed.
+Download a binary from the [releases](https://github.com/browsh-org/browsh/releases) (~11MB).
+You will need to have [Firefox](https://www.mozilla.org/en-US/firefox/new/) already installed.
 
 Or download and run the Docker image (~230MB) with:
     `docker run --rm -it browsh/browsh`

ctl.sh

@@ -1,11 +1,12 @@
-#!/bin/env bash
+#!/usr/bin/env bash
 set -e
 
 function_to_run=$1
 
-export PROJECT_ROOT && PROJECT_ROOT=$(git rev-parse --show-toplevel)
+export PROJECT_ROOT
 export GORELEASER_VERSION=1.10.2
-export GOBINDATA_VERSION=3.23.0
+
+PROJECT_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 function _includes_path {
 	echo "$PROJECT_ROOT"/scripts
@@ -26,4 +27,7 @@ if [[ $(type -t "$function_to_run") != function ]]; then
 fi
 
 shift
+
+pushd "$PROJECT_ROOT" || _panic
 "$function_to_run" "$@"
+popd || _panic

goreleaser.yml

@@ -57,15 +57,14 @@ nfpms:
 brews:
   - name: browsh
     tap:
-      owner: browsh-org
       name: homebrew-browsh
-    commit_author:
-      name: Goreleaser Bot care of Github Actions
-      email: [email protected]
     homepage: "https://www.brow.sh"
     description: "The modern, text-based browser"
     caveats: "You need Firefox 57 or newer to run Browsh"
+    # We do the upload manually because Goreleaser doesn't support Deploy Keys and Github
+    # doesn't support repo-specific Access Tokens 🙄
+    skip_upload: true
 
 release:
   extra_files:
-    - glob: ./webext/dist/web-ext-artifacts/browsh-{{ Env.BROWSH_VERSION }}-an+fx.xpi
+    - glob: ./browsh-*.xpi

interfacer/src/browsh/firefox.go

@@ -230,8 +230,6 @@ func firefoxMarionette() {
 	sendFirefoxCommand("WebDriver:NewSession", map[string]interface{}{})
 }
 
-// Install the Browsh extension that was bundled with `go-bindata` under
-// `webextension.go`.
 func installWebextension() {
 	data, err := browshXpi.ReadFile("browsh.xpi")
 	if err != nil {

interfacer/src/browsh/version.go

@@ -1,3 +1,3 @@
 package browsh
 
-var browshVersion = "1.6.4"
+var browshVersion = "1.8.0"

scripts/bundling.bash

@@ -1,7 +1,23 @@
 #!/usr/bin/env bash
 
-function build_webextension() {
-	local NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin
+export XPI_PATH="$PROJECT_ROOT"/interfacer/src/browsh/browsh.xpi
+export XPI_SOURCE_DIR=$PROJECT_ROOT/webext/dist/web-ext-artifacts
+export NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin
+MDN_USER="user:13243312:78"
+
+function versioned_xpi_file() {
+	echo "$XPI_SOURCE_DIR/browsh-$(browsh_version).xpi"
+}
+
+# You'll want to use this with `go run ./cmd/browsh --debug --firefox.use-existing`
+function build_webextension_watch() {
+	"$NODE_BIN"/web-ext run \
+		--firefox contrib/firefoxheadless.sh \
+		--verbose
+}
+
+function build_webextension_production() {
+	local version && version=$(browsh_version)
 
 	cd "$PROJECT_ROOT"/webext && "$NODE_BIN"/webpack
 	cd "$PROJECT_ROOT"/webext/dist && rm ./*.map
@@ -13,36 +29,63 @@ function build_webextension() {
 	"$NODE_BIN"/web-ext build --overwrite-dest
 	ls -alh web-ext-artifacts
 
-	version=$(browsh_version)
+	webextension_sign
+	local source_file && source_file=$(versioned_xpi_file)
+
+	echo "Bundling $source_file to $XPI_PATH"
+	cp -f "$source_file" "$XPI_PATH"
 
-	local source_file
-	local source_dir=$PROJECT_ROOT/webext/dist/web-ext-artifacts
-	local bundle_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi
+	echo "Making extra copy for Goreleaser to put in Github release:"
+	local goreleaser_pwd="$PROJECT_ROOT"/interfacer/
+	cp -a "$source_file" "$goreleaser_pwd"
+	ls -alh "$goreleaser_pwd"
+}
 
-	if [ "$BROWSH_ENV" == "RELEASE" ]; then
-		# The signed version. There can only be one canonical XPI for each semantic
-		# version.
-		source_file="$source_dir/browsh-$version-an+fx.xpi"
+# It is possible to use unsigned webextensions in Firefox but it requires that Firefox
+# uses problematically insecure config. I know it's a hassle having to jump through all
+# these signing hoops, but I think it's better to use a standard Firefox configuration.
+# Moving away from the webextension alltogether is another story, but something I'm still
+# thinking about.
+#
+# NB: There can only be one canonical XPI for each semantic version.
+#
+# shellcheck disable=2120
+function webextension_sign() {
+	local use_existing=$1
+	if [ "$use_existing" == "" ]; then
 		"$NODE_BIN"/web-ext sign --api-key "$MDN_USER" --api-secret "$MDN_KEY"
+		_rename_built_xpi
 	else
-		# TODO: This doesn't currently work with the Marionettte `tempAddon`
-		# installation method. Just use `web-ext run` and Browsh's `use-existing-ff`
-		# flag - which is better anyway as it auto-reloads the extension when files
-		# change. NB: If you fix this, don't forget to change the filename loaded
-		# by `Asset()` in `main.go`.
-		# In development/testing, we want to be able to bundle the webextension
-		# frequently without having to resort to version bumps.
-		source_file="$source_dir/browsh-$version.zip"
+		echo "Skipping signing, downloading existing webextension"
+		local base="https://github.com/browsh-org/browsh/releases/download"
+		curl -L \
+			-o "$(versioned_xpi_file)" \
+			"$base/v$LATEST_TAGGED_VERSION/browsh-$LATEST_TAGGED_VERSION.xpi"
 	fi
+}
 
-	echo "Bundling $source_file to $bundle_file"
-	cp -f "$source_file" "$bundle_file"
+function _rename_built_xpi() {
+	pushd "$XPI_SOURCE_DIR" || _panic
+	local xpi_file
+	xpi_file="$(
+		find ./*.xpi \
+			-printf "%T@ %f\n" |
+			sort |
+			cut -d' ' -f2 |
+			tail -n1
+	)"
+	cp -a "$xpi_file" "$(versioned_xpi_file)"
+	popd || _panic
 }
 
 function bundle_production_webextension() {
 	local version && version=$(browsh_version)
 	local base='https://github.com/browsh-org/browsh/releases/download'
 	local release_url="$base/v$version/browsh-$version-an.fx.xpi"
-	local xpi_file=$PROJECT_ROOT/interfacer/src/browsh/browsh.xpi
-	curl -L -o "$xpi_file" "$release_url"
+	echo "Downloading webextension from: $release_url"
+	local size && size=$(wc -c <"$XPI_PATH")
+	curl -L -o "$XPI_PATH" "$release_url"
+	if [ "$size" -lt 500 ]; then
+		_panic "Problem downloading latest webextension XPI"
+	fi
 }