ci: setup automatic releases

.github/workflows/ibis-main.yml

@@ -277,32 +277,20 @@ jobs:
 
   simulate_release:
     runs-on: ubuntu-latest
-    needs:
-      - nix
-      - lint
-      - test_no_backends
-      - benchmarks
-      - docs
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
-      - name: Configure git info
-        run: |
-          set -euo pipefail
-
-          git config --global user.name 'ibis-project-bot[bot]'
-          git config --global user.email 'ibis-project-bot[bot]@users.noreply.github.com'
-
-      - name: setup python
-        uses: actions/setup-python@v2
+      - uses: cachix/install-nix-action@v16
         with:
-          python-version: "3.x"
+          nix_path: nixpkgs=channel:nixos-unstable-small
 
-      - name: install poetry and semantic-release
-        run: python -m pip install poetry python-semantic-release
+      - uses: cachix/cachix-action@v10
+        with:
+          name: ibis
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+          extraPullNames: nix-community,poetry2nix
 
-      - name: semantic release
-        run: semantic-release publish --noop --verbosity=DEBUG
+      - name: run semantic-release
+        run: ./ci/release/dry_run.sh

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: Release
+
+on:
+  workflow_dispatch:
+
+# we do not want more than one release workflow executing at the same time, ever
+concurrency:
+  group: release
+  # cancelling in the middle of a release would create incomplete releases
+  # so cancel-in-progress is false
+  cancel-in-progress: false
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tibdex/github-app-token@v1
+        id: generate_token
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          token: ${{ steps.generate_token.outputs.token }}
+
+      - uses: cachix/install-nix-action@v16
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable-small
+          extra_nix_config: |
+            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+      - uses: cachix/cachix-action@v10
+        with:
+          name: ibis
+          extraPullNames: nix-community,poetry2nix
+
+      - name: run semantic-release
+        run: ./ci/release/run.sh
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}

.releaserc.json

@@ -0,0 +1,36 @@
+{
+  "branches": ["master"],
+  "tagFormat": "${version}",
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    [
+      "@semantic-release/changelog",
+      {
+        "changelogTitle": "# Release Notes",
+        "changelogFile": "docs/web/release_notes.md"
+      }
+    ],
+    [
+      "@semantic-release/exec",
+      {
+        "verifyConditionsCmd": "ci/release/verify.sh",
+        "prepareCmd": "ci/release/prepare.sh ${nextRelease.version}",
+        "publishCmd": "ci/release/publish.sh"
+      }
+    ],
+    [
+      "@semantic-release/github",
+      {
+        "assets": ["dist/*.whl"]
+      }
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": ["pyproject.toml", "docs/web/release_notes.md"],
+        "message": "chore(release): ${nextRelease.version}"
+      }
+    ]
+  ]
+}

ci/release/dry_run.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -I nixpkgs=channel:nixos-unstable-small --pure -p git nodejs nix -i bash
+# shellcheck shell=bash
+
+set -euo pipefail
+
+curdir="$PWD"
+worktree="$(mktemp -d)"
+branch="$(basename "$worktree")"
+
+git worktree add "$worktree"
+
+function cleanup() {
+  cd "$curdir" || exit 1
+  git worktree remove "$worktree"
+  git worktree prune
+  git branch -D "$branch"
+}
+
+trap cleanup EXIT ERR
+
+cd "$worktree" || exit 1
+
+npx --yes \
+  -p semantic-release \
+  -p "@semantic-release/commit-analyzer" \
+  -p "@semantic-release/release-notes-generator" \
+  -p "@semantic-release/changelog" \
+  -p "@semantic-release/exec" \
+  -p "@semantic-release/git" \
+  semantic-release \
+  --ci \
+  --dry-run \
+  --plugins \
+  --analyze-commits "@semantic-release/commit-analyzer" \
+  --generate-notes "@semantic-release/release-notes-generator" \
+  --verify-conditions "@semantic-release/changelog,@semantic-release/exec,@semantic-release/git" \
+  --prepare "@semantic-release/changelog,@semantic-release/exec" \
+  --branches "$branch" \
+  --repository-url "file://$PWD"

ci/release/prepare.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --pure -p poetry -i bash
+# shellcheck shell=bash
+
+set -euo pipefail
+
+# set version
+poetry version "$1"
+
+# build artifacts
+poetry build

ci/release/publish.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --pure --keep POETRY_PYPI_TOKEN_PYPI -p poetry -i bash
+# shellcheck shell=bash
+
+set -euo pipefail
+
+poetry publish

ci/release/run.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p cacert poetry git nodejs nix -i bash
+# shellcheck shell=bash
+
+set -euo pipefail
+
+npx --yes \
+  -p semantic-release \
+  -p "@semantic-release/commit-analyzer" \
+  -p "@semantic-release/release-notes-generator" \
+  -p "@semantic-release/changelog" \
+  -p "@semantic-release/github" \
+  -p "@semantic-release/exec" \
+  -p "@semantic-release/git" \
+  semantic-release --ci

ci/release/verify.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --pure --keep POETRY_PYPI_TOKEN_PYPI -p poetry -p git -i bash
+# shellcheck shell=bash
+
+set -euo pipefail
+
+# verify TOML is sane
+poetry check
+
+# verify that the lock file is up to date
+poetry lock --no-update
+git diff --exit-code poetry.lock
+
+# verify that we have a token available to push to pypi using set -u
+: "${POETRY_PYPI_TOKEN_PYPI}"

docs/web/release_notes.md

@@ -1,3 +1 @@
 # Release Notes
-
-<!--next-version-placeholder-->

pre-commit.nix

@@ -47,13 +47,14 @@ in
         enable = true;
         entry = lib.mkForce "shellcheck";
         files = "\\.sh$";
-        types_or = lib.mkForce [ ];
+        types_or = [ "file" ];
       };
 
       shfmt = {
         enable = true;
         entry = lib.mkForce "shfmt -i 2 -sr -d -s -l";
         files = "\\.sh$";
+        types_or = [ "file" ];
       };
 
       pyupgrade = {

pyproject.toml

@@ -135,20 +135,6 @@ pyspark = "ibis.backends.pyspark"
 spark = "ibis.backends.pyspark"
 sqlite = "ibis.backends.sqlite"
 
-[tool.semantic_release]
-version_toml = "pyproject.toml:tool.poetry.version"
-version_variable = [
-  "ibis/__init__.py:__version__",
-  "docs/source/conf.py:version"
-]
-branch = "master"
-upload_to_pypi = false
-upload_to_release = false
-build_command = "poetry build"
-commit_subject = "chore(release): {version}"
-commit_author = "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"
-changelog_file = "docs/web/release_notes.md"
-
 [tool.pytest.ini_options]
 xfail_strict = true
 addopts = [