fix(registry): heartbeat transitions provisioning→online on first heartbeat (#1784)

.github/workflows/auto-promote-staging.yml

@@ -0,0 +1,182 @@
+name: Auto-promote staging → main
+
+# Fires after any of the staging-branch quality gates complete. When ALL
+# required gates are green on the same staging SHA, fast-forwards `main`
+# to that SHA automatically — closing the gap that historically let
+# features sit on staging for weeks waiting for a bulk promotion PR
+# (see molecule-core#1496 for the 1172-commit example).
+#
+# Safety model:
+# - Runs ONLY on workflow_run events for the staging branch.
+# - Requires EVERY named gate workflow to have the same head_sha and
+#   all be `conclusion == success`. If any of them is red, skipped,
+#   cancelled, or pending, we abort (stay on the current main).
+# - Uses --ff-only: refuses to advance main if main has diverged from
+#   the staging history (e.g. a hotfix landed directly on main). In
+#   that case a human resolves the fork.
+# - Writes a commit summary so the promote shows up in git log as a
+#   deliberate act, not a stealth move.
+#
+# **Initial rollout:** ship this file but leave the `enabled` input set
+# such that nothing auto-promotes until staging CI has been reliably
+# green for a few days. Toggle via repo variable `AUTO_PROMOTE_ENABLED`.
+
+on:
+  workflow_run:
+    workflows:
+      - CI
+      - E2E Staging Canvas (Playwright)
+      - E2E API Smoke Test
+      - CodeQL
+    types: [completed]
+  workflow_dispatch:
+    inputs:
+      force:
+        description: "Force promote even when AUTO_PROMOTE_ENABLED is unset (manual override)"
+        required: false
+        default: "false"
+
+permissions:
+  contents: write
+
+jobs:
+  check-all-gates-green:
+    # Only consider staging pushes. PRs into staging don't promote.
+    if: >
+      (github.event_name == 'workflow_run' &&
+       github.event.workflow_run.head_branch == 'staging' &&
+       github.event.workflow_run.event == 'push')
+      || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    outputs:
+      all_green: ${{ steps.gates.outputs.all_green }}
+      head_sha: ${{ steps.gates.outputs.head_sha }}
+    steps:
+      - name: Check all required gates on this SHA
+        id: gates
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          REPO: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+
+          # Required gate workflow names. Must match the `name:` field
+          # in the respective .github/workflows/*.yml files.
+          GATES=(
+            "CI"
+            "E2E Staging Canvas (Playwright)"
+            "E2E API Smoke Test"
+            "CodeQL"
+          )
+
+          echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT"
+          echo "Checking gates on SHA ${HEAD_SHA}"
+
+          ALL_GREEN=true
+          for gate in "${GATES[@]}"; do
+            # Query the most recent run of this workflow on this SHA.
+            # event=push to avoid picking up PR runs. branch=staging to
+            # guard against someone dispatching the gate on a non-staging
+            # branch at the same SHA.
+            RESULT=$(gh run list \
+              --repo "$REPO" \
+              --workflow "$gate" \
+              --branch staging \
+              --event push \
+              --commit "$HEAD_SHA" \
+              --limit 1 \
+              --json status,conclusion \
+              --jq '.[0] | "\(.status)/\(.conclusion // "none")"' \
+              2>/dev/null || echo "missing/none")
+
+            echo "  $gate → $RESULT"
+
+            # Only completed/success counts. completed/failure or
+            # in_progress/anything or no record at all = abort.
+            if [ "$RESULT" != "completed/success" ]; then
+              ALL_GREEN=false
+            fi
+          done
+
+          echo "all_green=${ALL_GREEN}" >> "$GITHUB_OUTPUT"
+          if [ "$ALL_GREEN" != "true" ]; then
+            echo "::notice::auto-promote: not all gates are green on ${HEAD_SHA} — staying on current main"
+          fi
+
+  promote:
+    needs: check-all-gates-green
+    if: needs.check-all-gates-green.outputs.all_green == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check rollout gate
+        env:
+          AUTO_PROMOTE_ENABLED: ${{ vars.AUTO_PROMOTE_ENABLED }}
+          FORCE_INPUT: ${{ github.event.inputs.force }}
+        run: |
+          set -eu
+          # Repo variable AUTO_PROMOTE_ENABLED=true flips this on. While
+          # it's unset, the workflow dry-runs (logs what it would have
+          # done) but doesn't actually push to main. Set the variable in
+          # Settings → Secrets and variables → Actions → Variables.
+          if [ "${AUTO_PROMOTE_ENABLED:-}" != "true" ] && [ "${FORCE_INPUT:-false}" != "true" ]; then
+            {
+              echo "## ⏸ Auto-promote disabled"
+              echo
+              echo "Repo variable \`AUTO_PROMOTE_ENABLED\` is not set to \`true\`."
+              echo "All gates are green on staging; would have promoted to \`main\`."
+              echo
+              echo "To enable: Settings → Secrets and variables → Actions → Variables → \`AUTO_PROMOTE_ENABLED=true\`."
+              echo "To test once manually: workflow_dispatch with \`force=true\`."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::notice::auto-promote disabled — dry run only"
+            exit 0
+          fi
+
+      - name: Checkout main
+        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Fast-forward main → staging HEAD
+        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
+        env:
+          TARGET_SHA: ${{ needs.check-all-gates-green.outputs.head_sha }}
+        run: |
+          set -eu
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          git fetch origin staging
+          git fetch origin main
+
+          # Refuse to advance main if it's diverged from staging history.
+          # Someone landed a commit directly on main that's not on
+          # staging → human needs to decide how to reconcile.
+          if ! git merge-base --is-ancestor "$(git rev-parse origin/main)" "$TARGET_SHA"; then
+            {
+              echo "## ❌ Auto-promote refused — main has diverged"
+              echo
+              echo "\`main\` (\`$(git rev-parse --short origin/main)\`) is not an ancestor of staging (\`${TARGET_SHA:0:7}\`)."
+              echo "Someone committed directly to main or the histories forked."
+              echo
+              echo "Resolve manually: merge main into staging, get CI green on the merged commit,"
+              echo "then the auto-promote will succeed on the next run."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+
+          # Fast-forward main to the target SHA.
+          git checkout main
+          git merge --ff-only "$TARGET_SHA"
+          git push origin main
+
+          {
+            echo "## ✅ Auto-promoted main → ${TARGET_SHA:0:7}"
+            echo
+            echo "All gate workflows green on staging at this SHA."
+            echo "\`main\` fast-forwarded to match."
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/canary-verify.yml

@@ -34,11 +34,10 @@ jobs:
   canary-smoke:
     # Skip when the upstream workflow failed — no image to test against.
     if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
-    # Self-hosted mac mini — GitHub-hosted minutes are quota-blocked on
-    # this org (same reason publish/promote-latest moved earlier).
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     outputs:
       sha: ${{ steps.compute.outputs.sha }}
+      smoke_ran: ${{ steps.smoke.outputs.ran }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -49,11 +48,10 @@ jobs:
 
       - name: Wait for canary tenants to pick up :staging-<sha>
         # Poll canary health endpoints every 30s for up to 7 min instead
-        # of a fixed 6-min sleep. Exits as soon as ALL canaries report the
-        # new SHA, freeing the self-hosted runner slot sooner (~2-3 min
-        # typical vs 6 min fixed). Falls back to proceeding after 7 min
-        # even if not all canaries responded — the smoke suite will catch
-        # any that didn't update.
+        # of a fixed 6-min sleep. Exits as soon as ALL canaries report
+        # the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
+        # proceeding after 7 min even if not all canaries responded —
+        # the smoke suite will catch any that didn't update.
         env:
           CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
           EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
@@ -88,12 +86,38 @@ jobs:
           echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
 
       - name: Run canary smoke suite
+        id: smoke
+        # Graceful-skip when no canary fleet is configured (Phase 2 not yet
+        # stood up — see molecule-controlplane/docs/canary-tenants.md).
+        # Sets `ran=false` on skip so promote-to-latest stays off (we don't
+        # want every main merge auto-promoting without gating). Manual
+        # promote-latest.yml is the release gate while canary is absent.
+        # Once the fleet is real: delete the early-exit branch.
         env:
           CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
           CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
           CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
           CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
-        run: bash scripts/canary-smoke.sh
+        run: |
+          set -euo pipefail
+          if [ -z "${CANARY_TENANT_URLS:-}" ] \
+            || [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
+            || [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
+            {
+              echo "## ⚠️ canary-verify skipped"
+              echo
+              echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
+              echo "Phase 2 canary fleet has not been stood up yet —"
+              echo "see [canary-tenants.md](https://github.com/Molecule-AI/molecule-controlplane/blob/main/docs/canary-tenants.md)."
+              echo
+              echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "ran=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::canary-verify: skipped — no canary fleet configured"
+            exit 0
+          fi
+          bash scripts/canary-smoke.sh
+          echo "ran=true" >> "$GITHUB_OUTPUT"
 
       - name: Summary on failure
         if: ${{ failure() }}
@@ -112,23 +136,14 @@ jobs:
     # On green, retag :staging-<sha> → :latest for BOTH images.
     # crane is a lightweight registry client (no Docker daemon needed on
     # the runner) that can retag remotely with a single API call each.
+    # Gated on smoke_ran=true — without a real canary fleet the smoke
+    # step no-ops with success, and we don't want that to silently
+    # auto-promote every main merge.
     needs: canary-smoke
-    if: ${{ needs.canary-smoke.result == 'success' }}
-    runs-on: [self-hosted, macos, arm64]
+    if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
+    runs-on: ubuntu-latest
     steps:
-      - name: Ensure crane installed
-        # Matches the install pattern in promote-latest.yml — brew
-        # cleanup exits non-zero on the shared runner's /opt/homebrew
-        # symlinks, so skip it.
-        env:
-          HOMEBREW_NO_INSTALL_CLEANUP: "1"
-          HOMEBREW_NO_AUTO_UPDATE: "1"
-          HOMEBREW_NO_ENV_HINTS: "1"
-        run: |
-          if ! command -v crane >/dev/null 2>&1; then
-            brew install crane
-          fi
-          crane version
+      - uses: imjasonh/setup-crane@v0.4
 
       - name: GHCR login
         run: |

.github/workflows/ci.yml

@@ -7,17 +7,14 @@ on:
     branches: [main, staging]
 
 # Cancel in-progress CI runs when a new commit arrives on the same ref.
-# This prevents multiple stale runs from queuing behind each other and
-# monopolising the self-hosted macOS arm64 runner.
+# This prevents stale runs from queuing behind each other.
 concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   # Detect which paths changed so downstream jobs can skip when only
-  # docs/markdown files were modified. Uses plain `git diff` — no macOS
-  # dependency, so this runs on ubuntu-latest to free the self-hosted
-  # macOS arm64 runner for jobs that genuinely need it.
+  # docs/markdown files were modified.
   changes:
     name: Detect changes
     runs-on: ubuntu-latest
@@ -62,7 +59,7 @@ jobs:
     name: Platform (Go)
     needs: changes
     if: needs.changes.outputs.platform == 'true'
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: workspace-server
@@ -98,7 +95,7 @@ jobs:
     name: Canvas (Next.js)
     needs: changes
     if: needs.changes.outputs.canvas == 'true'
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: canvas
@@ -124,23 +121,18 @@ jobs:
     name: Shellcheck (E2E scripts)
     needs: changes
     if: needs.changes.outputs.scripts == 'true'
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Run shellcheck on tests/e2e/*.sh
-        # `ludeeus/action-shellcheck` is a Docker action (Linux-only). We rely
-        # on shellcheck being pre-installed on the self-hosted runner instead.
+        # shellcheck is pre-installed on ubuntu-latest runners (via apt).
         run: |
-          if ! command -v shellcheck >/dev/null 2>&1; then
-            echo "::error::shellcheck is not installed on the runner"
-            exit 1
-          fi
           find tests/e2e -type f -name '*.sh' -print0 \
             | xargs -0 shellcheck --severity=warning
 
   canvas-deploy-reminder:
     name: Canvas Deploy Reminder
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     needs: [changes, canvas-build]
     # Only fires on direct pushes to main (i.e. after staging→main promotion).
     if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -186,26 +178,21 @@ jobs:
     name: Python Lint & Test
     needs: changes
     if: needs.changes.outputs.python == 'true'
-    runs-on: [self-hosted, macos, arm64]
+    runs-on: ubuntu-latest
     env:
       WORKSPACE_ID: test
     defaults:
       run:
         working-directory: workspace
     steps:
       - uses: actions/checkout@v4
-      # setup-python@v5 cannot write to /Users/runner (GitHub-hosted path) on
-      # the self-hosted macOS arm64 runner (user: <runner-user>) and also hits
-      # EACCES on /usr/local/bin due to macOS SIP. Skip it — Homebrew installs
-      # Python 3.11 at /opt/homebrew/opt/python@3.11 which is already on PATH.
-      - name: Verify Python 3.11 (Homebrew)
-        run: |
-          export PATH="/opt/homebrew/opt/python@3.11/bin:/opt/homebrew/bin:$PATH"
-          python3.11 --version
-          echo "/opt/homebrew/opt/python@3.11/bin" >> "$GITHUB_PATH"
-          echo "/opt/homebrew/bin" >> "$GITHUB_PATH"
-      - run: pip3.11 install -r requirements.txt pytest pytest-asyncio pytest-cov
-      - run: python3.11 -m pytest --tb=short -q --cov=. --cov-report=term-missing
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov
+      - run: python -m pytest --tb=short -q --cov=. --cov-report=term-missing
 
       # SDK + plugin validation moved to standalone repo:
       # github.com/Molecule-AI/molecule-sdk-python