staging → main: auto-promote cdef893

.github/workflows/auto-promote-staging.yml

@@ -267,6 +267,32 @@ jobs:
           echo "promote_pr_num=${PR_NUM}" >> "$GITHUB_OUTPUT"
         id: promote_pr
 
+      # Mint a short-lived GitHub App installation token for the dispatch
+      # step below. We CANNOT use `secrets.GITHUB_TOKEN` to dispatch the
+      # downstream publish chain — workflow runs created by GITHUB_TOKEN
+      # do not fire `workflow_run` triggers on completion (the
+      # documented "no recursion" rule —
+      # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow).
+      #
+      # Symptom this caused (root-caused on 2026-04-30): publish-image
+      # ran successfully twice (21313dc 14:41Z, 59dec57 15:21Z) but
+      # canary-verify and redeploy-tenants-on-main never chained,
+      # because the publish run's `triggering_actor` was
+      # `github-actions[bot]` (i.e. GITHUB_TOKEN). A manual dispatch
+      # earlier in the day with the operator's PAT (d850ec7 06:52Z) did
+      # chain — same workflow file, only the actor differed.
+      #
+      # An App token's triggering_actor is the App user (e.g.
+      # `molecule-ai[bot]`), which IS allowed to fire downstream
+      # workflow_run cascades.
+      - name: Mint App token for downstream dispatch
+        if: steps.promote_pr.outputs.promote_pr_num != ''
+        id: app-token
+        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
+        with:
+          app-id: ${{ secrets.MOLECULE_AI_APP_ID }}
+          private-key: ${{ secrets.MOLECULE_AI_APP_PRIVATE_KEY }}
+
       - name: Wait for promote merge, then dispatch publish + redeploy (#2357)
         # GITHUB_TOKEN-initiated merges suppress downstream `push` events
         # (https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow).
@@ -276,18 +302,20 @@ jobs:
         # tenants stay on stale code (issue #2357).
         #
         # Workaround: poll for the merge to land, then explicitly
-        # `gh workflow run` publish-workspace-server-image. workflow_dispatch
-        # is the documented exception to the GITHUB_TOKEN suppression rule —
-        # dispatch DOES create a new workflow run. canary-verify chains via
-        # workflow_run (no branch filter) and redeploys to fleet via the
-        # existing chain.
+        # `gh workflow run` publish-workspace-server-image. The dispatch
+        # MUST authenticate as the molecule-ai App (App token minted
+        # above) — not GITHUB_TOKEN — so that the resulting publish
+        # run's completion event can fire the workflow_run cascade
+        # into canary-verify + redeploy-tenants-on-main. See the prior
+        # step's comment for the GITHUB_TOKEN no-recursion details.
         #
-        # Long-term fix: switch the auto-merge call above to a GitHub App
-        # token (actions/create-github-app-token) and remove this polling
-        # tail step. Tracked in #2357.
+        # Long-term fix: switch the auto-merge call above to use the
+        # same App token, so the merge's push event fires
+        # publish-workspace-server-image naturally and this polling tail
+        # becomes unnecessary. Tracked in #2357.
         if: steps.promote_pr.outputs.promote_pr_num != ''
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
           REPO: ${{ github.repository }}
           PR_NUM: ${{ steps.promote_pr.outputs.promote_pr_num }}
         run: |
@@ -318,17 +346,18 @@ jobs:
             exit 0
           fi
 
-          # Dispatch publish on main. workflow_dispatch via GITHUB_TOKEN
-          # IS allowed to create new workflow runs (per the linked docs).
+          # Dispatch publish on main using the App token. App-initiated
+          # workflow_dispatch DOES propagate the workflow_run cascade,
+          # unlike GITHUB_TOKEN-initiated dispatch.
           # publish completes → canary-verify chains via workflow_run →
           # redeploy-tenants-on-main chains via workflow_run + branches:[main].
           if gh workflow run publish-workspace-server-image.yml \
               --repo "$REPO" --ref main 2>&1; then
-            echo "::notice::Dispatched publish-workspace-server-image on ref=main — canary-verify and redeploy-tenants-on-main will chain via workflow_run."
+            echo "::notice::Dispatched publish-workspace-server-image on ref=main as molecule-ai App — canary-verify and redeploy-tenants-on-main will chain via workflow_run."
             {
               echo "## 🚀 Tenant redeploy chain dispatched"
               echo
-              echo "- publish-workspace-server-image (workflow_dispatch on \`main\`)"
+              echo "- publish-workspace-server-image (workflow_dispatch on \`main\`, actor: \`molecule-ai[bot]\`)"
               echo "- canary-verify will chain on completion"
               echo "- redeploy-tenants-on-main will chain on canary green"
             } >> "$GITHUB_STEP_SUMMARY"

workspace-server/internal/db/workspace_status_enum_drift_test.go

@@ -0,0 +1,227 @@
+package db_test
+
+// Static drift gate: every workspaces.status literal written in the Go
+// tree must exist in the workspace_status enum defined by the migrations.
+//
+// Why this exists: the `workspace_status` enum (migrations 043 + 046)
+// shipped without 'awaiting_agent' even though application code wrote
+// that value, and every UPDATE silently failed in production for five
+// days before the gap surfaced (see 046_workspace_status_awaiting_agent.up.sql).
+// The unit tests passed because sqlmock matches SQL by regex, not against
+// a live enum constraint.
+//
+// Approach: extract every Go string literal whose body matches
+// (?i)workspaces[^a-z_].*status (so "UPDATE workspaces SET status",
+// "FROM workspaces WHERE ... status", "INSERT INTO workspaces ... status",
+// CTEs that reference workspaces, etc.). For each such SQL fragment,
+// pull the single-quoted status values out of `status =`, `status IN`,
+// `THEN`, and `ELSE`. Every value must be in the union of CREATE TYPE +
+// ALTER TYPE ADD VALUE across all migrations.
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestWorkspaceStatusEnum_NoLiteralDrift(t *testing.T) {
+	t.Parallel()
+
+	repoRoot := findRepoRoot(t)
+	migrationsDir := filepath.Join(repoRoot, "workspace-server", "migrations")
+	internalDir := filepath.Join(repoRoot, "workspace-server", "internal")
+
+	enum := loadWorkspaceStatusEnum(t, migrationsDir)
+	if len(enum) == 0 {
+		t.Fatalf("could not parse workspace_status enum from %s — gate is non-functional", migrationsDir)
+	}
+
+	literals := collectWorkspacesStatusLiterals(t, internalDir)
+	if len(literals) == 0 {
+		t.Fatalf("found zero workspaces.status literals under %s — gate is non-functional", internalDir)
+	}
+
+	var rogue []string
+	for lit := range literals {
+		if _, ok := enum[lit]; ok {
+			continue
+		}
+		rogue = append(rogue, lit)
+	}
+	if len(rogue) > 0 {
+		sort.Strings(rogue)
+		t.Errorf(
+			"workspaces.status literal(s) %v are written by Go code but not in the workspace_status enum.\n"+
+				"Add a migration `ALTER TYPE workspace_status ADD VALUE 'X';` (see 046 for shape).\n"+
+				"Enum currently is: %v",
+			rogue, sortedKeys(enum),
+		)
+	}
+}
+
+// loadWorkspaceStatusEnum scans every *.up.sql file for either:
+//
+//	CREATE TYPE workspace_status AS ENUM ('a', 'b', ...)
+//	ALTER TYPE workspace_status ADD VALUE [IF NOT EXISTS] 'X' [BEFORE|AFTER 'Y']
+//
+// and returns the union of every value the enum will hold after all
+// migrations apply.
+func loadWorkspaceStatusEnum(t *testing.T, migrationsDir string) map[string]struct{} {
+	t.Helper()
+
+	out := make(map[string]struct{})
+
+	files, err := filepath.Glob(filepath.Join(migrationsDir, "*.up.sql"))
+	if err != nil {
+		t.Fatalf("glob migrations: %v", err)
+	}
+	sort.Strings(files)
+
+	createRE := regexp.MustCompile(`(?is)CREATE\s+TYPE\s+workspace_status\s+AS\s+ENUM\s*\(([^)]+)\)`)
+	addValueRE := regexp.MustCompile(`(?i)ALTER\s+TYPE\s+workspace_status\s+ADD\s+VALUE(?:\s+IF\s+NOT\s+EXISTS)?\s+'([^']+)'`)
+	literalRE := regexp.MustCompile(`'([^']+)'`)
+
+	for _, f := range files {
+		body, err := os.ReadFile(f)
+		if err != nil {
+			t.Fatalf("read %s: %v", f, err)
+		}
+		for _, m := range createRE.FindAllStringSubmatch(string(body), -1) {
+			for _, lit := range literalRE.FindAllStringSubmatch(m[1], -1) {
+				out[lit[1]] = struct{}{}
+			}
+		}
+		for _, m := range addValueRE.FindAllStringSubmatch(string(body), -1) {
+			out[m[1]] = struct{}{}
+		}
+	}
+	return out
+}
+
+// collectWorkspacesStatusLiterals walks every non-test .go file under
+// root, finds Go string literals that contain `UPDATE workspaces` or
+// `INSERT INTO workspaces`, and extracts the status literals appearing
+// inside the matching SQL statement.
+//
+// Why this scope: any UPDATE/INSERT against `workspaces` is the moment
+// a status literal hits the column constrained by the enum. Read-side
+// SQL (SELECT ... WHERE status = 'X') cannot fail on enum drift, so it's
+// out of scope. JOINs to `workspaces` from other tables (e.g. approvals
+// joining workspaces for display) write to a different table's status —
+// also out of scope. Anchoring on the leading `UPDATE workspaces` /
+// `INSERT INTO workspaces` keyword unambiguously identifies the writes
+// we care about.
+func collectWorkspacesStatusLiterals(t *testing.T, root string) map[string]struct{} {
+	t.Helper()
+
+	// Match raw-string and double-quoted Go string literals. Backtick
+	// strings can span multiple lines. Both forms are extracted via the
+	// same DOTALL regex over the whole file body.
+	rawRE := regexp.MustCompile("(?s)`([^`]*?)`")
+	dquoteRE := regexp.MustCompile(`"((?:[^"\\]|\\.)*)"`)
+
+	// A SQL string is in scope if it begins (after optional leading
+	// whitespace) with UPDATE workspaces or INSERT INTO workspaces.
+	// `(?i)` is case-insensitive; `\s*` allows the format-friendly
+	// leading newline and indent that the codebase uses.
+	updateWorkspacesRE := regexp.MustCompile(`(?is)^\s*UPDATE\s+workspaces\b`)
+	insertWorkspacesRE := regexp.MustCompile(`(?is)^\s*INSERT\s+INTO\s+workspaces\b`)
+
+	// Inside a scoped SQL fragment, status literals appear in:
+	//   status = 'X'           — assignment in SET (or filter in WHERE)
+	//   status IN ('X', ...)   — filter
+	//   status NOT IN ('X')    — filter
+	//   THEN 'X'               — CASE arm
+	//   ELSE 'X'               — CASE default
+	statusEqRE := regexp.MustCompile(`(?i)status\s*(?:=|!=|<>)\s*'([a-z_]+)'`)
+	statusInRE := regexp.MustCompile(`(?i)status\s+(?:NOT\s+)?IN\s*\(([^)]*)\)`)
+	thenRE := regexp.MustCompile(`(?i)THEN\s+'([a-z_]+)'`)
+	elseRE := regexp.MustCompile(`(?i)ELSE\s+'([a-z_]+)'`)
+	inListLiteralRE := regexp.MustCompile(`'([a-z_]+)'`)
+
+	out := make(map[string]struct{})
+
+	walkErr := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if !strings.HasSuffix(path, ".go") {
+			return nil
+		}
+		if strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+		body, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		text := string(body)
+
+		harvest := func(fragment string) {
+			if !updateWorkspacesRE.MatchString(fragment) && !insertWorkspacesRE.MatchString(fragment) {
+				return
+			}
+			for _, m := range statusEqRE.FindAllStringSubmatch(fragment, -1) {
+				out[m[1]] = struct{}{}
+			}
+			for _, m := range statusInRE.FindAllStringSubmatch(fragment, -1) {
+				for _, lit := range inListLiteralRE.FindAllStringSubmatch(m[1], -1) {
+					out[lit[1]] = struct{}{}
+				}
+			}
+			for _, m := range thenRE.FindAllStringSubmatch(fragment, -1) {
+				out[m[1]] = struct{}{}
+			}
+			for _, m := range elseRE.FindAllStringSubmatch(fragment, -1) {
+				out[m[1]] = struct{}{}
+			}
+		}
+
+		for _, m := range rawRE.FindAllStringSubmatch(text, -1) {
+			harvest(m[1])
+		}
+		for _, m := range dquoteRE.FindAllStringSubmatch(text, -1) {
+			harvest(m[1])
+		}
+		return nil
+	})
+	if walkErr != nil {
+		t.Fatalf("walk %s: %v", root, walkErr)
+	}
+	return out
+}
+
+func findRepoRoot(t *testing.T) string {
+	t.Helper()
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("getwd: %v", err)
+	}
+	for i := 0; i < 8; i++ {
+		if _, err := os.Stat(filepath.Join(dir, "workspace-server", "migrations")); err == nil {
+			return dir
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			break
+		}
+		dir = parent
+	}
+	t.Fatalf("could not locate repo root with workspace-server/migrations from %s", dir)
+	return ""
+}
+
+func sortedKeys(m map[string]struct{}) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}

workspace-server/internal/handlers/chat_files.go

@@ -259,24 +259,7 @@ func (h *ChatFilesHandler) Upload(c *gin.Context) {
 		req.ContentLength = c.Request.ContentLength
 	}
 
-	resp, err := h.httpClient.Do(req)
-	if err != nil {
-		log.Printf("chat_files Upload: forward to %s failed: %v", forwardURL, err)
-		c.JSON(http.StatusBadGateway, gin.H{"error": "workspace unreachable"})
-		return
-	}
-	defer resp.Body.Close()
-
-	// Stream response back. Copy headers we know are safe + the body.
-	if ct := resp.Header.Get("Content-Type"); ct != "" {
-		c.Header("Content-Type", ct)
-	}
-	c.Status(resp.StatusCode)
-	if _, err := io.Copy(c.Writer, resp.Body); err != nil {
-		// Mid-stream failure — too late to write a JSON error, just
-		// log so ops can correlate with the workspace's logs.
-		log.Printf("chat_files Upload: stream response back failed for %s: %v", workspaceID, err)
-	}
+	h.streamWorkspaceResponse(c, "upload", workspaceID, forwardURL, req, []string{"Content-Type"})
 }
 
 // Download handles GET /workspaces/:id/chat/download?path=<abs path>.
@@ -351,27 +334,42 @@ func (h *ChatFilesHandler) Download(c *gin.Context) {
 	}
 	req.Header.Set("Authorization", "Bearer "+secret)
 
+	h.streamWorkspaceResponse(c, "download", workspaceID, forwardURL, req,
+		[]string{"Content-Type", "Content-Length", "Content-Disposition"})
+}
+
+// streamWorkspaceResponse executes the prepared forward request and
+// streams the workspace's response back to the inbound caller.
+// Forwards the named response headers verbatim. Centralizes the
+// "do request → check err → defer close → copy headers → set status →
+// io.Copy" tail that's identical between Upload and Download.
+//
+// op is the human-readable feature label ("upload"/"download") used
+// in log messages so operators can distinguish which feature ran.
+func (h *ChatFilesHandler) streamWorkspaceResponse(
+	c *gin.Context,
+	op, workspaceID, forwardURL string,
+	req *http.Request,
+	forwardHeaders []string,
+) {
 	resp, err := h.httpClient.Do(req)
 	if err != nil {
-		log.Printf("chat_files Download: forward to %s failed: %v", forwardURL, err)
+		log.Printf("chat_files %s: forward to %s failed: %v", op, forwardURL, err)
 		c.JSON(http.StatusBadGateway, gin.H{"error": "workspace unreachable"})
 		return
 	}
 	defer resp.Body.Close()
 
-	// Stream response back, including the workspace's headers so the
-	// client gets the correct Content-Type + Content-Disposition (the
-	// workspace constructs them from the actual file's extension +
-	// basename — keeping that logic on the workspace side avoids a
-	// double-source-of-truth on filename encoding rules).
-	for _, hdr := range []string{"Content-Type", "Content-Length", "Content-Disposition"} {
+	for _, hdr := range forwardHeaders {
 		if v := resp.Header.Get(hdr); v != "" {
 			c.Header(hdr, v)
 		}
 	}
 	c.Status(resp.StatusCode)
 	if _, err := io.Copy(c.Writer, resp.Body); err != nil {
-		log.Printf("chat_files Download: stream response back failed for %s: %v", workspaceID, err)
+		// Mid-stream failure — too late to write a JSON error, just
+		// log so ops can correlate with the workspace's logs.
+		log.Printf("chat_files %s: stream response back failed for %s: %v", op, workspaceID, err)
 	}
 }