feat(workspace-server): GHCR digest watcher closes runtime CD chain

docs/workspace-runtime-package.md

@@ -183,6 +183,20 @@ needs Docker socket access (the compose stack mounts
 (`docker login ghcr.io` once per host). On a fresh host without GHCR auth,
 the pull step warns per runtime and the response surfaces the failures.
 
+**Fully hands-off (opt-in image auto-refresh):**
+
+Set `IMAGE_AUTO_REFRESH=true` on the platform process. A watcher polls
+GHCR every 5 minutes for digest changes on each `workspace-template-*:latest`
+tag and invokes the same refresh logic the admin endpoint exposes —
+no operator action required between "runtime PR merged" and
+"containers running new code". Disabled by default because SaaS deploy
+pipelines that already pull on every release would do redundant work.
+
+Optional companion env (same as the admin endpoint):
+
+- `GHCR_USER` + `GHCR_TOKEN` — required for private template images;
+  unused for the current public set, but harmless if set.
+
 ## Local dev (build the package without publishing)
 
 ```bash

workspace-server/cmd/server/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
@@ -16,6 +17,7 @@ import (
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/db"
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/events"
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/handlers"
+	"github.com/Molecule-AI/molecule-monorepo/platform/internal/imagewatch"
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/provisioner"
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/registry"
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/router"
@@ -265,6 +267,18 @@ func main() {
 	channelMgr := channels.NewManager(wh, broadcaster)
 	go supervised.RunWithRecover(ctx, "channel-manager", channelMgr.Start)
 
+	// Image auto-refresh — closes the runtime CD chain to "merge → containers
+	// running new code" with no human in between. Polls GHCR for digest
+	// changes on workspace-template-* :latest tags and invokes the same
+	// refresh logic /admin/workspace-images/refresh exposes. Opt-in:
+	// SaaS deploys whose pipeline already pulls every release should leave
+	// it off (would be redundant work). Self-hosters get true zero-touch.
+	if prov != nil && strings.EqualFold(os.Getenv("IMAGE_AUTO_REFRESH"), "true") {
+		svc := handlers.NewWorkspaceImageService(prov.DockerClient())
+		watcher := imagewatch.New(svc)
+		go supervised.RunWithRecover(ctx, "image-auto-refresh", watcher.Run)
+	}
+
 	// Wire channel manager into scheduler for auto-posting cron output to Slack
 	cronSched.SetChannels(channelMgr)
 

workspace-server/internal/handlers/admin_workspace_images.go

@@ -21,54 +21,53 @@ import (
 	"github.com/Molecule-AI/molecule-monorepo/platform/internal/provisioner"
 )
 
-// AdminWorkspaceImagesHandler serves POST /admin/workspace-images/refresh — the
-// production-side end of the runtime CD chain. Operators (or post-publish
-// automation) hit this to (1) pull the latest workspace template images from
-// GHCR via the Docker SDK and (2) recreate any running ws-* containers so
-// they adopt the new image. Without this, a freshly-published runtime sits
-// in the registry but containers keep running the old image until the next
-// manual restart.
+// WorkspaceImageService is the production-side end of the runtime CD chain.
+// It (1) pulls workspace template images from GHCR via the Docker SDK and
+// (2) recreates running ws-* containers so they adopt the new image.
 //
-// On a SaaS deployment the deploy pipeline already pulls on every release,
-// so the pull step is a no-op there; the recreate step is still the way to
-// make running workspaces adopt the new image without a full host restart.
-//
-// POST /admin/workspace-images/refresh
-//
-//	?runtime=claude-code   (optional; default = all 8 templates)
-//	&recreate=true|false   (default true; false = pull only)
-//
-// Returns JSON {pulled: [...], failed: [...], recreated: [...]}
-type AdminWorkspaceImagesHandler struct {
+// Two callers:
+//   - AdminWorkspaceImagesHandler — POST /admin/workspace-images/refresh, the
+//     manual end-of-chain trigger documented in
+//     docs/workspace-runtime-package.md.
+//   - imagewatch.Watcher — the auto-refresh goroutine that polls GHCR
+//     digests and invokes Refresh when an image changes upstream. This is
+//     what closes the chain to "merge → containers running new code" with
+//     no human in between.
+type WorkspaceImageService struct {
 	docker *dockerclient.Client
 }
 
-func NewAdminWorkspaceImagesHandler(docker *dockerclient.Client) *AdminWorkspaceImagesHandler {
-	return &AdminWorkspaceImagesHandler{docker: docker}
+func NewWorkspaceImageService(docker *dockerclient.Client) *WorkspaceImageService {
+	return &WorkspaceImageService{docker: docker}
 }
 
-// allRuntimes is the canonical list mirroring docs/workspace-runtime-package.md.
+// AllRuntimes is the canonical list mirroring docs/workspace-runtime-package.md.
 // Update both when a new template is added.
-var allRuntimes = []string{
+var AllRuntimes = []string{
 	"claude-code", "langgraph", "crewai", "autogen",
 	"deepagents", "hermes", "gemini-cli", "openclaw",
 }
 
-type refreshResult struct {
+// RefreshResult is the per-call outcome surfaced to HTTP callers AND logged
+// by the auto-refresh watcher.
+type RefreshResult struct {
 	Pulled    []string `json:"pulled"`
 	Failed    []string `json:"failed"`
 	Recreated []string `json:"recreated"`
 }
 
+// TemplateImageRef returns the canonical GHCR ref for a runtime's template
+// image. Single source of truth shared with imagewatch.
+func TemplateImageRef(runtime string) string {
+	return fmt.Sprintf("ghcr.io/molecule-ai/workspace-template-%s:latest", runtime)
+}
+
 // ghcrAuthHeader returns the base64-encoded JSON auth payload Docker's
 // ImagePull expects in PullOptions.RegistryAuth, or empty string when no
 // GHCR_USER/GHCR_TOKEN env is set (lets public images pull through).
 //
 // The Docker SDK doesn't read ~/.docker/config.json — every authenticated
-// pull needs an explicit RegistryAuth string. Format per the Docker
-// engine API: {"username":"…","password":"…","serveraddress":"ghcr.io"}
-// → base64-encoded JSON with no trailing padding stripped (engine handles
-// either form).
+// pull needs an explicit RegistryAuth string.
 func ghcrAuthHeader() string {
 	user := strings.TrimSpace(os.Getenv("GHCR_USER"))
 	token := strings.TrimSpace(os.Getenv("GHCR_TOKEN"))
@@ -82,63 +81,40 @@ func ghcrAuthHeader() string {
 	}
 	js, err := json.Marshal(payload)
 	if err != nil {
-		// Should be unreachable for a static map[string]string. Log so a
-		// future contributor adding a non-marshallable field notices.
 		log.Printf("workspace-images: failed to marshal GHCR auth: %v", err)
 		return ""
 	}
 	return base64.URLEncoding.EncodeToString(js)
 }
 
-func (h *AdminWorkspaceImagesHandler) Refresh(c *gin.Context) {
-	runtimes := allRuntimes
-	if r := c.Query("runtime"); r != "" {
-		// Accept a single runtime; reject anything not in the canonical list
-		// so a typo doesn't silently no-op.
-		found := false
-		for _, known := range allRuntimes {
-			if known == r {
-				found = true
-				break
-			}
-		}
-		if !found {
-			c.JSON(http.StatusBadRequest, gin.H{
-				"error":          fmt.Sprintf("unknown runtime: %s", r),
-				"known_runtimes": allRuntimes,
-			})
-			return
-		}
-		runtimes = []string{r}
-	}
-	recreate := c.DefaultQuery("recreate", "true") == "true"
-
-	res := refreshResult{Pulled: []string{}, Failed: []string{}, Recreated: []string{}}
+// Refresh pulls the requested runtimes' template images from GHCR and (if
+// recreate) force-removes any matching ws-* containers so the platform
+// re-provisions them on next interaction.
+//
+// Soft-fails per runtime: one missing image (e.g. unpublished template)
+// doesn't abort the others. Per-runtime failures are in RefreshResult.Failed.
+// Returns a non-nil error only when the recreate phase couldn't enumerate
+// containers at all (caller should surface that as 500).
+func (s *WorkspaceImageService) Refresh(ctx context.Context, runtimes []string, recreate bool) (RefreshResult, error) {
+	res := RefreshResult{Pulled: []string{}, Failed: []string{}, Recreated: []string{}}
 	auth := ghcrAuthHeader()
 
-	// 1. Pull each template image via the Docker SDK. Soft-fail per-runtime
-	//    so one missing image (e.g. unpublished template) doesn't abort
-	//    the others. Each pull's progress stream is drained to completion
-	//    — the engine treats early-close as "abandon", leaving partial
-	//    layers around with no reference.
-	pullCtx, cancel := context.WithTimeout(c.Request.Context(), 5*time.Minute)
+	pullCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()
 	for _, rt := range runtimes {
-		image := fmt.Sprintf("ghcr.io/molecule-ai/workspace-template-%s:latest", rt)
+		image := TemplateImageRef(rt)
 		opts := dockerimage.PullOptions{Platform: provisioner.DefaultImagePlatform()}
 		if auth != "" {
 			opts.RegistryAuth = auth
 		}
-		rc, err := h.docker.ImagePull(pullCtx, image, opts)
+		rc, err := s.docker.ImagePull(pullCtx, image, opts)
 		if err != nil {
 			log.Printf("workspace-images/refresh: pull %s failed: %v", rt, err)
 			res.Failed = append(res.Failed, rt)
 			continue
 		}
-		// Drain to completion. We discard progress payload because no
-		// caller renders it; the platform log already records pulled/failed
-		// per runtime. If a future caller wants live progress, decode the
-		// JSON-line stream into events here.
+		// Drain to completion. The engine treats early-close as "abandon",
+		// leaving partial layers around with no reference.
 		if _, err := io.Copy(io.Discard, rc); err != nil {
 			rc.Close()
 			log.Printf("workspace-images/refresh: drain %s failed: %v", rt, err)
@@ -150,23 +126,18 @@ func (h *AdminWorkspaceImagesHandler) Refresh(c *gin.Context) {
 	}
 
 	if !recreate {
-		c.JSON(http.StatusOK, res)
-		return
+		return res, nil
 	}
 
-	// 2. Find ws-* containers running an image we just pulled. Recreate
-	//    them — kill+remove and let the platform's normal provisioning
-	//    flow re-create on next canvas interaction.
-	listCtx, listCancel := context.WithTimeout(c.Request.Context(), 30*time.Second)
+	listCtx, listCancel := context.WithTimeout(ctx, 30*time.Second)
 	defer listCancel()
-	containers, err := h.docker.ContainerList(listCtx, container.ListOptions{
+	containers, err := s.docker.ContainerList(listCtx, container.ListOptions{
 		All:     true,
 		Filters: filters.NewArgs(filters.Arg("name", "ws-")),
 	})
 	if err != nil {
 		log.Printf("workspace-images/refresh: container list failed: %v", err)
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "container list failed", "partial_result": res})
-		return
+		return res, fmt.Errorf("container list: %w", err)
 	}
 
 	pulledSet := map[string]struct{}{}
@@ -175,14 +146,10 @@ func (h *AdminWorkspaceImagesHandler) Refresh(c *gin.Context) {
 	}
 	for _, ctr := range containers {
 		// ContainerList's ctr.Image is the *resolved digest* (sha256:…),
-		// not the human-readable tag. Use ContainerInspect to get the
-		// original Config.Image (e.g. "ghcr.io/molecule-ai/workspace-
-		// template-claude-code:latest") so we can match against the
-		// pulled-runtime set. The cost is one extra round-trip per
-		// ws-* container — there are at most 8 typically, so this is
-		// well below any UX threshold.
-		inspectCtx, inspectCancel := context.WithTimeout(c.Request.Context(), 10*time.Second)
-		full, err := h.docker.ContainerInspect(inspectCtx, ctr.ID)
+		// not the human-readable tag. Inspect to get Config.Image so we
+		// can match against the pulled-runtime set.
+		inspectCtx, inspectCancel := context.WithTimeout(ctx, 10*time.Second)
+		full, err := s.docker.ContainerInspect(inspectCtx, ctr.ID)
 		inspectCancel()
 		if err != nil {
 			log.Printf("workspace-images/refresh: inspect %s failed: %v", ctr.ID[:12], err)
@@ -203,25 +170,69 @@ func (h *AdminWorkspaceImagesHandler) Refresh(c *gin.Context) {
 			continue
 		}
 		name := strings.TrimPrefix(ctr.Names[0], "/")
-		// Remove with force — the workspace will re-provision on the next
-		// canvas interaction. This drops in-flight conversations on the
-		// removed container; document via the response so callers can
-		// schedule the refresh during a quiet window.
-		rmCtx, rmCancel := context.WithTimeout(c.Request.Context(), 30*time.Second)
-		err = h.docker.ContainerRemove(rmCtx, ctr.ID, container.RemoveOptions{Force: true})
+		rmCtx, rmCancel := context.WithTimeout(ctx, 30*time.Second)
+		err = s.docker.ContainerRemove(rmCtx, ctr.ID, container.RemoveOptions{Force: true})
 		rmCancel()
 		if err != nil {
 			log.Printf("workspace-images/refresh: remove %s failed: %v", name, err)
 			continue
 		}
 		res.Recreated = append(res.Recreated, name)
 	}
+	return res, nil
+}
+
+// AdminWorkspaceImagesHandler serves POST /admin/workspace-images/refresh.
+//
+//	?runtime=claude-code   (optional; default = all 8 templates)
+//	&recreate=true|false   (default true; false = pull only)
+//
+// Returns JSON {pulled: [...], failed: [...], recreated: [...]}
+type AdminWorkspaceImagesHandler struct {
+	svc *WorkspaceImageService
+}
+
+func NewAdminWorkspaceImagesHandler(docker *dockerclient.Client) *AdminWorkspaceImagesHandler {
+	return &AdminWorkspaceImagesHandler{svc: NewWorkspaceImageService(docker)}
+}
+
+// Service exposes the underlying refresh logic so the auto-refresh watcher
+// in cmd/server can share the exact code path the HTTP handler uses.
+func (h *AdminWorkspaceImagesHandler) Service() *WorkspaceImageService {
+	return h.svc
+}
+
+func (h *AdminWorkspaceImagesHandler) Refresh(c *gin.Context) {
+	runtimes := AllRuntimes
+	if r := c.Query("runtime"); r != "" {
+		found := false
+		for _, known := range AllRuntimes {
+			if known == r {
+				found = true
+				break
+			}
+		}
+		if !found {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error":          fmt.Sprintf("unknown runtime: %s", r),
+				"known_runtimes": AllRuntimes,
+			})
+			return
+		}
+		runtimes = []string{r}
+	}
+	recreate := c.DefaultQuery("recreate", "true") == "true"
 
+	res, err := h.svc.Refresh(c.Request.Context(), runtimes, recreate)
 	authStatus := "no GHCR auth (public images only)"
-	if auth != "" {
+	if ghcrAuthHeader() != "" {
 		authStatus = "GHCR_USER/GHCR_TOKEN auth"
 	}
 	log.Printf("workspace-images/refresh: pulled=%d failed=%d recreated=%d (%s)",
 		len(res.Pulled), len(res.Failed), len(res.Recreated), authStatus)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "partial_result": res})
+		return
+	}
 	c.JSON(http.StatusOK, res)
 }