staging → main: e2e teardown patience (#2201) one-time bridge

.github/workflows/publish-runtime.yml

@@ -79,6 +79,7 @@ jobs:
       id-token: write   # PyPI Trusted Publisher (OIDC) — no PYPI_TOKEN needed
     outputs:
       version: ${{ steps.version.outputs.version }}
+      wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -129,6 +130,28 @@ jobs:
         working-directory: ${{ runner.temp }}/runtime-build
         run: python -m build
 
+      - name: Capture wheel SHA256 for cascade content-verification
+        # Recorded BEFORE upload so the cascade probe can verify the
+        # bytes Fastly serves under the new version's URL match what
+        # we built. Closes a hole left by #2197: that probe verified
+        # pip can resolve the version (catches propagation lag) but
+        # not that the wheel content matches (would silently pass a
+        # Fastly stale-content scenario where the new version's URL
+        # serves an old wheel binary).
+        id: wheel_hash
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: |
+          set -eu
+          WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
+          if [ -z "$WHEEL" ]; then
+            echo "::error::No .whl in dist/ — `python -m build` must have failed silently"
+            exit 1
+          fi
+          HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
+          echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
+          echo "Local wheel SHA256 (pre-upload): ${HASH}"
+          echo "Wheel filename: $(basename "$WHEEL")"
+
       - name: Verify package contents (sanity)
         working-directory: ${{ runner.temp }}/runtime-build
         run: |
@@ -212,6 +235,56 @@ jobs:
               default_output_modes=['text/plain', 'application/json'],
           )
           print('✓ AgentCard call-shape smoke passed')
+
+          # Well-known agent-card path probe alignment. main.py's
+          # _send_initial_prompt() polls AGENT_CARD_WELL_KNOWN_PATH
+          # to know when the local A2A server is ready. If the SDK
+          # ever splits the constant value from the path that
+          # create_agent_card_routes() actually mounts at, every
+          # workspace silently drops its initial_prompt:
+          #   - Probe gets 404 every attempt.
+          #   - Falls through to 'server not ready after 30s,
+          #     skipping' even though the server is fine.
+          #   - The user hits a fresh chat with no kickoff context.
+          # This was the #2193 incident class — the v0.x → v1.x
+          # rename of /.well-known/agent.json → /.well-known/agent-card.json
+          # plus the constant itself moving to a2a.utils.constants.
+          # source-tree pytest (test_agent_card_well_known_path.py)
+          # catches main.py-side regressions; this catches the
+          # SDK-side ones BEFORE PyPI upload.
+          from a2a.utils.constants import AGENT_CARD_WELL_KNOWN_PATH
+          from a2a.server.routes import create_agent_card_routes
+          mounted_paths = [
+              getattr(r, 'path', None)
+              for r in create_agent_card_routes(
+                  AgentCard(
+                      name='wk-smoke',
+                      description='well-known mount alignment',
+                      version='0.0.0-smoke',
+                  )
+              )
+          ]
+          assert AGENT_CARD_WELL_KNOWN_PATH in mounted_paths, (
+              f'AGENT_CARD_WELL_KNOWN_PATH ({AGENT_CARD_WELL_KNOWN_PATH!r}) '
+              f'is NOT among paths mounted by create_agent_card_routes '
+              f'({mounted_paths!r}). The SDK constant and its own route '
+              f'factory have drifted — workspace probes will 404 forever, '
+              f'silently dropping every workspace initial_prompt.'
+          )
+          print(f'✓ well-known mount alignment OK ({AGENT_CARD_WELL_KNOWN_PATH})')
+
+          # Message helper smoke. a2a-sdk renamed
+          # new_agent_text_message → new_text_message in the v1.x
+          # protobuf-flat migration (per the v0→v1 cheat sheet). main.py
+          # and a2a_executor.py call new_text_message in hot paths; if
+          # the import breaks, every reply errors with ImportError before
+          # the message even leaves the workspace. Importing here
+          # catches a future v2.x rename at publish time.
+          from a2a.helpers import new_text_message
+          msg = new_text_message('smoke')
+          assert msg is not None, 'new_text_message returned None'
+          print('✓ message helper import + call OK')
+
           print('✓ smoke import passed')
           "
 
@@ -239,28 +312,98 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Wait for PyPI to propagate the new version
-        # PyPI accepts the upload, then takes a few seconds to make it
-        # available via the package index. If the cascade fires too
-        # fast, downstream template builds run `pip install` against
-        # an index that hasn't seen the new version yet — they resolve
-        # to the previous one, and docker layer cache then locks that
-        # in for subsequent rebuilds (the cache trap that bit us five
-        # times tonight).
+        # PyPI accepts the upload, then takes a few seconds to make the
+        # new version visible across all THREE surfaces pip touches:
+        #   1. /pypi/<pkg>/<ver>/json — metadata endpoint
+        #   2. /simple/<pkg>/         — pip's primary download index
+        #   3. files.pythonhosted.org — CDN-fronted wheel binary
+        # Each has its own cache. The previous check polled only (1)
+        # and would let the cascade fire while (2) or (3) still served
+        # the previous version, so downstream `pip install` resolved
+        # to the old wheel. Docker layer cache then locked that stale
+        # resolution in for subsequent rebuilds (the cache trap that
+        # bit us five times in one night).
         #
-        # Poll PyPI's JSON API for up to 60s. Cheap (~50ms per poll),
-        # avoids over-trusting "publish job said success."
+        # Two-stage probe per poll:
+        #   (a) `pip install --no-cache-dir PACKAGE==VERSION` — succeeds
+        #       only when the version is resolvable. Catches surface (1)
+        #       and (2) propagation lag.
+        #   (b) `pip download` of the same wheel + SHA256 compare against
+        #       the just-built dist's hash. Catches surface (3) lag AND
+        #       Fastly serving stale content under the new version's URL
+        #       (a separate Fastly-corruption mode that pip-install alone
+        #       can't see, since pip install resolves+unpacks against
+        #       whatever bytes Fastly returns and never inspects them).
+        # Both must pass before the cascade fans out.
+        #
+        # The venv is reused across polls; only `pip install`/`pip
+        # download` run in the loop, with --force-reinstall +
+        # --no-cache-dir so the previous poll's cached state doesn't
+        # mask propagation lag.
         env:
           RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
+          EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
         run: |
           set -eu
+          if [ -z "$EXPECTED_SHA256" ]; then
+            echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
+            exit 1
+          fi
+          python -m venv /tmp/propagation-probe
+          PROBE=/tmp/propagation-probe/bin
+          $PROBE/pip install --upgrade --quiet pip
+          # Poll budget: 30 attempts × (~3-5s pip install + ~3s pip
+          # download + 4s sleep) ≈ 5-6 min wall on a slow GH runner.
+          # Generous vs PyPI's typical few-seconds propagation;
+          # failures past this are signal of a real PyPI / Fastly
+          # issue, not just lag.
           for i in $(seq 1 30); do
-            if curl -fsS "https://pypi.org/pypi/molecule-ai-workspace-runtime/${RUNTIME_VERSION}/json" >/dev/null 2>&1; then
-              echo "::notice::✓ PyPI serving ${RUNTIME_VERSION} after ${i} polls"
-              exit 0
+            # Stage (a): can pip resolve and install the version?
+            if $PROBE/pip install \
+                  --quiet \
+                  --no-cache-dir \
+                  --force-reinstall \
+                  --no-deps \
+                  "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+                  >/dev/null 2>&1; then
+              INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
+                          | awk -F': ' '/^Version:/{print $2}')
+              if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
+                # Stage (b): does Fastly serve the bytes we uploaded?
+                # `pip download` writes the actual .whl file to disk so
+                # we can sha256sum it (vs `pip install` which unpacks
+                # and discards).
+                rm -rf /tmp/probe-dl
+                mkdir -p /tmp/probe-dl
+                if $PROBE/pip download \
+                      --quiet \
+                      --no-cache-dir \
+                      --no-deps \
+                      --dest /tmp/probe-dl \
+                      "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+                      >/dev/null 2>&1; then
+                  WHEEL=$(ls /tmp/probe-dl/*.whl 2>/dev/null | head -1)
+                  if [ -n "$WHEEL" ]; then
+                    ACTUAL=$(sha256sum "$WHEEL" | awk '{print $1}')
+                    if [ "$ACTUAL" = "$EXPECTED_SHA256" ]; then
+                      echo "::notice::✓ pip resolves AND wheel content matches after ${i} poll(s) (sha256=${EXPECTED_SHA256})"
+                      exit 0
+                    fi
+                    # Hash mismatch: PyPI accepted our upload but Fastly
+                    # is serving different bytes under the version's URL.
+                    # Most often this is propagation lag of the BINARY
+                    # surface — the version is resolvable but the wheel
+                    # cache hasn't caught up. Retry.
+                    echo "::warning::poll ${i}: wheel content mismatch (got ${ACTUAL:0:12}…, want ${EXPECTED_SHA256:0:12}…) — Fastly likely still serving stale binary, retrying"
+                  fi
+                fi
+              fi
             fi
-            sleep 2
+            sleep 4
           done
-          echo "::error::PyPI never propagated ${RUNTIME_VERSION} within 60s — refusing to fan out cascade against stale index"
+          echo "::error::pip never resolved molecule-ai-workspace-runtime==${RUNTIME_VERSION} with matching wheel content within ~5 min."
+          echo "::error::Expected wheel SHA256: ${EXPECTED_SHA256}"
+          echo "::error::Refusing to fan out cascade against stale or corrupt PyPI surfaces."
           exit 1
 
       - name: Fan out repository_dispatch

.github/workflows/runtime-pin-compat.yml

@@ -10,15 +10,29 @@ name: Runtime Pin Compatibility
 #   4. Every tenant workspace crashed; the canary tenant caught it but
 #      only after 5 hours of degraded staging
 #
-# This workflow installs the runtime in a fresh Python venv from PyPI
-# and tries the same import the EC2 user-data does. If pip resolution
-# silently produces a broken combo, this gate fails before the tenant
-# image gets published.
+# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
+# top of `workspace/requirements.txt` and smoke-imports. Catches:
+#   - Upstream PyPI yanks
+#   - Bad re-releases of molecule-ai-workspace-runtime
+#   - Already-shipped wheels that stop importing because a transitive
+#     dep moved underneath
+#
+# This is the "PyPI artifact health" half of pin compatibility. The
+# companion workflow `runtime-prbuild-compat.yml` covers the
+# "PR-introduced breakage" half by building the wheel from THIS PR's
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter — the pypi-latest job no longer fires on doc-only
+# workspace/ edits whose content can't change what's currently on PyPI.
 
 on:
   push:
     branches: [main, staging]
     paths:
+      # Narrow filter: pypi-latest is sensitive only to changes that
+      # affect what we're INSTALLING (requirements.txt) or WHAT THE
+      # CHECK ITSELF DOES (this workflow file). Edits to workspace/
+      # source code don't change what's on PyPI right now, so they
+      # don't change this gate's verdict.
       - 'workspace/requirements.txt'
       - '.github/workflows/runtime-pin-compat.yml'
   pull_request:
@@ -42,8 +56,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  default-install:
-    name: Default install + import smoke
+  pypi-latest-install:
+    name: PyPI-latest install + import smoke
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/runtime-prbuild-compat.yml

@@ -0,0 +1,100 @@
+name: Runtime PR-Built Compatibility
+
+# Companion to `runtime-pin-compat.yml`. That workflow tests what's
+# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
+# PUBLISHED if THIS PR merges.
+#
+# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
+# wheel" job to the original runtime-pin-compat.yml, but both jobs
+# shared a `paths:` filter that was the union of their needs
+# (`workspace/**`). That meant the PyPI-latest job ran on every doc
+# edit even though the upstream PyPI artifact can't change with our
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter that matches the inputs it actually depends on.
+#
+# Catches the failure mode where a PR adds an import requiring a newer
+# SDK than `workspace/requirements.txt` pins:
+#   1. Pip resolves the existing PyPI wheel + the old SDK pin → smoke
+#      passes (it imports the OLD main.py from the wheel, not the PR's
+#      new main.py).
+#   2. Merge → publish-runtime.yml ships a wheel WITH the new import.
+#   3. Tenant images redeploy → all crash on first boot with
+#      ImportError.
+#
+# By building from the PR's source and smoke-importing THAT wheel, we
+# fail at PR-time instead of after publish.
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      # Broad filter: this workflow's verdict can change whenever any
+      # workspace/ source file changes (because the wheel we build is
+      # produced from those files), or when the build script itself
+      # changes (it controls the wheel layout).
+      - 'workspace/**'
+      - 'scripts/build_runtime_package.py'
+      - '.github/workflows/runtime-prbuild-compat.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace/**'
+      - 'scripts/build_runtime_package.py'
+      - '.github/workflows/runtime-prbuild-compat.yml'
+  workflow_dispatch:
+  # Required-check support: when this becomes a branch-protection gate,
+  # merge_group runs let the queue green-check this in addition to PRs.
+  merge_group:
+    types: [checks_requested]
+  # No cron: the same pre-merge run already covered the commit, and
+  # re-running daily wouldn't surface anything new (workspace/ doesn't
+  # change between cron firings unless a PR already passed this gate).
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  local-build-install:
+    # Builds the wheel from THIS PR's workspace/ + scripts/ and tests
+    # IT — the artifact that WOULD be published if this PR merges.
+    name: PR-built wheel + import smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install build tooling
+        run: pip install build
+      - name: Build wheel from PR source (mirrors publish-runtime.yml)
+        # Use a fixed test version so the wheel filename is predictable.
+        # Doesn't reach PyPI — this build is local-only for the smoke.
+        # Use the SAME build script with the SAME args as
+        # publish-runtime.yml's build step. The temp dir path differs
+        # (`/tmp/runtime-build` here vs `${{ runner.temp }}/runtime-build`
+        # in publish-runtime.yml — they coincide on ubuntu-latest but
+        # the call sites are not byte-identical). The smoke import is
+        # also intentionally narrower than publish's: this gate exists
+        # to catch SDK-version-import drift specifically; full invariant
+        # coverage lives in publish-runtime.yml's own pre-PyPI smoke.
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "0.0.0.dev0+pin-compat" \
+            --out /tmp/runtime-build
+          cd /tmp/runtime-build && python -m build
+      - name: Install built wheel + workspace requirements
+        run: |
+          python -m venv /tmp/venv-built
+          /tmp/venv-built/bin/pip install --upgrade pip
+          /tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
+          /tmp/venv-built/bin/pip install -r workspace/requirements.txt
+          /tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import the PR-built wheel
+        env:
+          WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
+        run: |
+          /tmp/venv-built/bin/python -c "from molecule_runtime.main import main_sync; print('PR-built runtime imports OK')"

docs/ecosystem-watch.md

@@ -1,6 +1,6 @@
 # Ecosystem Watch — Phase 30 Competitive Tracking
 **Created by:** PMM
-**Date:** 2026-04-21
+**Date:** 2026-04-27
 **Status:** ACTIVE — competitor monitoring in progress
 **Phase:** 30 — Remote Workspaces + Cross-Network Federation
 
@@ -118,7 +118,7 @@ Track competitor releases and market events that affect Phase 30 positioning. En
 - **Check frequency:** Every marketing cycle
 - **Trigger:** Any competitor shipping something that invalidates a Phase 30 positioning claim
 - **File location:** `docs/ecosystem-watch.md` (origin/main)
-- **Last updated by:** PMM | 2026-04-23 (LangGraph PRs verified OPEN; new feat PRs #1730/#1702/#1731 logged; release note written)
+- **Last updated by:** PMM | 2026-04-27 (weekly refresh — all competitor versions, stars, and PRs verified current; Clawith v1.9.0 added)
 
 ---