test(mv3): prove real downloads compatibility

scripts/ci/run_mv3_compatibility.py

@@ -5,7 +5,7 @@
 W3C WebDriver HTTP protocol only to prove that a real Chrome for Testing build
 can load the controlled MV3 fixture and repeatedly exercise service-worker,
 content-script, storage, declarative-net-request, tabs, windows, scripting,
-commands, side-panel, bookmarks, history, real browser-click, and
+commands, side-panel, bookmarks, history, downloads, real browser-click, and
 restart-persistence behavior.
 """
 
@@ -36,6 +36,54 @@
 MAX_WEBDRIVER_RESPONSE_BYTES = 1_048_576
 W3C_ELEMENT_KEY = "element-6066-11e4-a52e-4f735466cecf"
 PATH_TOKEN_CHARACTERS = frozenset(string.ascii_letters + string.digits + "-_.")
+SURFACE_EVIDENCE_KEYS = (
+    "content",
+    "storage",
+    "storagePersistence",
+    "workerReply",
+    "workerState",
+    "workerStartCount",
+    "dnr",
+    "tabs",
+    "windows",
+    "scripting",
+    "scriptingExecuted",
+    "commands",
+    "sidePanel",
+    "bookmarks",
+    "history",
+    "downloads",
+    "downloadsDiagnostic",
+)
+SURFACE_EVIDENCE_VALUES = frozenset(
+    {"ready", "missing", "initialized", "persisted", "pong", "installed", "blocked"}
+)
+DOWNLOAD_DIAGNOSTIC_VALUES = frozenset(
+    {
+        "download-source-rejected",
+        "download-start-rejected",
+        "download-search-missing",
+        "download-interrupted",
+        "download-url-mismatch",
+        "download-byte-count-mismatch",
+        "download-exists-false",
+        "download-timeout",
+        "download-complete-ready",
+        "download-not-evaluated",
+    }
+)
+
+
+class CompatibilitySurfaceError(RuntimeError):
+    """Report only bounded fixture-surface state when real-browser evidence does not converge."""
+
+    def __init__(self, observed: dict[str, str]) -> None:
+        self.observed = {
+            key: _safe_surface_value(key, observed[key])
+            for key in SURFACE_EVIDENCE_KEYS
+            if key in observed
+        }
+        super().__init__("Manifest V3 fixture surfaces did not converge")
 
 
 class QuietFixtureHandler(http.server.SimpleHTTPRequestHandler):
@@ -45,6 +93,30 @@ def log_message(self, _format: str, *args: object) -> None:
         """Suppress request logs because the fixture contains no diagnostic value."""
 
 
+def _safe_surface_value(key: str, value: str) -> str:
+    """Reduce one controlled DOM evidence value to a non-sensitive diagnostic token."""
+
+    if key == "workerStartCount":
+        return value if value.isdecimal() and len(value) <= 20 else "invalid"
+    if key == "downloadsDiagnostic":
+        return value if value in DOWNLOAD_DIAGNOSTIC_VALUES else "unexpected"
+    return value if value in SURFACE_EVIDENCE_VALUES else "unexpected"
+
+
+def _failure_evidence(error: BaseException) -> dict[str, Any]:
+    """Classify one browser-trial failure without retaining raw exception text."""
+
+    if isinstance(error, CompatibilitySurfaceError):
+        return {"failure_kind": "surface_mismatch", "observed": error.observed}
+    if isinstance(error, json.JSONDecodeError):
+        return {"failure_kind": "json_decode_error"}
+    if isinstance(error, OSError):
+        return {"failure_kind": "io_error"}
+    if isinstance(error, ValueError):
+        return {"failure_kind": "value_error"}
+    return {"failure_kind": "runtime_error"}
+
+
 def _free_loopback_port() -> int:
     """Reserve and release one loopback TCP port for a short-lived local service."""
 
@@ -178,7 +250,10 @@ def _wait_for_extension_evidence(
   commands: document.documentElement.dataset.originweaveCommands || "missing",
   sidePanel: document.documentElement.dataset.originweaveSidePanel || "missing",
   bookmarks: document.documentElement.dataset.originweaveBookmarks || "missing",
-  history: document.documentElement.dataset.originweaveHistory || "missing"
+  history: document.documentElement.dataset.originweaveHistory || "missing",
+  downloads: document.documentElement.dataset.originweaveDownloads || "missing",
+  downloadsDiagnostic:
+    document.documentElement.dataset.originweaveDownloadsDiagnostic || "download-not-evaluated"
 };
 """
     expected = {
@@ -196,6 +271,8 @@ def _wait_for_extension_evidence(
         "sidePanel": "ready",
         "bookmarks": "ready",
         "history": "ready",
+        "downloads": "ready",
+        "downloadsDiagnostic": "download-complete-ready",
     }
     deadline = time.monotonic() + FIXTURE_TIMEOUT_SECONDS
     latest: dict[str, str] = {}
@@ -212,9 +289,7 @@ def _wait_for_extension_evidence(
             ):
                 return latest
         time.sleep(0.1)
-    raise RuntimeError(
-        f"MV3 fixture did not converge: expected={expected!r}, observed={latest!r}"
-    )
+    raise CompatibilitySurfaceError(latest)
 
 
 def _exercise_real_click(driver_port: int, session_id: str) -> str:
@@ -268,6 +343,8 @@ def _run_browser_pass(
 
     driver_port = _free_loopback_port()
     session_id: str | None = None
+    download_dir = pathlib.Path(profile_dir) / "downloads"
+    download_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
     driver = subprocess.Popen(
         [str(chromedriver_bin), f"--port={driver_port}", "--allowed-ips=127.0.0.1"],
         stdout=subprocess.DEVNULL,
@@ -298,6 +375,11 @@ def _run_browser_pass(
                                 f"--disable-extensions-except={FIXTURE}",
                                 f"--load-extension={FIXTURE}",
                             ],
+                            "prefs": {
+                                "download.default_directory": str(download_dir),
+                                "download.prompt_for_download": False,
+                                "download.directory_upgrade": True,
+                            },
                         },
                     }
                 }
@@ -349,6 +431,7 @@ def _run_browser_pass(
                 "side-panel": surfaces["sidePanel"] == "ready",
                 "bookmarks": surfaces["bookmarks"] == "ready",
                 "history": surfaces["history"] == "ready",
+                "downloads": surfaces["downloads"] == "ready",
                 "real-browser-click": click_result == "clicked",
             },
         }
@@ -468,13 +551,13 @@ def main() -> int:
                         trial_number,
                     )
                 )
-            except (OSError, ValueError, RuntimeError, json.JSONDecodeError):
-                trial_results.append(
-                    {
-                        "trial_number": trial_number,
-                        "passed": False,
-                    }
-                )
+            except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc:
+                failed_trial: dict[str, Any] = {
+                    "trial_number": trial_number,
+                    "passed": False,
+                }
+                failed_trial.update(_failure_evidence(exc))
+                trial_results.append(failed_trial)
 
         successful_trials = sum(
             1 for trial in trial_results if trial.get("passed") is True

tests/fixtures/mv3_basic/content_script.js

@@ -27,4 +27,7 @@
   document.documentElement.dataset.originweaveSidePanel = response?.sidePanel ?? "missing";
   document.documentElement.dataset.originweaveBookmarks = response?.bookmarks ?? "missing";
   document.documentElement.dataset.originweaveHistory = response?.history ?? "missing";
+  document.documentElement.dataset.originweaveDownloads = response?.downloads ?? "missing";
+  document.documentElement.dataset.originweaveDownloadsDiagnostic =
+    response?.downloadsDiagnostic ?? "download-not-evaluated";
 })();

tests/fixtures/mv3_basic/download.txt

@@ -0,0 +1 @@
+OriginWeave deterministic MV3 download fixture.

tests/fixtures/mv3_basic/manifest.json

@@ -11,7 +11,8 @@
     "scripting",
     "sidePanel",
     "bookmarks",
-    "history"
+    "history",
+    "downloads"
   ],
   "host_permissions": [
     "http://127.0.0.1/*"

tests/fixtures/mv3_basic/service_worker.js

@@ -1,5 +1,9 @@
 "use strict";
 
+const DOWNLOAD_PAYLOAD = "OriginWeave deterministic MV3 download fixture.\n";
+const DOWNLOAD_POLL_ATTEMPTS = 100;
+const DOWNLOAD_POLL_INTERVAL_MS = 50;
+
 const workerStartPromise = (async () => {
   const values = await chrome.storage.local.get("originweave_worker_start_count");
   const previous = Number(values.originweave_worker_start_count ?? 0);
@@ -16,6 +20,85 @@ async function ensureWorkerState() {
   return "installed";
 }
 
+async function waitForDownload(downloadId, expectedUrl) {
+  const expectedBytes = new TextEncoder().encode(DOWNLOAD_PAYLOAD).byteLength;
+  let observedDownload = false;
+  for (let attempt = 0; attempt < DOWNLOAD_POLL_ATTEMPTS; attempt += 1) {
+    let items;
+    try {
+      items = await chrome.downloads.search({ id: downloadId, limit: 1 });
+    } catch (_error) {
+      return { ready: false, diagnostic: "download-search-missing" };
+    }
+    if (!Array.isArray(items) || items.length !== 1) {
+      await new Promise((resolve) => setTimeout(resolve, DOWNLOAD_POLL_INTERVAL_MS));
+      continue;
+    }
+    observedDownload = true;
+    const item = items[0];
+    if (item.state === "interrupted") {
+      return { ready: false, diagnostic: "download-interrupted" };
+    }
+    if (item.state === "complete") {
+      if (item.url !== expectedUrl) {
+        return { ready: false, diagnostic: "download-url-mismatch" };
+      }
+      if (item.bytesReceived !== expectedBytes || item.totalBytes !== expectedBytes) {
+        return { ready: false, diagnostic: "download-byte-count-mismatch" };
+      }
+      if (item.exists === false) {
+        return { ready: false, diagnostic: "download-exists-false" };
+      }
+      return { ready: true, diagnostic: "download-complete-ready" };
+    }
+    await new Promise((resolve) => setTimeout(resolve, DOWNLOAD_POLL_INTERVAL_MS));
+  }
+  return {
+    ready: false,
+    diagnostic: observedDownload ? "download-timeout" : "download-search-missing",
+  };
+}
+
+async function exerciseDownload(sender) {
+  const sourceUrl = sender?.tab?.url;
+  if (typeof sourceUrl !== "string") {
+    return { ready: false, diagnostic: "download-source-rejected" };
+  }
+
+  let parsed;
+  try {
+    parsed = new URL(sourceUrl);
+  } catch (_error) {
+    return { ready: false, diagnostic: "download-source-rejected" };
+  }
+  if (
+    parsed.protocol !== "http:" ||
+    parsed.hostname !== "127.0.0.1" ||
+    parsed.pathname !== "/page.html" ||
+    parsed.username !== "" ||
+    parsed.password !== ""
+  ) {
+    return { ready: false, diagnostic: "download-source-rejected" };
+  }
+
+  const url = new URL("download.txt", sourceUrl).href;
+  let downloadId;
+  try {
+    downloadId = await chrome.downloads.download({
+      url,
+      filename: "originweave-mv3/download.txt",
+      conflictAction: "overwrite",
+      saveAs: false,
+    });
+  } catch (_error) {
+    return { ready: false, diagnostic: "download-start-rejected" };
+  }
+  if (!Number.isInteger(downloadId)) {
+    return { ready: false, diagnostic: "download-start-rejected" };
+  }
+  return waitForDownload(downloadId, url);
+}
+
 async function exerciseCoreApis(sender) {
   const tabId = sender?.tab?.id;
   if (!Number.isInteger(tabId)) {
@@ -56,6 +139,9 @@ async function exerciseCoreApis(sender) {
   });
   const historyReady = Array.isArray(historyItems);
 
+  const downloadResult = await exerciseDownload(sender);
+  const downloadsReady = downloadResult.ready;
+
   return {
     tabs: tabReady ? "ready" : "missing",
     windows: windowReady ? "ready" : "missing",
@@ -64,6 +150,8 @@ async function exerciseCoreApis(sender) {
     sidePanel: sidePanelReady ? "ready" : "missing",
     bookmarks: bookmarksReady ? "ready" : "missing",
     history: historyReady ? "ready" : "missing",
+    downloads: downloadsReady ? "ready" : "missing",
+    downloadsDiagnostic: downloadResult.diagnostic,
   };
 }
 
@@ -91,6 +179,8 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
         sidePanel: "missing",
         bookmarks: "missing",
         history: "missing",
+        downloads: "missing",
+        downloadsDiagnostic: "download-not-evaluated",
       });
     }
   );

tests/test_mv3_compatibility_contract.py

@@ -162,6 +162,23 @@ def test_runner_reports_repeated_trial_pass_rate(self) -> None:
             with self.subTest(expected=expected):
                 self.assertIn(expected, runner)
 
+    def test_runner_preserves_safe_surface_failure_evidence(self) -> None:
+        """A failed trial must identify the bounded fixture surface without leaking raw errors."""
+
+        namespace = runpy.run_path(str(RUNNER), run_name="mv3_contract")
+        surface_error = namespace["CompatibilitySurfaceError"]
+        failure_evidence = namespace["_failure_evidence"]
+
+        observed = {"downloads": "missing", "storage": "ready"}
+        diagnostic = failure_evidence(surface_error(observed))
+        self.assertEqual(diagnostic["failure_kind"], "surface_mismatch")
+        self.assertEqual(diagnostic["observed"], observed)
+
+        generic = failure_evidence(
+            RuntimeError("secret-token https://example.invalid /home/runner/private")
+        )
+        self.assertEqual(generic, {"failure_kind": "runtime_error"})
+
     def test_workflow_runs_the_real_browser_lane_without_model_credentials(self) -> None:
         """Compatibility evidence must execute Chromium and never require LLM secrets."""