fix(ci): backport hourly IndexNow to NEXT; stop branch deployment event flood

.github/workflows/indexnow.yml

@@ -1,39 +1,61 @@
 name: Notify IndexNow
 
+# Hourly, not on deployment_status. deployment_status has no branch or
+# environment filter at the trigger, so every Vercel preview status of every PR
+# started a run here that the job gate then skipped (about 2,500 skipped runs an
+# hour on 2026-10-31). The notifier reads the live production sitemap, so a
+# schedule submits only what is deployed, and the state marker below keeps an
+# hourly run from resubmitting URLs IndexNow has already been sent.
 on:
-  deployment_status:
+  schedule:
+    - cron: "17 * * * *"
   workflow_dispatch:
 
 permissions:
   contents: read
 
+concurrency:
+  group: indexnow-notify
+  cancel-in-progress: false
+
 jobs:
   notify:
     name: Notify IndexNow
-    # deployment_status has no native branch/path filters. Keep the admission
-    # gate exact so only the production deployment on main can submit URLs.
-    if: >-
-      (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') ||
-      (github.event_name == 'deployment_status' &&
-       github.event.deployment_status.state == 'success' &&
-       github.event.deployment.environment == 'Production – cmux' &&
-       github.event.deployment.ref == 'main' &&
-       github.event.deployment.task == 'deploy')
+    # Only the canonical repository on main submits URLs; forks never do.
+    if: github.repository == 'manaflow-ai/cmux' && github.ref == 'refs/heads/main'
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
     timeout-minutes: 6
     steps:
       # Only public inputs are needed. Read the live sitemap, so a newer main
       # commit cannot cause us to submit URLs that have not been deployed yet.
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          ref: ${{ github.event.deployment.sha || github.sha }}
+          ref: ${{ github.sha }}
           persist-credentials: false
+      # The (url, lastmod) pairs submitted last time. A missing or evicted
+      # entry costs one resubmission of the lookback window, as before.
+      - name: Restore submitted URLs
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .indexnow-state
+          key: indexnow-state-${{ github.run_id }}
+          restore-keys: indexnow-state-
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
-          bun-version: "1.4.4"
+          bun-version: "1.3.14"
       - name: Submit changed URLs
+        id: submit
+        # bash -eo pipefail, so a failed submission is not hidden by tee.
+        shell: bash
         run: |
           bun -e '
             import { indexNowKey, indexNowEndpoint, indexNowLookbackHours } from "./web/app/lib/indexnow.ts";
             console.log(JSON.stringify({ key: indexNowKey, endpoint: indexNowEndpoint, lookbackHours: indexNowLookbackHours }));
-          ' | python3 scripts/ci/notify-indexnow.py
+          ' | python3 scripts/ci/notify-indexnow.py --state .indexnow-state/submitted.json | tee "$RUNNER_TEMP/indexnow-result.json"
+          python3 -c 'import json, sys; print("state_changed=" + str(bool(json.load(open(sys.argv[1])).get("stateChanged"))).lower())' "$RUNNER_TEMP/indexnow-result.json" >> "$GITHUB_OUTPUT"
+      - name: Save submitted URLs
+        if: steps.submit.outputs.state_changed == 'true'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .indexnow-state
+          key: indexnow-state-${{ github.run_id }}

scripts/ci/notify-indexnow.py

@@ -3,11 +3,17 @@
 
 Read the existing web/app/lib/indexnow.ts configuration as JSON on stdin.
 No CRON_SECRET or independently synchronized GitHub credential is involved.
+
+The workflow runs on a schedule, so with --state PATH this keeps the
+(url, lastmod) pairs it last submitted and submits only pairs not in that file.
+A page is sent again only when its lastmod changes, not on every run.
 """
 from __future__ import annotations
 
+import argparse
 from datetime import datetime, timedelta, timezone
 import json
+from pathlib import Path
 import re
 import sys
 import time
@@ -40,6 +46,11 @@ def request(url: str, *, data: bytes | None = None) -> tuple[int, bytes]:
 
 
 def select_urls(xml: bytes, lookback_hours: int, now: datetime | None = None) -> list[str]:
+    return list(dict.fromkeys(url for url, _ in select_entries(xml, lookback_hours, now)))
+
+
+def select_entries(xml: bytes, lookback_hours: int, now: datetime | None = None) -> list[tuple[str, str]]:
+    """The (url, lastmod) pairs in the lookback window, in sitemap order, deduplicated."""
     try:
         root = ET.fromstring(xml)
     except ET.ParseError as error:
@@ -61,13 +72,13 @@ def select_urls(xml: bytes, lookback_hours: int, now: datetime | None = None) ->
         if date.tzinfo is None:
             date = date.replace(tzinfo=timezone.utc)
         if date <= now:
-            entries.append((url, date))
+            entries.append((url, date, modified))
     if not entries:
         return []
     # Match the web notifier's content-relative window, including delayed
     # deployments. Reading the public sitemap excludes unshipped main changes.
-    earliest = max(date for _, date in entries) - timedelta(hours=lookback_hours)
-    return list(dict.fromkeys(url for url, date in entries if date >= earliest))
+    earliest = max(date for _, date, _ in entries) - timedelta(hours=lookback_hours)
+    return list(dict.fromkeys((url, modified) for url, date, modified in entries if date >= earliest))
 
 
 def submit_batches(config: dict, urls: list[str]) -> dict:
@@ -91,7 +102,24 @@ def submit_batches(config: dict, urls: list[str]) -> dict:
     return {"submitted": submitted, "batches": batch_count, "status": statuses[-1] if statuses else 0}
 
 
-def run(config: dict) -> dict:
+def load_state(path: Path | None) -> set[tuple[str, str]]:
+    if path is None or not path.exists():
+        return set()
+    try:
+        pairs = json.loads(path.read_text())["submitted"]
+        return {(str(url), str(modified)) for url, modified in pairs}
+    except (ValueError, KeyError, TypeError) as error:
+        # A damaged marker costs one resubmission of the window, never a failure.
+        print(f"Ignoring unreadable IndexNow state {path}: {error}", file=sys.stderr)
+        return set()
+
+
+def save_state(path: Path, entries: list[tuple[str, str]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps({"submitted": [list(entry) for entry in entries]}, indent=0) + "\n")
+
+
+def run(config: dict, state_path: Path | None = None) -> dict:
     key = config["key"]
     if not isinstance(key, str) or not re.fullmatch(r"[a-zA-Z0-9-]{8,128}", key):
         raise ValueError("Invalid IndexNow public key")
@@ -106,15 +134,24 @@ def run(config: dict) -> dict:
     if deployed_key.decode("utf-8").strip() != key:
         raise ValueError("Deployed IndexNow key does not match the configured public key")
     _, sitemap = request(f"{ORIGIN}/sitemap.xml")
-    urls = select_urls(sitemap, lookback)
-    if not urls:
-        return {"submitted": 0, "status": 0}
-    return submit_batches(config, urls)
+    entries = select_entries(sitemap, lookback)
+    already = load_state(state_path)
+    urls = list(dict.fromkeys(url for url, modified in entries if (url, modified) not in already))
+    result = submit_batches(config, urls) if urls else {"submitted": 0, "status": 0}
+    # Written only after every batch succeeded, and only the current window, so
+    # the marker stays as small as the window and a failed run retries next time.
+    if state_path is not None and set(entries) != already:
+        save_state(state_path, entries)
+        result["stateChanged"] = True
+    return result
 
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--state", type=Path, help="JSON file of the (url, lastmod) pairs already submitted")
+    args = parser.parse_args()
     try:
-        print(json.dumps(run(json.load(sys.stdin))))
+        print(json.dumps(run(json.load(sys.stdin), args.state)))
     except (ValueError, KeyError, OSError) as error:
         print(f"IndexNow notification failed: {error}", file=sys.stderr)
         sys.exit(1)

tests/test_indexnow_notification.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 import sys
 import unittest
+import tempfile
 from unittest.mock import patch
 import urllib.error
 from datetime import datetime, timezone
@@ -96,7 +97,59 @@ def test_workflow_uses_public_shared_configuration_and_no_private_secret(self):
         self.assertIn("web/app/lib/indexnow.ts", workflow)
         self.assertIn("scripts/ci/notify-indexnow.py", workflow)
         self.assertIn("timeout-minutes: 6", workflow)
-        self.assertIn("ref: ${{ github.event.deployment.sha || github.sha }}", workflow)
+        self.assertIn("ref: ${{ github.sha }}", workflow)
+
+    def test_workflow_is_scheduled_not_triggered_by_every_deployment_status(self):
+        # deployment_status cannot be filtered at the trigger, so every Vercel
+        # preview status started a skipped run. The schedule plus state marker
+        # replaces it.
+        workflow = (ROOT / ".github/workflows/indexnow.yml").read_text()
+        triggers = workflow.split("\non:\n", 1)[1].split("\npermissions:", 1)[0]
+        self.assertNotIn("deployment_status", triggers)
+        self.assertIn("schedule:", triggers)
+        self.assertIn("--state .indexnow-state/submitted.json", workflow)
+        self.assertIn("actions/cache/restore@", workflow)
+        self.assertIn("actions/cache/save@", workflow)
+        self.assertIn("steps.submit.outputs.state_changed == 'true'", workflow)
+        self.assertIn("shell: bash", workflow)
+
+    def test_state_marker_submits_each_url_lastmod_pair_once(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            state = Path(tmp) / "state" / "submitted.json"
+            with patch.object(notify, "request", side_effect=[(200, KEY.encode()), (200, XML), (202, b"")]):
+                first = notify.run(CONFIG, state)
+            self.assertEqual(first, {"submitted": 2, "batches": 1, "status": 202, "stateChanged": True})
+
+            # Same sitemap an hour later: nothing new is posted and the marker is untouched.
+            with patch.object(notify, "request", side_effect=[(200, KEY.encode()), (200, XML)]) as request:
+                second = notify.run(CONFIG, state)
+            self.assertEqual(second, {"submitted": 0, "status": 0})
+            self.assertEqual(request.call_count, 2)
+
+            # A page whose lastmod changes is sent again, and only that page.
+            changed = XML.replace(b"<loc>https://cmux.com/recent</loc><lastmod>2026-03-02", b"<loc>https://cmux.com/recent</loc><lastmod>2026-03-04")
+            with patch.object(notify, "request", side_effect=[(200, KEY.encode()), (200, changed), (200, b"")]) as request:
+                third = notify.run(CONFIG, state)
+            self.assertEqual(third["submitted"], 1)
+            self.assertEqual(json.loads(request.call_args_list[2].kwargs["data"])["urlList"], ["https://cmux.com/recent"])
+
+    def test_failed_submission_keeps_the_previous_marker(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            state = Path(tmp) / "submitted.json"
+            error = RuntimeError("IndexNow down")
+            with patch.object(notify, "request", side_effect=[(200, KEY.encode()), (200, XML), error]):
+                with self.assertRaises(RuntimeError):
+                    notify.run(CONFIG, state)
+            self.assertFalse(state.exists())
+
+    def test_unreadable_marker_resubmits_instead_of_failing(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            state = Path(tmp) / "submitted.json"
+            state.write_text("not json")
+            with patch.object(notify, "request", side_effect=[(200, KEY.encode()), (200, XML), (202, b"")]):
+                result = notify.run(CONFIG, state)
+            self.assertEqual(result["submitted"], 2)
+            self.assertEqual(len(json.loads(state.read_text())["submitted"]), 2)
 
 if __name__ == "__main__":
     unittest.main()