From ceb6407bbbd578efa900c055ef7e4f6276c0b796 Mon Sep 17 00:00:00 2001 From: AIalliAI Date: Thu, 11 Jun 2026 21:38:06 +0000 Subject: [PATCH] fix(skills-index): give the offline builder a walk budget the full ClawHub catalog fits in The interactive 12s CATALOG_WALK_BUDGET_SECONDS (added to stop browse/ search hanging on a slow catalog walk) also bounds the offline index builder, truncating the ~50k-skill ClawHub walk at ~16 pages. Every skills-index/deploy-site run since 2026-06-10 06:22 UTC fails the EXPECTED_FLOORS health check (clawhub 1797-3195 < 20000), freezing the docs site and skills index. Let ClawHubSource take an explicit per-instance walk budget and pass 600s from scripts/build_skills_index.py (full walk needs ~260s at the observed ~1 page/s). Interactive callers keep the 12s default. Fixes #38240 --- scripts/build_skills_index.py | 7 +++- .../scripts/test_build_skills_index_health.py | 6 ++- tests/tools/test_skills_hub_clawhub.py | 42 +++++++++++++++++++ tools/skills_hub.py | 9 ++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/scripts/build_skills_index.py b/scripts/build_skills_index.py index a5bf900d831a..4344a47a7671 100644 --- a/scripts/build_skills_index.py +++ b/scripts/build_skills_index.py @@ -255,7 +255,12 @@ def main(): "official": OptionalSkillSource(), "well-known": WellKnownSkillSource(), "github": GitHubSource(auth=auth), - "clawhub": ClawHubSource(), + # The interactive 12s walk budget truncates the full ~50k catalog at + # ~16 pages (~3.2k skills), which trips the EXPECTED_FLOORS health + # check below and blocks the deploy. The offline builder walks to + # exhaustion (~250 sequential pages, ~3-4 min), so give it a budget + # sized for that. + "clawhub": ClawHubSource(catalog_walk_budget_seconds=600), "claude-marketplace": ClaudeMarketplaceSource(auth=auth), "lobehub": LobeHubSource(), "browse-sh": BrowseShSource(), diff --git a/tests/scripts/test_build_skills_index_health.py b/tests/scripts/test_build_skills_index_health.py index c5116917d1b0..f112f367bc9a 100644 --- a/tests/scripts/test_build_skills_index_health.py +++ b/tests/scripts/test_build_skills_index_health.py @@ -49,7 +49,11 @@ def _install_fake_sources(monkeypatch, *, github_count, claude_count=40, build_mod, "GitHubSource", lambda auth: _FakeSource("github", github_count, rate_limited=github_rate_limited), ) - monkeypatch.setattr(build_mod, "ClawHubSource", lambda: _FakeSource("clawhub", 69000)) + monkeypatch.setattr( + build_mod, + "ClawHubSource", + lambda catalog_walk_budget_seconds=None: _FakeSource("clawhub", 69000), + ) monkeypatch.setattr( build_mod, "ClaudeMarketplaceSource", lambda auth: _FakeSource("claude-marketplace", claude_count, rate_limited=github_rate_limited), diff --git a/tests/tools/test_skills_hub_clawhub.py b/tests/tools/test_skills_hub_clawhub.py index 972175999fd4..5ac6f3159839 100644 --- a/tests/tools/test_skills_hub_clawhub.py +++ b/tests/tools/test_skills_hub_clawhub.py @@ -422,6 +422,48 @@ def side_effect(url, *args, **kwargs): self.assertEqual(results[0].identifier, "only-skill") mock_write_cache.assert_called_once() + @patch("tools.skills_hub._write_index_cache") + @patch("tools.skills_hub._read_index_cache", return_value=None) + @patch("tools.skills_hub.httpx.get") + def test_catalog_walk_budget_instance_override_wins_over_class_default( + self, mock_get, _mock_read_cache, mock_write_cache + ): + """The offline index builder passes an explicit walk budget so the + interactive 12s default cannot truncate the full-catalog walk (which + shipped a degenerate ~3.2k-skill index and failed the EXPECTED_FLOORS + deploy check). With the class default forced to an already-expired + deadline, an instance constructed with its own budget must still walk + to natural termination and cache the result.""" + pages = {"n": 0} + + def side_effect(url, *args, **kwargs): + if url.endswith("/skills"): + idx = pages["n"] + pages["n"] += 1 + last = idx == 2 + return _MockResponse( + status_code=200, + json_data={ + "items": [ + {"slug": f"skill-{idx}", "displayName": f"Skill {idx}"} + ], + **({} if last else {"nextCursor": f"cursor-{idx + 1}"}), + }, + ) + return _MockResponse(status_code=404, json_data={}) + + mock_get.side_effect = side_effect + + with patch.object(ClawHubSource, "CATALOG_WALK_BUDGET_SECONDS", -1): + src = ClawHubSource(catalog_walk_budget_seconds=60) + results = src._load_catalog_index() + + # Walked all three pages to natural termination despite the expired + # class-level deadline, and cached the complete catalog. + self.assertEqual(pages["n"], 3) + self.assertEqual(len(results), 3) + mock_write_cache.assert_called_once() + class TestClawHubCatalogWalkBounded(unittest.TestCase): """max_items bounds the walk so browse's cold-start fallback renders one diff --git a/tools/skills_hub.py b/tools/skills_hub.py index 7750eb1b96e0..258ebc4a3de2 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -1952,6 +1952,15 @@ class ClawHubSource(SkillSource): # minutes. Bound it so a slow/large catalog cannot hang the caller. CATALOG_WALK_BUDGET_SECONDS = 12 + def __init__(self, catalog_walk_budget_seconds: Optional[float] = None): + # Interactive callers (browse/search cold start) keep the tight class + # default so a slow catalog cannot hang them. Walk-to-exhaustion + # callers — the offline index builder — need ~250 sequential pages for + # the full 50k+ catalog, which no interactive budget can cover, so + # they pass an explicit larger budget here. + if catalog_walk_budget_seconds is not None: + self.CATALOG_WALK_BUDGET_SECONDS = catalog_walk_budget_seconds + def source_id(self) -> str: return "clawhub"