From 37c8928f88037395acf0925b51276a5f9fd5362f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 18:23:43 +0000 Subject: [PATCH 01/19] =?UTF-8?q?spike(p3-m0):=203=20hook=20prototype=20?= =?UTF-8?q?=C2=B7=20strip=20+=20cache=20+=20budget=20=C2=B7=20fail-closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P3-M0 首轮 fork 侧 spike · Gate 0 CONDITIONAL PASS + Codex 批准。 本 commit 与 Hermes_AI 侧 claude/p3-m0-contracts 契约互链。 双仓 baseline SHA(冻结): - fork(本方): spike/p3-m0 @ base main @ 326bdfb7a27e292a25aa1a8a073e6fac43460a98 - Hermes_AI: claude/p3-m0-contracts @ base aa5c3c4(PR #27 merge commit) - fixture SHA-256(跨仓一致): 0cd6ac14035292103ecf574aa6434cf3fcc4d0bdcbe9f21f32a0ba7e9ae26212 3 minimal prototypes(遵 AGENTS.md · 不改 core files · 只加 plugin dirs): plugins/spike_p3_m0_strip/ - hook: transform_llm_output(agent/turn_finalizer.py:561) - 覆盖出口 ① 普通最终响应 · 其他 4 出口 fixture 冻结不实装 - fail-open on sanitizer exception · 保留 raw canonical transcript - 使用现有 24 lifecycle hook · 无 core hook 变更 plugins/spike_p3_m0_cache/ - seam: run_llm_execution_middleware(hermes_cli/middleware.py:187 · byte-identical fresh) - fail-closed 无 tenant · fail-closed 非严格集合 request/response - 只缓存 non-streaming + tool-free + finish=stop + tenant/permission_scope 完整 - key schema v1(sha256 · 12 字段 · 含 profile_version + collection_version) - usage 拆分 origin/billable=0/saved · origin 不进 session totals - 内存 LRU cap=256 · 非 persistent · 非 cross-session - module-level API 供 fork core middleware 接线(接线由独立后续 PR) plugins/spike_p3_m0_budget/ - hooks: pre_api_request reserve + post_api_request settle(T2 · profile.yaml) - 明令不用 HERMES_DAILY_TOKEN_BUDGET 冒充 per-tenant budget - fail-open 无 tenant · fail-close BudgetExceeded 超预算 - cache hit(response.cache_meta.hit=True)· settle 不 charge - retry / fallback · reservation 每 api_request_id 独立 plugins/spike_p3_m0_fixtures/ - citation_cases.py 副本 · SHA-256 与 Hermes_AI 侧完全一致 - 17 case: 三态 + Unicode + 字节切分 + tool_result + history replay - SHA-256 校验测试锁死跨仓一致性 Test results(在 fresh-ci venv + `HERMES_HOME=temp` + `SPIKE_*_ENABLED=1`): - scripts/run_tests.sh plugins/spike_p3_m0_{strip,cache,budget} - 3 files, 49 tests passed, 0 failed, 0.6s 明确不做(M0 边界): - 生产接线 fork core middleware(需独立 PR · spike 只暴露 module API) - streaming/interim/interrupt/history-replay/tool-result 各出口实装 - CAS · 跨进程 budget 同步 · persistent cache - fork core files 改动 - M1 rebrand · Provider 正式接线 · MCP · Flutter - 转 Ready / merge 后续:双 Draft PR 互链后停 · 交 Codex 复核。 Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01FiKVo54oX4Uv2qksAHjYsZ --- plugins/spike_p3_m0_budget/__init__.py | 203 ++++++++++++++ plugins/spike_p3_m0_budget/plugin.yaml | 7 + plugins/spike_p3_m0_budget/tests/__init__.py | 0 .../tests/test_budget_reserve.py | 142 ++++++++++ plugins/spike_p3_m0_cache/__init__.py | 250 ++++++++++++++++++ plugins/spike_p3_m0_cache/plugin.yaml | 4 + plugins/spike_p3_m0_cache/tests/__init__.py | 0 .../tests/test_cache_middleware.py | 199 ++++++++++++++ plugins/spike_p3_m0_fixtures/__init__.py | 0 .../spike_p3_m0_fixtures/citation_cases.py | 220 +++++++++++++++ plugins/spike_p3_m0_strip/__init__.py | 53 ++++ plugins/spike_p3_m0_strip/plugin.yaml | 6 + plugins/spike_p3_m0_strip/sanitize.py | 55 ++++ plugins/spike_p3_m0_strip/tests/__init__.py | 0 .../tests/test_strip_final.py | 105 ++++++++ 15 files changed, 1244 insertions(+) create mode 100644 plugins/spike_p3_m0_budget/__init__.py create mode 100644 plugins/spike_p3_m0_budget/plugin.yaml create mode 100644 plugins/spike_p3_m0_budget/tests/__init__.py create mode 100644 plugins/spike_p3_m0_budget/tests/test_budget_reserve.py create mode 100644 plugins/spike_p3_m0_cache/__init__.py create mode 100644 plugins/spike_p3_m0_cache/plugin.yaml create mode 100644 plugins/spike_p3_m0_cache/tests/__init__.py create mode 100644 plugins/spike_p3_m0_cache/tests/test_cache_middleware.py create mode 100644 plugins/spike_p3_m0_fixtures/__init__.py create mode 100644 plugins/spike_p3_m0_fixtures/citation_cases.py create mode 100644 plugins/spike_p3_m0_strip/__init__.py create mode 100644 plugins/spike_p3_m0_strip/plugin.yaml create mode 100644 plugins/spike_p3_m0_strip/sanitize.py create mode 100644 plugins/spike_p3_m0_strip/tests/__init__.py create mode 100644 plugins/spike_p3_m0_strip/tests/test_strip_final.py diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py new file mode 100644 index 0000000000000..20c827ecc7545 --- /dev/null +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -0,0 +1,203 @@ +"""P3-M0 Spike · tenant budget prototype(pre/post_api_request · T2 · fail-open). + +**范围严守**(见 Hermes_AI 侧 `docs/fork/budget-hook.md`): + +- Reserve @ `pre_api_request` @ `agent/conversation_loop.py:2586` +- Settle @ `post_api_request` @ `agent/conversation_loop.py:6114` +- **明令删除**:`HERMES_DAILY_TOKEN_BUDGET` 冒充 per-tenant budget +- **T2 首选**:tenant-per-process/profile · 从 `HERMES_HOME/profile.yaml` 读 + `tenant.daily_budget_tokens` · 非用户可见 env +- **fail-open**:context 缺失 · 不 reserve · 不 settle · 不阻断 chat · 不声称多租户已启用 +- cache hit(response 带 `cache_meta.hit=True`)· settle 时不扣(reservation 已在 pre 阶段 + 发生 · 但可选:M0 spike 简化为 pre 只在 middleware short-circuit 前判断) + +**M0 首轮 spike 实装**: +- `_get_tenant_budget_context()` 从 profile.yaml 读 +- module-level `_tenant_budget_bucket`(简 dict · 单进程 · 非 persistent) +- reserve / settle 契约 +- BudgetExceeded exception + +**M0 首轮 spike 不做**: +- 生产接线 fork core · plugin ctx.register_hook 已挂 · 但 fork core 需装配 `hermes tools` 类似方式启用 +- CAS · 跨进程 budget 同步 +- Hermes_AI SQL sync(等 M1+ · 走 out-of-band gateway sync) +""" +from __future__ import annotations + +import logging +import os +import threading +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +BUDGET_CONTRACT_VERSION = "v1" + + +class BudgetExceeded(RuntimeError): + """租户预算超支 · 阻断本次 API 调用。""" + + +# ── Module state · 单进程 · 非 persistent(M0) ── + +_lock = threading.Lock() +_reservations: Dict[str, Dict[str, Any]] = {} # api_request_id → {tenant_id, tokens} +_daily_used: Dict[str, int] = {} # tenant_id → tokens_used_today(no daily reset · M0 spike) + +_ENABLED = os.environ.get("SPIKE_P3_M0_BUDGET_ENABLED", "0") == "1" + + +def _get_tenant_budget_context() -> Optional[Dict[str, Any]]: + """T2 · 从 HERMES_HOME/profile.yaml 读 tenant + budget。 + + Returns dict with: tenant_id, daily_budget_tokens(int|None), budget_contract_version + """ + hermes_home_env = os.environ.get("HERMES_HOME") + if not hermes_home_env: + return None + profile_path = Path(hermes_home_env) / "profile.yaml" + if not profile_path.exists(): + return None + try: + import yaml # type: ignore + + raw = yaml.safe_load(profile_path.read_text(encoding="utf-8")) or {} + except Exception as exc: # noqa: BLE001 + logger.warning("spike-budget: profile.yaml read failed: %s", exc) + return None + tenant_block = raw.get("tenant") or {} + tid = tenant_block.get("tenant_id") + if not tid: + return None + daily = tenant_block.get("daily_budget_tokens") + if daily is not None: + try: + daily = int(daily) + except (TypeError, ValueError): + daily = None + return { + "tenant_id": str(tid), + "daily_budget_tokens": daily, + "budget_contract_version": BUDGET_CONTRACT_VERSION, + } + + +# ── Hook callbacks ── + + +def on_pre_api_request( + *, + task_id: str = "", + turn_id: str = "", + api_request_id: str = "", + session_id: str = "", + platform: str = "", + model: str = "", + provider: str = "", + base_url: str = "", + api_mode: str = "", + api_call_count: int = 0, + retry_count: int = 0, + request_messages: Any = None, + message_count: int = 0, + tool_count: int = 0, + approx_input_tokens: int = 0, + request_char_count: int = 0, + max_tokens: int = 0, + started_at: float = 0.0, + middleware_trace: Any = None, + request: Any = None, + **_ctx: Any, +) -> None: + """Reserve tokens against tenant daily budget. + + Fail-open policy(§4 budget-hook.md): + - `_ENABLED` False → skip + - no tenant context → skip · **不声称多租户已启用** + - no daily_budget set → skip + - reservation exceeds budget → raise `BudgetExceeded`(fail-close · 阻断) + """ + if not _ENABLED or not api_request_id: + return + ctx = _get_tenant_budget_context() + if not ctx or ctx.get("daily_budget_tokens") is None: + return + tid = ctx["tenant_id"] + budget = ctx["daily_budget_tokens"] + tokens_needed = int(approx_input_tokens or 0) + int(max_tokens or 0) + with _lock: + used = _daily_used.get(tid, 0) + if used + tokens_needed > budget: + raise BudgetExceeded( + f"tenant {tid} daily budget exhausted " + f"(used={used}, need={tokens_needed}, budget={budget})" + ) + _reservations[api_request_id] = {"tenant_id": tid, "tokens": tokens_needed} + + +def on_post_api_request( + *, + task_id: str = "", + turn_id: str = "", + api_request_id: str = "", + session_id: str = "", + platform: str = "", + model: str = "", + provider: str = "", + base_url: str = "", + api_mode: str = "", + api_call_count: int = 0, + api_duration: float = 0.0, + started_at: float = 0.0, + ended_at: float = 0.0, + finish_reason: str = "", + message_count: int = 0, + response_model: Any = None, + response: Any = None, + usage: Any = None, + assistant_message: Any = None, + assistant_content_chars: int = 0, + assistant_tool_call_count: int = 0, + **_ctx: Any, +) -> None: + """Settle actual usage(usage 已 canonicalized by fork · conversation_loop:3437-3448)。 + + Cache hit(response 带 `cache_meta.hit=True`)· settle 不扣(reservation 已 pop)。 + """ + if not api_request_id: + return + with _lock: + reservation = _reservations.pop(api_request_id, None) + if not reservation: + return + # Cache hit 保护:若 response 是 cache hit · 不 charge actual usage + is_cache_hit = False + if isinstance(response, dict): + cache_meta = response.get("cache_meta") or {} + is_cache_hit = bool(cache_meta.get("hit")) + if is_cache_hit: + return # cache hit · reserve 归零 · 不 charge + actual = 0 + if isinstance(usage, dict): + actual = int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) + with _lock: + _daily_used[reservation["tenant_id"]] = _daily_used.get(reservation["tenant_id"], 0) + actual + + +# ── Spike helpers · 测试用 ── + + +def reset_state() -> None: + with _lock: + _reservations.clear() + _daily_used.clear() + + +def get_daily_used(tenant_id: str) -> int: + return _daily_used.get(tenant_id, 0) + + +def register(ctx: Any) -> None: + ctx.register_hook("pre_api_request", on_pre_api_request) + ctx.register_hook("post_api_request", on_post_api_request) diff --git a/plugins/spike_p3_m0_budget/plugin.yaml b/plugins/spike_p3_m0_budget/plugin.yaml new file mode 100644 index 0000000000000..a6b9afd014db6 --- /dev/null +++ b/plugins/spike_p3_m0_budget/plugin.yaml @@ -0,0 +1,7 @@ +name: spike-p3-m0-budget +version: "0.0.1" +description: "P3-M0 Spike · tenant budget prototype · pre_api_request reserve + post_api_request settle · T2 tenant-per-process/profile · host-owned session context · fail-open without tenant context · MUST NOT use HERMES_DAILY_TOKEN_BUDGET to impersonate per-tenant budget. WIP · off by default." +author: "Hermes_AI × NousResearch/hermes-agent Spike" +hooks: + - pre_api_request + - post_api_request diff --git a/plugins/spike_p3_m0_budget/tests/__init__.py b/plugins/spike_p3_m0_budget/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py new file mode 100644 index 0000000000000..f4f869c3655a0 --- /dev/null +++ b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py @@ -0,0 +1,142 @@ +"""P3-M0 Spike · budget prototype 测试。 + +覆盖(见 `docs/fork/budget-hook.md` §9): +- 无 tenant context → fail-open(不 reserve · 不 settle · 不 raise) +- tenant context + budget=None → 不 reserve +- reserve 成功 → settle 匹配 usage +- reserve 失败(超预算)→ raise BudgetExceeded +- cache hit → settle 不 charge(reserve 后被 middleware short-circuit 时) +- retry / fallback → reservation 独立(每 api_request_id 一次) +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from plugins.spike_p3_m0_budget import ( + BudgetExceeded, + get_daily_used, + on_post_api_request, + on_pre_api_request, + reset_state, +) + + +@pytest.fixture(autouse=True) +def _budget_isolation(monkeypatch, tmp_path): + monkeypatch.setenv("SPIKE_P3_M0_BUDGET_ENABLED", "1") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + import plugins.spike_p3_m0_budget as mod + + monkeypatch.setattr(mod, "_ENABLED", True) + reset_state() + yield + reset_state() + + +def _write_profile(tmp_path: Path, *, tenant_id="t_test", daily=None): + import yaml # type: ignore + + tenant_block = {"tenant_id": tenant_id} + if daily is not None: + tenant_block["daily_budget_tokens"] = daily + (tmp_path / "profile.yaml").write_text( + yaml.safe_dump({"tenant": tenant_block}), encoding="utf-8" + ) + + +# ── fail-open scenarios ── + + +def test_no_tenant_context_no_reserve(tmp_path): + """无 profile.yaml → fail-open · 不 raise · 不影响 state。""" + on_pre_api_request(api_request_id="req_1", approx_input_tokens=100, max_tokens=200) + on_post_api_request(api_request_id="req_1", usage={"prompt_tokens": 100, "completion_tokens": 50}) + assert get_daily_used("t_test") == 0 + + +def test_tenant_without_budget_no_reserve(tmp_path): + """tenant 有 · daily_budget=None → 不 reserve · 不 settle · 不 raise。""" + _write_profile(tmp_path, daily=None) + on_pre_api_request(api_request_id="req_2", approx_input_tokens=100, max_tokens=200) + on_post_api_request(api_request_id="req_2", usage={"prompt_tokens": 100, "completion_tokens": 50}) + assert get_daily_used("t_test") == 0 + + +# ── reserve / settle 正常路径 ── + + +def test_reserve_then_settle_charges_actual(tmp_path): + _write_profile(tmp_path, daily=10000) + on_pre_api_request(api_request_id="req_A", approx_input_tokens=100, max_tokens=200) + on_post_api_request( + api_request_id="req_A", + usage={"prompt_tokens": 80, "completion_tokens": 40}, + ) + # actual = 80+40 = 120(reserve 是 300 · settle 时按 actual 扣) + assert get_daily_used("t_test") == 120 + + +# ── reserve 失败 · raise BudgetExceeded ── + + +def test_reserve_exceeds_budget_raises(tmp_path): + _write_profile(tmp_path, daily=100) + # 首次 reserve 200 · 超预算 100 · raise + with pytest.raises(BudgetExceeded, match="daily budget exhausted"): + on_pre_api_request(api_request_id="req_over", approx_input_tokens=100, max_tokens=200) + + +def test_multiple_reserves_accumulate(tmp_path): + _write_profile(tmp_path, daily=500) + # 第一次 reserve 100 + settle 90(实用)· daily_used=90 + on_pre_api_request(api_request_id="r1", approx_input_tokens=50, max_tokens=50) + on_post_api_request(api_request_id="r1", usage={"prompt_tokens": 50, "completion_tokens": 40}) + assert get_daily_used("t_test") == 90 + # 第二次 reserve 100 · 累计 used=90 · 90+100 <= 500 · 通过 + on_pre_api_request(api_request_id="r2", approx_input_tokens=50, max_tokens=50) + on_post_api_request(api_request_id="r2", usage={"prompt_tokens": 50, "completion_tokens": 60}) + assert get_daily_used("t_test") == 200 + + +# ── cache hit · settle 不 charge ── + + +def test_cache_hit_settle_no_charge(tmp_path): + _write_profile(tmp_path, daily=10000) + on_pre_api_request(api_request_id="req_cache", approx_input_tokens=100, max_tokens=200) + # response 带 cache_meta.hit=True · settle 不 charge + on_post_api_request( + api_request_id="req_cache", + response={"cache_meta": {"hit": True}}, + usage={"prompt_tokens": 80, "completion_tokens": 40}, + ) + assert get_daily_used("t_test") == 0 + + +# ── retry · reservation 独立 ── + + +def test_multiple_api_request_ids_independent(tmp_path): + _write_profile(tmp_path, daily=10000) + on_pre_api_request(api_request_id="attempt_1", approx_input_tokens=50, max_tokens=50) + on_pre_api_request(api_request_id="attempt_2", approx_input_tokens=100, max_tokens=100) + on_post_api_request(api_request_id="attempt_1", usage={"prompt_tokens": 40, "completion_tokens": 30}) + on_post_api_request(api_request_id="attempt_2", usage={"prompt_tokens": 80, "completion_tokens": 60}) + assert get_daily_used("t_test") == 210 # 70 + 140 + + +# ── env not enabled · no-op ── + + +def test_disabled_no_op(monkeypatch, tmp_path): + monkeypatch.setenv("SPIKE_P3_M0_BUDGET_ENABLED", "0") + import plugins.spike_p3_m0_budget as mod + + monkeypatch.setattr(mod, "_ENABLED", False) + _write_profile(tmp_path, daily=100) + # 即使 profile.yaml 有 tenant + budget · disabled 状态不 raise + on_pre_api_request(api_request_id="req_off", approx_input_tokens=1000, max_tokens=1000) + on_post_api_request(api_request_id="req_off", usage={"prompt_tokens": 500}) + assert get_daily_used("t_test") == 0 diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py new file mode 100644 index 0000000000000..34fc2f784db20 --- /dev/null +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -0,0 +1,250 @@ +"""P3-M0 Spike · tenant-safe cache prototype(middleware seam · fail-closed). + +**范围严守**(见 Hermes_AI 侧 `docs/fork/cache-layer.md`): + +- Seam:`run_llm_execution_middleware` @ `hermes_cli/middleware.py:187`(byte-identical baseline→fresh) +- **fail-closed 无 tenant**:context 缺失 → 走 `next_call(request)` 不 lookup 不 insert +- 只缓存 non-streaming + tool-free + successful terminal · 其他一律不缓存 +- **cache key schema v1**:tenant_id / principal_id / permission_scope_version / provider / model / + system_prompt_version / tool_schema_version / profile_version / knowledge_collection / + knowledge_collection_version / normalized_semantic_messages / cache_contract_version +- **usage 拆分**:origin_usage(元数据 · 不进 totals)· billable_usage=0 · saved_usage · cache_lookup_cost +- **cache hit 仍触发**:post_api_request hook · 独立 rate limit · audit `cache.hit` metric + +**M0 首轮 spike 实装**: +- `_get_tenant_context()` = 从 `HERMES_HOME/profile.yaml` 读(T2 · 单进程单 tenant) +- `_is_cacheable_request/response` = §1 严格布尔 +- `_build_cache_key` = §2 schema · sha256 +- `_cache` = 内存 LRU dict(**不 persistent · 不 cross-session · 不 cross-process**) +- `_attach_cache_meta` = 附 cache_meta 字段 + +**M0 首轮 spike 不做**: +- 生产接线到 fork core middleware(需 fork core PR · 独立)· spike 只提供 middleware + callback 可挂式接口 + 完整 fixture 测试 +- persistent / cross-session cache +- streaming 缓冲 codec +""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +from collections import OrderedDict +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +logger = logging.getLogger(__name__) + +CACHE_CONTRACT_VERSION = "v1" +CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent + +# ── module-level state · spike scope · fork core middleware 接线由后续 PR ── + +_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() +_ENABLED = os.environ.get("SPIKE_P3_M0_CACHE_ENABLED", "0") == "1" # 默认 OFF · config.yaml 可以覆盖 · 见 spec + + +# ── Tenant context helper(T2 · profile.yaml)── + + +def _get_tenant_context() -> Optional[Dict[str, Any]]: + """从 HERMES_HOME/profile.yaml 读 tenant metadata(T2 · 单进程单 tenant)。 + + 返回 dict 含:tenant_id / principal_id / permission_scope_version / + profile_version(全部 required · 缺一 fail-closed)。 + + **T1(单进程多租户)未启用** · host session metadata 通道待 M2+。 + """ + hermes_home_env = os.environ.get("HERMES_HOME") + if not hermes_home_env: + return None + profile_path = Path(hermes_home_env) / "profile.yaml" + if not profile_path.exists(): + return None + try: + import yaml # type: ignore + + with profile_path.open("r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + except Exception as exc: # noqa: BLE001 + logger.warning("spike-cache: profile.yaml read failed: %s", exc) + return None + tenant_block = raw.get("tenant") or {} + tid = tenant_block.get("tenant_id") + if not tid: + return None + return { + "tenant_id": str(tid), + "principal_id": str(tenant_block.get("principal_id") or "system"), + "permission_scope_version": str(tenant_block.get("permission_scope_version") or "0"), + "profile_version": str(tenant_block.get("profile_version") or "0"), + } + + +# ── Cacheability guards(§1 严格集合)── + + +def is_cacheable_request(request: Dict[str, Any]) -> bool: + """仅接收 non-streaming + tool-free 请求。""" + if request.get("stream"): + return False + tools = request.get("tools") or [] + if tools: + return False + return True + + +def is_cacheable_response(response: Any) -> bool: + """仅接收 successful terminal response(无 tool_calls · finish=stop)。""" + try: + choices = response.get("choices") if isinstance(response, dict) else getattr(response, "choices", None) + if not choices: + return False + first = choices[0] + msg = first.get("message") if isinstance(first, dict) else getattr(first, "message", None) + if not msg: + return False + tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + if tool_calls: + return False + finish = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None) + if finish not in ("stop",): + return False + return True + except Exception: # noqa: BLE001 + return False + + +# ── Cache key builder(§2 schema)── + + +def build_cache_key( + request: Dict[str, Any], + tenant_ctx: Dict[str, Any], + *, + model: str, + provider: str, +) -> str: + """SHA-256 hex of the deterministic key components.""" + messages_bytes = json.dumps(request.get("messages") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") + system_prompt_hash = hashlib.sha256(messages_bytes[:1024]).hexdigest()[:16] # 简 hash · M0 spike + tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") + tools_hash = hashlib.sha256(tools_bytes).hexdigest()[:16] + parts = { + "tenant_id": tenant_ctx["tenant_id"], + "principal_id": tenant_ctx["principal_id"], + "permission_scope_version": tenant_ctx["permission_scope_version"], + "provider": provider, + "model": model, + "system_prompt_version": system_prompt_hash, + "tool_schema_version": tools_hash, + "profile_version": tenant_ctx["profile_version"], + "knowledge_collection": request.get("_kb_collection") or "", + "knowledge_collection_version": request.get("_kb_version") or "", + "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), + "cache_contract_version": CACHE_CONTRACT_VERSION, + } + canonical = json.dumps(parts, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +# ── Cache middleware(可挂载到 run_llm_execution_middleware · fork core 接线待另 PR)── + + +def cache_middleware( + request: Dict[str, Any], + next_call: Callable[[Any], Any], + **ctx: Any, +) -> Any: + """Middleware callback for `run_llm_execution_middleware`(spike 骨架)。 + + fail-closed · fail-open · pass-through 决策: + 1. `_ENABLED` False → pass-through + 2. tenant context 缺失 → pass-through(fail-closed 语义:不 lookup / 不 insert) + 3. request 不可缓存 → pass-through + 4. lookup 命中 → 返回 cached response + cache_meta(不调 next_call) + 5. lookup miss → 调 next_call · response 可缓存则 insert · 返回原 response + """ + if not _ENABLED: + return next_call(request) + + tenant_ctx = _get_tenant_context() + if not tenant_ctx: + return next_call(request) # fail-closed + + if not is_cacheable_request(request): + return next_call(request) + + provider = ctx.get("provider", "") + model = ctx.get("model", "") + key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + + hit = _cache.get(key) + if hit is not None: + # LRU touch + _cache.move_to_end(key) + return _attach_cache_meta(hit, hit_meta={"key": key, "tenant_id": tenant_ctx["tenant_id"]}) + + response = next_call(request) + + if is_cacheable_response(response): + _cache[key] = _extract_cacheable_response(response) + _cache.move_to_end(key) + while len(_cache) > CACHE_MAX_SIZE: + _cache.popitem(last=False) + + return response + + +def _extract_cacheable_response(response: Any) -> Dict[str, Any]: + """归一 provider response 为可 codec 化 dict(§5 codec_v1)。""" + # 简化 spike · 只取 chat_completions dict-like 分支 + if isinstance(response, dict): + return { + "model": response.get("model"), + "choices": response.get("choices"), + "usage": response.get("usage"), + "codec_version": "codec_v1", + } + # 非 dict(SDK 对象)· 提取属性 + return { + "model": getattr(response, "model", None), + "choices": getattr(response, "choices", None), + "usage": getattr(response, "usage", None), + "codec_version": "codec_v1", + } + + +def _attach_cache_meta(cached: Dict[str, Any], *, hit_meta: Dict[str, Any]) -> Dict[str, Any]: + """附 cache_meta(usage 拆分 · §4)。""" + origin_usage = cached.get("usage") or {} + if hasattr(origin_usage, "__dict__"): + origin_usage = {k: getattr(origin_usage, k, None) for k in ("prompt_tokens", "completion_tokens")} + result = dict(cached) + result["cache_meta"] = { + "hit": True, + "cache_contract_version": CACHE_CONTRACT_VERSION, + "origin_usage": origin_usage, + "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "saved_usage": origin_usage, + "cache_lookup_cost": None, # in-memory 通常 0 + "key": hit_meta["key"], + } + return result + + +def clear_cache() -> None: + """spike helper · 测试用。""" + _cache.clear() + + +def register(ctx: Any) -> None: + """Spike registration · 目前不注册任何 hook · fork core middleware 接线由独立 PR。 + + 通过 module-level API(`cache_middleware`, `is_cacheable_request`, ...)供 + 上游 middleware 挂载。fork core 侧的实际挂载(在 + `hermes_cli/middleware.py::LLM_EXECUTION_MIDDLEWARE` list 上加一项)不属于 + plugin ctx 范围 · 需 fork core PR。 + """ + return # M0 spike 不注册 hook · 仅 export API diff --git a/plugins/spike_p3_m0_cache/plugin.yaml b/plugins/spike_p3_m0_cache/plugin.yaml new file mode 100644 index 0000000000000..24451b8a16a68 --- /dev/null +++ b/plugins/spike_p3_m0_cache/plugin.yaml @@ -0,0 +1,4 @@ +name: spike-p3-m0-cache +version: "0.0.1" +description: "P3-M0 Spike · tenant-safe cache prototype · run_llm_execution_middleware seam · memory-only LRU · fail-closed without tenant context. non-streaming + tool-free + successful terminal responses only. usage split into origin/billable/saved. WIP · off by default." +author: "Hermes_AI × NousResearch/hermes-agent Spike" diff --git a/plugins/spike_p3_m0_cache/tests/__init__.py b/plugins/spike_p3_m0_cache/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py new file mode 100644 index 0000000000000..f972d10d6a898 --- /dev/null +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -0,0 +1,199 @@ +"""P3-M0 Spike · cache prototype 测试。 + +覆盖: +- 无 tenant context → fail-closed(pass-through · 不 lookup · 不 insert) +- streaming request → 不 lookup +- tools request → 不 lookup +- response 含 tool_calls → 不 insert +- response finish=length → 不 insert +- lookup hit → 返回 cached + cache_meta(usage 拆分) +- LRU cap · SHA-256 key 稳定 · fallback 后 provider 变 key 变 + +**未接线 fork core middleware**(需独立 PR)· 本测试直接调 `cache_middleware()` +callback 验证语义。 +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Dict + +import pytest + +from plugins.spike_p3_m0_cache import ( + CACHE_CONTRACT_VERSION, + build_cache_key, + cache_middleware, + clear_cache, + is_cacheable_request, + is_cacheable_response, +) + + +@pytest.fixture(autouse=True) +def _cache_isolation(monkeypatch, tmp_path): + """Enable spike cache + isolate HERMES_HOME per test.""" + monkeypatch.setenv("SPIKE_P3_M0_CACHE_ENABLED", "1") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + # Reload the module-level flag(_ENABLED 是 import-time 常量 · 直接 monkeypatch 模块属性) + import plugins.spike_p3_m0_cache as mod + + monkeypatch.setattr(mod, "_ENABLED", True) + clear_cache() + yield + clear_cache() + + +def _write_profile(tmp_path: Path, **overrides) -> None: + import yaml # type: ignore + + profile = { + "tenant": { + "tenant_id": "t_test", + "principal_id": "p_test", + "permission_scope_version": "1", + "profile_version": "1", + **overrides, + } + } + (tmp_path / "profile.yaml").write_text(yaml.safe_dump(profile), encoding="utf-8") + + +# ── Cacheability guards ── + + +def test_streaming_request_not_cacheable(): + assert is_cacheable_request({"stream": True, "messages": []}) is False + + +def test_tools_request_not_cacheable(): + assert is_cacheable_request({"messages": [], "tools": [{"name": "x"}]}) is False + + +def test_normal_request_cacheable(): + assert is_cacheable_request({"messages": [{"role": "user", "content": "hi"}]}) is True + + +def test_response_with_tool_calls_not_cacheable(): + resp = {"choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}]} + assert is_cacheable_response(resp) is False + + +def test_response_finish_not_stop_not_cacheable(): + resp = {"choices": [{"message": {"content": "x"}, "finish_reason": "length"}]} + assert is_cacheable_response(resp) is False + + +def test_response_stop_cacheable(): + resp = {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 5}} + assert is_cacheable_response(resp) is True + + +# ── fail-closed 无 tenant ── + + +def test_no_tenant_context_pass_through(tmp_path): + """无 profile.yaml → pass-through · 不 lookup 不 insert。""" + # HERMES_HOME set but profile 未写 + calls = [] + + def next_call(_req): + calls.append(1) + return {"choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], "usage": {}} + + request = {"messages": [{"role": "user", "content": "hi"}]} + resp = cache_middleware(request, next_call, provider="openai", model="gpt-4o") + assert calls == [1] # 必调 next + assert "cache_meta" not in resp + + +# ── 有 tenant · hit / miss / codec ── + + +def test_miss_then_hit(tmp_path): + _write_profile(tmp_path) + calls = [] + + def next_call(_req): + calls.append(1) + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached_answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 50}, + } + + request = {"messages": [{"role": "user", "content": "hi"}]} + + # 首次 miss + resp1 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") + assert calls == [1] + assert resp1["choices"][0]["message"]["content"] == "cached_answer" + assert "cache_meta" not in resp1 # miss 无 cache_meta + + # 二次 hit + resp2 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") + assert calls == [1] # next 未再调 + assert resp2["cache_meta"]["hit"] is True + assert resp2["cache_meta"]["cache_contract_version"] == CACHE_CONTRACT_VERSION + assert resp2["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp2["cache_meta"]["origin_usage"] == {"prompt_tokens": 100, "completion_tokens": 50} + assert resp2["cache_meta"]["saved_usage"] == {"prompt_tokens": 100, "completion_tokens": 50} + + +def test_provider_change_invalidates_key(tmp_path): + """Fallback 后 provider 变 · key 变 · miss。""" + _write_profile(tmp_path) + call_providers = [] + + def next_call(_req): + return { + "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], + "usage": {}, + } + + request = {"messages": [{"role": "user", "content": "same"}]} + r1 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") + r2 = cache_middleware(request, next_call, provider="anthropic", model="gpt-4o") + # 两次都 miss(不同 provider)· 均无 cache_meta + assert "cache_meta" not in r1 + assert "cache_meta" not in r2 + + +def test_tool_calls_response_not_inserted(tmp_path): + """tool_calls 响应不 insert · 下次同 request 仍 miss。""" + _write_profile(tmp_path) + calls = [] + + def next_call(_req): + calls.append(1) + return { + "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], + "usage": {}, + } + + request = {"messages": [{"role": "user", "content": "hi"}]} + cache_middleware(request, next_call, provider="p", model="m") + cache_middleware(request, next_call, provider="p", model="m") + assert calls == [1, 1] # 两次都调 next(未 insert) + + +# ── key schema ── + + +def test_key_schema_stable(tmp_path): + _write_profile(tmp_path) + request = {"messages": [{"role": "user", "content": "hi"}]} + ctx = { + "tenant_id": "t1", "principal_id": "p1", + "permission_scope_version": "1", "profile_version": "1", + } + k1 = build_cache_key(request, ctx, model="gpt-4o", provider="openai") + k2 = build_cache_key(request, ctx, model="gpt-4o", provider="openai") + assert k1 == k2 and len(k1) == 64 # sha256 hex + + +def test_key_changes_on_tenant(tmp_path): + request = {"messages": []} + c1 = {"tenant_id": "A", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"} + c2 = {"tenant_id": "B", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"} + assert build_cache_key(request, c1, model="m", provider="p") != build_cache_key(request, c2, model="m", provider="p") diff --git a/plugins/spike_p3_m0_fixtures/__init__.py b/plugins/spike_p3_m0_fixtures/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_fixtures/citation_cases.py b/plugins/spike_p3_m0_fixtures/citation_cases.py new file mode 100644 index 0000000000000..08738cb0fc9c3 --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/citation_cases.py @@ -0,0 +1,220 @@ +"""P3-M0 · Shared strip-sources fixture(冻结 · Hermes_AI + fork 双仓共享). + +**跨仓 SHA-256 校验**:两仓 CI / spike 测试 read 同一份文件内容 · sha256 匹配 +才允许消费(见双仓 PR body 互链字段 "fixture SHA-256")· 任一侧修改 · 双仓 +PR 必须同步更新 hash · 否则跨仓不一致视为 BLOCKER。 + +**覆盖面**: +1. 三态最终响应(normal / with citation / mixed) +2. Unicode 边界(多字节 · surrogate pair · emoji · CJK) +3. 任意字节切分点(streaming S1 每一 byte boundary 都 sanitize 正确) +4. 多来源标记形态(inline `[1]` · fenced `` · yaml frontmatter · html tag) +5. interrupt 未闭合标记(partial 到一半 · sanitize 结果不留半括号) +6. Tool result 含来源(transform_tool_result 挂点候选) +7. History replay 转录(session.list/resume presentation 边界) + +**M0 首轮 strip prototype 覆盖出口** ①(final) + ④(history);其余 ②③⑤ 由 +fixture 冻结契约 · 不实装。 +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable + + +@dataclass(frozen=True) +class StripCase: + """一个 strip 测试样本。 + + Attributes: + name: 测试用例唯一 id(供 pytest parametrize) + raw: 未 strip 的原始字符串(可含 citation) + expected: strip 后期望的字符串 + mode: 触发出口 · "final"/"history"/"streaming"/"interim"/"tool" + notes: 说明(不参与断言 · 供 review) + """ + + name: str + raw: str + expected: str + mode: str + notes: str = "" + + +# ===== 基础 3 态最终响应 ===== +NORMAL_FINAL: list[StripCase] = [ + StripCase( + name="no_citation_pass_through", + raw="Hello world. This is a normal response.", + expected="Hello world. This is a normal response.", + mode="final", + notes="baseline · 无 citation · 原样透传", + ), + StripCase( + name="inline_bracket_citation", + raw="The answer is 42 [1]. Reference: doc_abc.", + expected="The answer is 42. Reference: doc_abc.", + mode="final", + notes="inline `[N]` 形态 · 最常见 · strip 移除包括前置空格", + ), + StripCase( + name="multiple_inline_citations", + raw="Fact one [1]. Fact two [2]. Fact three [3][4].", + expected="Fact one. Fact two. Fact three.", + mode="final", + notes="多来源标记 · 连续 [3][4] 也须移除", + ), + StripCase( + name="fenced_source_block", + raw="Answer is X.\n\n\n- doc_abc\n- doc_xyz\n", + expected="Answer is X.", + mode="final", + notes="fenced ... 块 · 整块移除 · 保留末尾空行处理由 sanitizer 决定", + ), + StripCase( + name="mixed_inline_and_fenced", + raw="Answer [1].\n\n\n[1] doc_abc\n", + expected="Answer.", + mode="final", + notes="inline + fenced 组合 · 两者都移除", + ), +] + + +# ===== Unicode / CJK / emoji 边界 ===== +UNICODE_CASES: list[StripCase] = [ + StripCase( + name="cjk_with_citation", + raw="发货时间是 48 小时 [1]。", + expected="发货时间是 48 小时。", + mode="final", + notes="CJK 中文 + `[N]` inline citation · 中文标点保留", + ), + StripCase( + name="emoji_with_citation", + raw="🚀 Launched! [1] 🎉", + expected="🚀 Launched! 🎉", + mode="final", + notes="Emoji 是多字节 UTF-8 · sanitizer 不得断字符", + ), + StripCase( + name="surrogate_pair_preserved", + raw="𝐇𝐞𝐥𝐥𝐨 [1] 𝕎𝕠𝕣𝕝𝕕", + expected="𝐇𝐞𝐥𝐥𝐨 𝕎𝕠𝕣𝕝𝕕", + mode="final", + notes="astral plane 字符(surrogate pair)· strip 不切裂", + ), + StripCase( + name="rtl_with_citation", + raw="مرحبا [1] بالعالم", + expected="مرحبا بالعالم", + mode="final", + notes="RTL 阿拉伯文 · sanitizer 不打乱方向", + ), +] + + +# ===== Interrupt / partial 未闭合 ===== +INTERRUPT_PARTIAL: list[StripCase] = [ + StripCase( + name="partial_open_bracket", + raw="The answer is 42 [", + expected="The answer is 42", + mode="interim", + notes="interrupt 时未闭合 `[` · 移除半个开括号", + ), + StripCase( + name="partial_open_source_block", + raw="Answer.\n\n\n[1] doc", + expected="Answer.", + mode="interim", + notes="interrupt 时 `` 未闭合 · 从 tag 起截断", + ), + StripCase( + name="partial_open_bracket_with_number", + raw="Fact [1", + expected="Fact", + mode="interim", + notes="`[1` 未闭合 · 移除", + ), +] + + +# ===== Streaming 字节切分点(S1 完整缓冲策略验证) ===== +# 每个 case: 完整字符串 raw + 期望 stripped · 测试时将 raw 按 1..len(raw) 每一 byte +# 切分点分片(chunks) · 送 buffered sanitizer · 最终 emit 必须等于 expected +STREAMING_BYTE_SPLIT: list[StripCase] = [ + StripCase( + name="stream_split_short", + raw="A [1]. B [2].", + expected="A. B.", + mode="streaming", + notes="短字符串 · 每字节 boundary 全测", + ), + StripCase( + name="stream_split_cjk", + raw="答案是 42 [1]。", + expected="答案是 42。", + mode="streaming", + notes="CJK · UTF-8 多字节 · 切分点可能落在字符中间", + ), + StripCase( + name="stream_split_emoji_boundary", + raw="Done 🎉 [1] end", + expected="Done 🎉 end", + mode="streaming", + notes="emoji 字节边界 · sanitizer 需 utf-8 aware buffering", + ), +] + + +# ===== Tool result 含来源(transform_tool_result 挂点) ===== +TOOL_RESULT_CASES: list[StripCase] = [ + StripCase( + name="tool_result_with_inline_source", + raw='{"answer": "X [1]", "sources": ["doc_a"]}', + expected='{"answer": "X", "sources": ["doc_a"]}', + mode="tool", + notes="JSON tool_result 内含 inline citation · 只 strip answer 里的 · 不改 sources 数组", + ), +] + + +# ===== History replay / session.resume 转录 ===== +HISTORY_REPLAY_CASES: list[StripCase] = [ + StripCase( + name="history_assistant_msg_with_citation", + raw="Previous answer: The result is 42 [1]. [2] concurs.", + expected="Previous answer: The result is 42. concurs.", + mode="history", + notes="从 session.resume 拉出的历史 assistant message · 走 presentation sanitizer · raw canonical 不改", + ), +] + + +# ===== 汇总(供 pytest parametrize) ===== +ALL_CASES: list[StripCase] = ( + NORMAL_FINAL + + UNICODE_CASES + + INTERRUPT_PARTIAL + + STREAMING_BYTE_SPLIT + + TOOL_RESULT_CASES + + HISTORY_REPLAY_CASES +) + + +def cases_by_mode(mode: str) -> list[StripCase]: + """按出口模式筛选 cases · 供各 hook prototype 测试消费。""" + return [c for c in ALL_CASES if c.mode == mode] + + +def enumerate_byte_split_chunks(raw: str) -> list[tuple[bytes, bytes]]: + """将 raw 转 UTF-8 后 · 按每 1..N-1 字节切分点返回 (head, tail) 对。 + + 专供 streaming S1 缓冲策略测试:每一切分点送 buffered sanitizer · 最终 + emit 必须等于 sanitize(raw)。切在多字节字符中间(如 UTF-8 continuation + byte)· sanitizer 必须**等待更多字节**,不得强解码为 replacement char。 + """ + encoded = raw.encode("utf-8") + n = len(encoded) + return [(encoded[:i], encoded[i:]) for i in range(1, n)] diff --git a/plugins/spike_p3_m0_strip/__init__.py b/plugins/spike_p3_m0_strip/__init__.py new file mode 100644 index 0000000000000..1fc958fa73a1e --- /dev/null +++ b/plugins/spike_p3_m0_strip/__init__.py @@ -0,0 +1,53 @@ +"""P3-M0 Spike · strip-sources prototype(exit ① 普通最终响应). + +**范围严守**(见 `../spike-p3-m0-fixtures/README.md` 与 Hermes_AI 侧 +`docs/fork/strip-contract.md`): + +- 仅覆盖出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` +- streaming/interim/interrupt/history-replay/tool-result 各出口 fixture 冻结 + · **本 spike 不实装** +- `_sanitize_surrogates`(fresh baseline 已在 turn_finalizer 内)独立于本 hook + · 顺序上 sanitize 后于 transform_llm_output · 二者协同不干扰 +- raw canonical transcript 完全不改(finalizer 已在 :317/:333 写入原文) +- 无第三方 plugin 冲突:hook 返回 non-None str 首个胜出 · 本 plugin 无 None 短路 + +**不是** C4 单点方案(只覆盖 5 出口之 1)。M0 首轮证据 + 契约冻结 · 不称 C4 兑现。 +""" +from __future__ import annotations + +import logging +from typing import Any, Optional + +from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-open 测试 + +logger = logging.getLogger(__name__) + + +def on_transform_llm_output( + *, + response_text: str, + session_id: str = "", + model: str = "", + platform: str = "", + **_kwargs: Any, +) -> Optional[str]: + """Return stripped text · None if unchanged. + + Hook contract(`hermes_cli/plugins.py::VALID_HOOKS`): + - First non-empty str return wins across all plugins consuming this hook. + - None / empty str leaves `final_response` unchanged. + - Fired at `agent/turn_finalizer.py:561`. + """ + try: + stripped = _sanitize_mod.sanitize_presentation(response_text or "", mode="final") + except Exception as exc: # noqa: BLE001 — fail-open · never break chat + logger.warning("spike-p3-m0-strip on_transform_llm_output failed: %s", exc) + return None + if stripped == (response_text or ""): + return None # 无改动 · 保留 + return stripped + + +def register(ctx: Any) -> None: + """PluginContext hook · called at plugin discovery.""" + ctx.register_hook("transform_llm_output", on_transform_llm_output) diff --git a/plugins/spike_p3_m0_strip/plugin.yaml b/plugins/spike_p3_m0_strip/plugin.yaml new file mode 100644 index 0000000000000..93c3ba1bb50ef --- /dev/null +++ b/plugins/spike_p3_m0_strip/plugin.yaml @@ -0,0 +1,6 @@ +name: spike-p3-m0-strip +version: "0.0.1" +description: "P3-M0 Spike · strip-sources prototype · transform_llm_output hook · covers exit ① (normal final response) only. Streaming ② / interim ③ / tool ⑤ / history-replay ④ are fixture-frozen only, not implemented. WIP · do not enable in production." +author: "Hermes_AI × NousResearch/hermes-agent Spike" +hooks: + - transform_llm_output diff --git a/plugins/spike_p3_m0_strip/sanitize.py b/plugins/spike_p3_m0_strip/sanitize.py new file mode 100644 index 0000000000000..ac23efeaf5f2f --- /dev/null +++ b/plugins/spike_p3_m0_strip/sanitize.py @@ -0,0 +1,55 @@ +"""Presentation-boundary sanitizer(spike 骨架 · 只实装 mode="final")。 + +**冻结契约**(见 Hermes_AI 侧 `docs/fork/strip-contract.md` §6): + +``` +sanitize_presentation(text: str, *, mode: str) -> str + mode: + - "final" : 出口 ① `transform_llm_output`(**M0 首轮实装**) + - "history" : 出口 ④ session.resume/list(fixture 冻结 · 不实装) + - "streaming" : 出口 ② 缓冲后 sanitize(fixture 冻结 · 不实装) + - "interim" : 出口 ③ interim_assistant_callback wrap(fixture 冻结) + - "tool" : 出口 ⑤ transform_tool_result(fixture 冻结) +``` + +**策略**(见 strip-contract §4): +- streaming 默认完整缓冲后 sanitize(牺牲实时性 · 保零泄漏)· 有状态 parser 是 + M2+ 优化项 · 必须通过每字节切分点 + Unicode 边界 + interrupt 未闭合测试 +- final mode 只针对完整字符串 · 无跨 chunk 复杂性 +""" +from __future__ import annotations + +import re + +# ── 3 种常见 citation 形态(见 fixture citation_cases.py NORMAL_FINAL) ── + +# 1. inline `[N]` `[N][M]` 形态 · 移除包括前置单空格 +_INLINE_CITATION_RE = re.compile(r" ?\[\d+\](?:\[\d+\])*") + +# 2. fenced `...` 块 · 整块移除(含前后空行) +_FENCED_SOURCES_RE = re.compile( + r"\n{0,2}[\s\S]*?\n?", + re.MULTILINE, +) + + +def sanitize_presentation(text: str, *, mode: str) -> str: + """Strip citation / source markers · 按 mode 返回 sanitized text. + + M0 首轮:仅实装 mode="final"。其他 mode 返回原字符串(不改)· 契约冻结 · 待 + M1+ 分阶段实装。 + """ + if mode != "final": + return text + return _sanitize_final(text) + + +def _sanitize_final(text: str) -> str: + """出口 ① · 普通最终响应 sanitize。""" + if not text: + return text + # 先移 fenced · 避免 fenced 内 `[N]` 被 inline 规则误命中(应整块删) + stripped = _FENCED_SOURCES_RE.sub("", text) + stripped = _INLINE_CITATION_RE.sub("", stripped) + # 收尾 trailing whitespace(移 citation 后可能留) + return stripped.rstrip() diff --git a/plugins/spike_p3_m0_strip/tests/__init__.py b/plugins/spike_p3_m0_strip/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_final.py b/plugins/spike_p3_m0_strip/tests/test_strip_final.py new file mode 100644 index 0000000000000..eb0963acb197b --- /dev/null +++ b/plugins/spike_p3_m0_strip/tests/test_strip_final.py @@ -0,0 +1,105 @@ +"""P3-M0 Spike · strip prototype 出口 ① 测试。 + +**Fixture 双仓共享**: +- Hermes_AI: `tests/fixtures/strip_sources/citation_cases.py` +- fork: `plugins/spike_p3_m0_fixtures/citation_cases.py` +- SHA-256: `0cd6ac14035292103ecf574aa6434cf3fcc4d0bdcbe9f21f32a0ba7e9ae26212` +- 跨仓校验:PR body 互链字段 · 任一侧修改双仓 PR 必须同步更新 hash +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from plugins.spike_p3_m0_fixtures.citation_cases import ( + ALL_CASES, + StripCase, + cases_by_mode, +) +from plugins.spike_p3_m0_strip import on_transform_llm_output +from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + +_HERE = Path(__file__).parent +_FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" + + +# ── Fixture SHA-256 校验(跨仓一致性 门槛)── + +EXPECTED_FIXTURE_SHA256 = "0cd6ac14035292103ecf574aa6434cf3fcc4d0bdcbe9f21f32a0ba7e9ae26212" + + +def test_fixture_sha256_matches_hermes_ai(): + """跨仓 fixture 内容一致性 · SHA-256 必须匹配。""" + fixture_path = _FIXTURE_DIR / "citation_cases.py" + actual = hashlib.sha256(fixture_path.read_bytes()).hexdigest() + assert actual == EXPECTED_FIXTURE_SHA256, ( + f"fixture SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " + f"actual: {actual}\n跨仓 fixture 不一致 · 需同步" + ) + + +# ── 出口 ① 普通最终响应 sanitize(M0 首轮 spike 覆盖)── + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_sanitize_final_mode(case: StripCase): + """全 final-mode fixture case · sanitize_presentation(mode='final') 正确。""" + got = sanitize_presentation(case.raw, mode="final") + assert got == case.expected, ( + f"case={case.name}\n raw: {case.raw!r}\n " + f"expected: {case.expected!r}\n got: {got!r}\n notes: {case.notes}" + ) + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_transform_llm_output_hook(case: StripCase): + """`on_transform_llm_output` hook 契约: + - 有改动 · 返回 stripped str + - 无改动 · 返回 None(第一个 non-None str 胜出)""" + result = on_transform_llm_output(response_text=case.raw) + if case.raw == case.expected: + assert result is None, f"unchanged input should return None · case={case.name}" + else: + assert result == case.expected, f"case={case.name}: got {result!r}" + + +def test_hook_none_returns_none(): + """空输入 · 返回 None(与"不改动"契约一致)· 不 crash。""" + assert on_transform_llm_output(response_text="") is None + assert on_transform_llm_output(response_text=None) is None # type: ignore[arg-type] + + +def test_hook_fail_open_on_exception(monkeypatch): + """sanitizer 崩 · hook 返回 None(fail-open · 不阻断 chat)。""" + from plugins.spike_p3_m0_strip import sanitize as sanitize_mod + + def boom(*args, **kwargs): + raise RuntimeError("simulated sanitizer crash") + + monkeypatch.setattr(sanitize_mod, "sanitize_presentation", boom) + # hook 内 try/except 保护 · 返回 None(final_response 未被覆盖) + result = on_transform_llm_output(response_text="Answer [1].") + assert result is None + + +# ── 其他 mode 冻结契约(M0 首轮不实装 · 只 skeleton)── + + +@pytest.mark.parametrize("mode", ["history", "streaming", "interim", "tool"]) +def test_other_modes_skeleton_pass_through(mode: str): + """非 final mode · 当前 spike skeleton 原样返回(契约冻结 · M1+ 分阶段实装)。""" + text = "Answer [1]. Reference doc_abc." + got = sanitize_presentation(text, mode=mode) + assert got == text, f"mode={mode} should pass-through in M0 spike" + + +# ── 冻结契约 · 出口 ②/③/④/⑤ 待 M1+(fixture 已覆盖 · 实装待议) ── + + +@pytest.mark.parametrize("mode", ["streaming", "interim", "history", "tool"]) +def test_fixture_frozen_for_non_final_modes(mode: str): + """fixture 有非 final mode case · 供后续实装参照。""" + cases = cases_by_mode(mode) + assert len(cases) >= 1, f"fixture 应含 {mode} mode case" From f3aca3124865c5762edf816c5e57cc87066dc0f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 18:57:46 +0000 Subject: [PATCH 02/19] =?UTF-8?q?spike(p3-m0):=209-part=20Codex=20REQUEST?= =?UTF-8?q?=20CHANGES=20fix=20=C2=B7=20orchestrator=20middleware=20=C2=B7?= =?UTF-8?q?=20fail-closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-round Codex REQUEST CHANGES on fork PR #1 · all 9 blockers fixed: 1. Budget enforcement moved from pre_api_request hook (production swallows hook exceptions) into llm_execution middleware seam. cache plugin now registers as orchestrator: cache lookup → hit (no reserve, top-level usage=0) / miss → budget.reserve → next_call → success settle actual / exception release + re-raise. 2. Removed illegitimate config sources · no SPIKE_* env · no fork profile.yaml read · single source of truth = hermes_cli.config.load_config() reading plugins.spike_p3_m0_cache.{enabled,tenant}. 3. Strict tenant fail-closed · all 4 fields required (tenant_id, principal_id, permission_scope_version, profile_version) · non-empty · str type · whitelist chars · len ≤ 128 · rejects default fills like "system"/"0". 4. Cache response usage split · top-level usage = billable_usage = 0 · origin_usage lives only in cache_meta. 5. UTC daily bucket keyed by (tenant_id, YYYY-MM-DD) · threading.RLock · verified by 40-thread barrier concurrent reserve/settle test. 6. Strip regex backward-compat with Hermes_AI ops/audit.py::strip_sources baseline · only strips explicit [source|来源:...] / independent-line source:/来源: · rejects unconditional bare [N] stripping · fail-CLOSED on sanitizer exception returns FAIL_CLOSED_PLACEHOLDER (NOT original text). 7. Fixture SHA-256 cross-repo gate · new SHA 47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666 · 8 marker_types · 24 cases · byte-split enumeration for streaming S1 verification. 8. Real PluginManager discovery tests · new shared helper spike_p3_m0_fixtures/pluginmgr_helper.py · fresh PluginManager() + tmp HERMES_HOME + monkeypatched global · discover_and_load(force=True) · 3 spike test suites all invoke via run_llm_execution_middleware (cache/budget) or invoke_hook("transform_llm_output", ...) (strip) · validates has_middleware/has_hook · does not pollute global manager · disabled-plugin negative-case coverage. 9. Documentation fixes across docs/fork/ · orchestration order added · observer-only contract for budget hook · 4-state budget clarified · M0 final-only scope for strip (2/3/4/5 fixture-frozen · skeleton). Test results: - plugins/spike_p3_m0_cache/tests: 19 passed - plugins/spike_p3_m0_budget/tests: 23 passed - plugins/spike_p3_m0_strip/tests: 63 passed - total: 105 passed in 10.72s Zero non-plugin file changes · opt-in via plugins.enabled · discovery manifest scan side-effect free. Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01FiKVo54oX4Uv2qksAHjYsZ --- plugins/spike_p3_m0_budget/__init__.py | 302 +++++------ plugins/spike_p3_m0_budget/plugin.yaml | 4 +- .../tests/test_budget_reserve.py | 282 ++++++---- plugins/spike_p3_m0_cache/__init__.py | 414 +++++++++------ plugins/spike_p3_m0_cache/plugin.yaml | 4 +- .../tests/test_cache_middleware.py | 490 +++++++++++++----- .../spike_p3_m0_fixtures/citation_cases.py | 366 +++++++++---- .../spike_p3_m0_fixtures/pluginmgr_helper.py | 98 ++++ plugins/spike_p3_m0_strip/__init__.py | 78 ++- plugins/spike_p3_m0_strip/plugin.yaml | 4 +- plugins/spike_p3_m0_strip/sanitize.py | 84 +-- .../tests/test_strip_final.py | 195 +++++-- 12 files changed, 1565 insertions(+), 756 deletions(-) create mode 100644 plugins/spike_p3_m0_fixtures/pluginmgr_helper.py diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py index 20c827ecc7545..5dfc67ea01421 100644 --- a/plugins/spike_p3_m0_budget/__init__.py +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -1,33 +1,29 @@ -"""P3-M0 Spike · tenant budget prototype(pre/post_api_request · T2 · fail-open). - -**范围严守**(见 Hermes_AI 侧 `docs/fork/budget-hook.md`): - -- Reserve @ `pre_api_request` @ `agent/conversation_loop.py:2586` -- Settle @ `post_api_request` @ `agent/conversation_loop.py:6114` -- **明令删除**:`HERMES_DAILY_TOKEN_BUDGET` 冒充 per-tenant budget -- **T2 首选**:tenant-per-process/profile · 从 `HERMES_HOME/profile.yaml` 读 - `tenant.daily_budget_tokens` · 非用户可见 env -- **fail-open**:context 缺失 · 不 reserve · 不 settle · 不阻断 chat · 不声称多租户已启用 -- cache hit(response 带 `cache_meta.hit=True`)· settle 时不扣(reservation 已在 pre 阶段 - 发生 · 但可选:M0 spike 简化为 pre 只在 middleware short-circuit 前判断) - -**M0 首轮 spike 实装**: -- `_get_tenant_budget_context()` 从 profile.yaml 读 -- module-level `_tenant_budget_bucket`(简 dict · 单进程 · 非 persistent) -- reserve / settle 契约 -- BudgetExceeded exception - -**M0 首轮 spike 不做**: -- 生产接线 fork core · plugin ctx.register_hook 已挂 · 但 fork core 需装配 `hermes tools` 类似方式启用 -- CAS · 跨进程 budget 同步 -- Hermes_AI SQL sync(等 M1+ · 走 out-of-band gateway sync) +"""P3-M0 Spike · tenant budget service module(no hooks · used by cache orchestrator). + +**Codex §I / §V 契约**: +- **不**通过 `pre_api_request` 抛 BudgetExceeded(生产调用点会吞异常) +- 由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内部按序调用: + cache lookup → miss → **budget reserve** → next_call → success settle actual / + exception release reservation +- `pre_api_request/post_api_request` 只做观察 audit(见 __init__.py 底部可选注册) + +**4 态 budget**(与 Hermes_AI P2-0.1 一致): +- 缺失(config 无 `tenant.daily_budget_tokens` 键)· 不启用租户 budget +- `null`(显式 None)· 不启用(provisioning 明确无限期不 enforce · 与 0 区分) +- `0`(显式 int 0)· 显式无限 · reserve 恒 True · used 仍计 +- `>0`(int)· 具体上限 +- **拒**:bool / 负数 / str / float · 配置错误 · 视 as 缺失 · fail-open + audit + +**UTC daily bucket**:key = (tenant_id, utc_date_str) · 天粒度 · pending reservations 与 +used 一起判断超支。 + +**并发**:threading.Lock · reserve/settle/release 全序 · api_request_id 唯一. """ from __future__ import annotations +import datetime import logging -import os import threading -from pathlib import Path from typing import Any, Dict, Optional logger = logging.getLogger(__name__) @@ -36,168 +32,148 @@ class BudgetExceeded(RuntimeError): - """租户预算超支 · 阻断本次 API 调用。""" + """租户预算超支 · **不由 hook 抛** · 由 llm_execution middleware 内部捕获转 audit。""" -# ── Module state · 单进程 · 非 persistent(M0) ── +class BudgetConfigError(ValueError): + """budget 配置字段类型/值非法 · 视 as fail-open(不 enforce · 记录 audit)。""" -_lock = threading.Lock() -_reservations: Dict[str, Dict[str, Any]] = {} # api_request_id → {tenant_id, tokens} -_daily_used: Dict[str, int] = {} # tenant_id → tokens_used_today(no daily reset · M0 spike) -_ENABLED = os.environ.get("SPIKE_P3_M0_BUDGET_ENABLED", "0") == "1" +# ── Module state · 单进程 · 非 persistent ───────────────────────────── +_lock = threading.RLock() # reentrant · reserve/settle 内部可能连续调用 +# 天粒度桶 · key = (tenant_id, "YYYY-MM-DD" UTC) · value = { "used": int, "reservations": {req_id: tokens} } +_bucket: Dict[str, Dict[str, Any]] = {} -def _get_tenant_budget_context() -> Optional[Dict[str, Any]]: - """T2 · 从 HERMES_HOME/profile.yaml 读 tenant + budget。 - Returns dict with: tenant_id, daily_budget_tokens(int|None), budget_contract_version +def _today_utc() -> str: + """UTC 日期字符串 · YYYY-MM-DD 格式 · 天粒度 bucket key。""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + + +def _bucket_key(tenant_id: str, utc_date: Optional[str] = None) -> str: + return f"{tenant_id}::{utc_date or _today_utc()}" + + +def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: + key = _bucket_key(tenant_id) + b = _bucket.get(key) + if b is None: + b = {"used": 0, "reservations": {}} + _bucket[key] = b + return b + + +# ── 4 态 budget validation ─────────────────────────────────────────── + +def validate_daily_budget(value: Any) -> Optional[int]: + """校验 config `tenant.daily_budget_tokens` 字段 · 返回归一化 int | None。 + + None → None(未启用) + 0 → 0(显式无限) + 正整数 → 正整数 + bool / 负数 / str / float → raise BudgetConfigError · 上层视 as None(fail-open + audit) """ - hermes_home_env = os.environ.get("HERMES_HOME") - if not hermes_home_env: + if value is None: return None - profile_path = Path(hermes_home_env) / "profile.yaml" - if not profile_path.exists(): - return None - try: - import yaml # type: ignore + # bool 是 int 子类 · 必须先拒 + if isinstance(value, bool): + raise BudgetConfigError(f"daily_budget_tokens type=bool 非法 · got {value!r}") + if not isinstance(value, int): + raise BudgetConfigError(f"daily_budget_tokens 需 int · got {type(value).__name__} {value!r}") + if value < 0: + raise BudgetConfigError(f"daily_budget_tokens 需 >=0 · got {value}") + return value - raw = yaml.safe_load(profile_path.read_text(encoding="utf-8")) or {} - except Exception as exc: # noqa: BLE001 - logger.warning("spike-budget: profile.yaml read failed: %s", exc) - return None - tenant_block = raw.get("tenant") or {} - tid = tenant_block.get("tenant_id") - if not tid: - return None - daily = tenant_block.get("daily_budget_tokens") - if daily is not None: - try: - daily = int(daily) - except (TypeError, ValueError): - daily = None - return { - "tenant_id": str(tid), - "daily_budget_tokens": daily, - "budget_contract_version": BUDGET_CONTRACT_VERSION, - } - - -# ── Hook callbacks ── - - -def on_pre_api_request( - *, - task_id: str = "", - turn_id: str = "", - api_request_id: str = "", - session_id: str = "", - platform: str = "", - model: str = "", - provider: str = "", - base_url: str = "", - api_mode: str = "", - api_call_count: int = 0, - retry_count: int = 0, - request_messages: Any = None, - message_count: int = 0, - tool_count: int = 0, - approx_input_tokens: int = 0, - request_char_count: int = 0, - max_tokens: int = 0, - started_at: float = 0.0, - middleware_trace: Any = None, - request: Any = None, - **_ctx: Any, -) -> None: - """Reserve tokens against tenant daily budget. - - Fail-open policy(§4 budget-hook.md): - - `_ENABLED` False → skip - - no tenant context → skip · **不声称多租户已启用** - - no daily_budget set → skip - - reservation exceeds budget → raise `BudgetExceeded`(fail-close · 阻断) + +# ── Reserve / Settle / Release · 由 orchestrator middleware 调用 ───── + +def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: Optional[int]) -> None: + """尝试预留 · 超预算 raise BudgetExceeded · budget=None 直接 no-op(未启用)。 + + `budget=0` 是显式无限 · reserve 恒成功 · used 仍会累加供审计观察。 """ - if not _ENABLED or not api_request_id: - return - ctx = _get_tenant_budget_context() - if not ctx or ctx.get("daily_budget_tokens") is None: - return - tid = ctx["tenant_id"] - budget = ctx["daily_budget_tokens"] - tokens_needed = int(approx_input_tokens or 0) + int(max_tokens or 0) + if not api_request_id: + raise BudgetExceeded("reserve: empty api_request_id · 拒绝 anonymous 预留") + if tokens_needed < 0: + tokens_needed = 0 + if budget is None: + return # 未启用 with _lock: - used = _daily_used.get(tid, 0) - if used + tokens_needed > budget: + b = _get_or_create_bucket(tenant_id) + # api_request_id 唯一 · 重复 reserve 视 as 错误 + if api_request_id in b["reservations"]: raise BudgetExceeded( - f"tenant {tid} daily budget exhausted " - f"(used={used}, need={tokens_needed}, budget={budget})" + f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" ) - _reservations[api_request_id] = {"tenant_id": tid, "tokens": tokens_needed} - - -def on_post_api_request( - *, - task_id: str = "", - turn_id: str = "", - api_request_id: str = "", - session_id: str = "", - platform: str = "", - model: str = "", - provider: str = "", - base_url: str = "", - api_mode: str = "", - api_call_count: int = 0, - api_duration: float = 0.0, - started_at: float = 0.0, - ended_at: float = 0.0, - finish_reason: str = "", - message_count: int = 0, - response_model: Any = None, - response: Any = None, - usage: Any = None, - assistant_message: Any = None, - assistant_content_chars: int = 0, - assistant_tool_call_count: int = 0, - **_ctx: Any, -) -> None: - """Settle actual usage(usage 已 canonicalized by fork · conversation_loop:3437-3448)。 - - Cache hit(response 带 `cache_meta.hit=True`)· settle 不扣(reservation 已 pop)。 - """ - if not api_request_id: - return + pending_total = sum(b["reservations"].values()) + # budget=0 无限 · used + pending + new 不做上限比较 + if budget > 0: + projected = b["used"] + pending_total + tokens_needed + if projected > budget: + raise BudgetExceeded( + f"tenant {tenant_id} daily budget exhausted " + f"(used={b['used']}, pending={pending_total}, need={tokens_needed}, budget={budget})" + ) + b["reservations"][api_request_id] = tokens_needed + + +def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: + """结算实际用量 · 移除 reservation · 累加 used。""" + if actual_tokens < 0: + actual_tokens = 0 + with _lock: + b = _get_or_create_bucket(tenant_id) + b["reservations"].pop(api_request_id, None) + b["used"] = b["used"] + actual_tokens + + +def release(tenant_id: str, api_request_id: str) -> None: + """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。""" + with _lock: + b = _get_or_create_bucket(tenant_id) + b["reservations"].pop(api_request_id, None) + + +# ── 观察 helpers · 测试用 ───────────────────────────────────────────── + +def get_used(tenant_id: str) -> int: with _lock: - reservation = _reservations.pop(api_request_id, None) - if not reservation: - return - # Cache hit 保护:若 response 是 cache hit · 不 charge actual usage - is_cache_hit = False - if isinstance(response, dict): - cache_meta = response.get("cache_meta") or {} - is_cache_hit = bool(cache_meta.get("hit")) - if is_cache_hit: - return # cache hit · reserve 归零 · 不 charge - actual = 0 - if isinstance(usage, dict): - actual = int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) + return _get_or_create_bucket(tenant_id)["used"] + + +def get_pending_total(tenant_id: str) -> int: with _lock: - _daily_used[reservation["tenant_id"]] = _daily_used.get(reservation["tenant_id"], 0) + actual + return sum(_get_or_create_bucket(tenant_id)["reservations"].values()) -# ── Spike helpers · 测试用 ── +def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: + with _lock: + return _get_or_create_bucket(tenant_id)["reservations"].get(api_request_id) def reset_state() -> None: + """测试用 · 清空全部 bucket。""" with _lock: - _reservations.clear() - _daily_used.clear() + _bucket.clear() + +# ── register: 只挂 observer hooks · 无 enforcement · 无 middleware ──── -def get_daily_used(tenant_id: str) -> int: - return _daily_used.get(tenant_id, 0) +def _on_pre_api_request_observer(**_kwargs: Any) -> None: + """观察者:reserve 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return + + +def _on_post_api_request_observer(**_kwargs: Any) -> None: + """观察者:settle 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return def register(ctx: Any) -> None: - ctx.register_hook("pre_api_request", on_pre_api_request) - ctx.register_hook("post_api_request", on_post_api_request) + """PluginContext register · 只挂观察 hooks · 不含 enforcement。 + + 真实 enforcement 在 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内。 + """ + ctx.register_hook("pre_api_request", _on_pre_api_request_observer) + ctx.register_hook("post_api_request", _on_post_api_request_observer) diff --git a/plugins/spike_p3_m0_budget/plugin.yaml b/plugins/spike_p3_m0_budget/plugin.yaml index a6b9afd014db6..dfa81c27c21da 100644 --- a/plugins/spike_p3_m0_budget/plugin.yaml +++ b/plugins/spike_p3_m0_budget/plugin.yaml @@ -1,7 +1,7 @@ name: spike-p3-m0-budget version: "0.0.1" -description: "P3-M0 Spike · tenant budget prototype · pre_api_request reserve + post_api_request settle · T2 tenant-per-process/profile · host-owned session context · fail-open without tenant context · MUST NOT use HERMES_DAILY_TOKEN_BUDGET to impersonate per-tenant budget. WIP · off by default." +description: "P3-M0 Spike · tenant budget service (UTC daily bucket, 4-state validation, thread-safe reserve/settle/release) called from spike-p3-m0-cache orchestrator middleware. Registers observer-only pre_api_request / post_api_request hooks (NO enforcement — production call sites swallow hook exceptions). MUST NOT use HERMES_DAILY_TOKEN_BUDGET to impersonate per-tenant budget. Off by default (opt-in via plugins.enabled)." author: "Hermes_AI × NousResearch/hermes-agent Spike" -hooks: +provides_hooks: - pre_api_request - post_api_request diff --git a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py index f4f869c3655a0..23a392dc857b1 100644 --- a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py +++ b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py @@ -1,142 +1,224 @@ -"""P3-M0 Spike · budget prototype 测试。 - -覆盖(见 `docs/fork/budget-hook.md` §9): -- 无 tenant context → fail-open(不 reserve · 不 settle · 不 raise) -- tenant context + budget=None → 不 reserve -- reserve 成功 → settle 匹配 usage -- reserve 失败(超预算)→ raise BudgetExceeded -- cache hit → settle 不 charge(reserve 后被 middleware short-circuit 时) -- retry / fallback → reservation 独立(每 api_request_id 一次) +"""P3-M0 Spike · budget service 测试(观察 hook · 无 enforcement). + +**Codex §I 契约调整**: +- Budget 不再由 `pre_api_request` hook 抛 BudgetExceeded(生产调用点吞异常) +- 现由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 调用 budget service +- 本文件测 service 函数 · orchestration 端到端由 `spike_p3_m0_cache` 测试覆盖 +- Discovery 侧仍通过真实 PluginManager 验证:hook 挂上、observer 只做 audit """ from __future__ import annotations -from pathlib import Path +import threading import pytest -from plugins.spike_p3_m0_budget import ( - BudgetExceeded, - get_daily_used, - on_post_api_request, - on_pre_api_request, - reset_state, +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + write_config, ) -@pytest.fixture(autouse=True) -def _budget_isolation(monkeypatch, tmp_path): - monkeypatch.setenv("SPIKE_P3_M0_BUDGET_ENABLED", "1") - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - import plugins.spike_p3_m0_budget as mod +BUDGET_KEY = "spike-p3-m0-budget" - monkeypatch.setattr(mod, "_ENABLED", True) - reset_state() - yield - reset_state() +def _enable_budget(tmp_path, monkeypatch): + write_config(tmp_path, [BUDGET_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, BUDGET_KEY) + mod.reset_state() + return manager, mod -def _write_profile(tmp_path: Path, *, tenant_id="t_test", daily=None): - import yaml # type: ignore - tenant_block = {"tenant_id": tenant_id} - if daily is not None: - tenant_block["daily_budget_tokens"] = daily - (tmp_path / "profile.yaml").write_text( - yaml.safe_dump({"tenant": tenant_block}), encoding="utf-8" - ) +# ── Registration(§VIII 真实 PluginManager 门槛) ────────────────── -# ── fail-open scenarios ── +def test_discovery_registers_observer_hooks(tmp_path, monkeypatch): + """PluginContext.register_hook 生效 · pre/post_api_request 均挂上。""" + from hermes_cli.plugins import has_hook + manager, mod = _enable_budget(tmp_path, monkeypatch) + # observer 挂上(无 enforcement · 见模块 docstring) + assert has_hook("pre_api_request") is True + assert has_hook("post_api_request") is True + # 不 register llm_execution middleware(orchestration 是 cache 的责任) + assert manager._middleware.get("llm_execution") in (None, []) -def test_no_tenant_context_no_reserve(tmp_path): - """无 profile.yaml → fail-open · 不 raise · 不影响 state。""" - on_pre_api_request(api_request_id="req_1", approx_input_tokens=100, max_tokens=200) - on_post_api_request(api_request_id="req_1", usage={"prompt_tokens": 100, "completion_tokens": 50}) - assert get_daily_used("t_test") == 0 +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook -def test_tenant_without_budget_no_reserve(tmp_path): - """tenant 有 · daily_budget=None → 不 reserve · 不 settle · 不 raise。""" - _write_profile(tmp_path, daily=None) - on_pre_api_request(api_request_id="req_2", approx_input_tokens=100, max_tokens=200) - on_post_api_request(api_request_id="req_2", usage={"prompt_tokens": 100, "completion_tokens": 50}) - assert get_daily_used("t_test") == 0 + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + # 无 hook 挂上 + assert has_hook("pre_api_request") is False -# ── reserve / settle 正常路径 ── +# ── validate_daily_budget · 4 态 + 拒非法(§V) ───────────────────── -def test_reserve_then_settle_charges_actual(tmp_path): - _write_profile(tmp_path, daily=10000) - on_pre_api_request(api_request_id="req_A", approx_input_tokens=100, max_tokens=200) - on_post_api_request( - api_request_id="req_A", - usage={"prompt_tokens": 80, "completion_tokens": 40}, - ) - # actual = 80+40 = 120(reserve 是 300 · settle 时按 actual 扣) - assert get_daily_used("t_test") == 120 +def test_validate_daily_budget_states(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.validate_daily_budget(None) is None + assert mod.validate_daily_budget(0) == 0 # 显式无限 + assert mod.validate_daily_budget(1000) == 1000 -# ── reserve 失败 · raise BudgetExceeded ── +@pytest.mark.parametrize( + "bad_value", + [True, False, -1, "1000", 1.5, [], {}], + ids=["true", "false", "negative", "string", "float", "list", "dict"], +) +def test_validate_daily_budget_rejects(tmp_path, monkeypatch, bad_value): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetConfigError): + mod.validate_daily_budget(bad_value) -def test_reserve_exceeds_budget_raises(tmp_path): - _write_profile(tmp_path, daily=100) - # 首次 reserve 200 · 超预算 100 · raise - with pytest.raises(BudgetExceeded, match="daily budget exhausted"): - on_pre_api_request(api_request_id="req_over", approx_input_tokens=100, max_tokens=200) +# ── reserve / settle · 基本路径 ───────────────────────────────────── -def test_multiple_reserves_accumulate(tmp_path): - _write_profile(tmp_path, daily=500) - # 第一次 reserve 100 + settle 90(实用)· daily_used=90 - on_pre_api_request(api_request_id="r1", approx_input_tokens=50, max_tokens=50) - on_post_api_request(api_request_id="r1", usage={"prompt_tokens": 50, "completion_tokens": 40}) - assert get_daily_used("t_test") == 90 - # 第二次 reserve 100 · 累计 used=90 · 90+100 <= 500 · 通过 - on_pre_api_request(api_request_id="r2", approx_input_tokens=50, max_tokens=50) - on_post_api_request(api_request_id="r2", usage={"prompt_tokens": 50, "completion_tokens": 60}) - assert get_daily_used("t_test") == 200 +def test_reserve_none_budget_no_op(tmp_path, monkeypatch): + """budget=None(未启用)· reserve no-op · 不 raise。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_a", 500, budget=None) + assert mod.get_pending_total("t1") == 0 + assert mod.get_reservation("t1", "req_a") is None -# ── cache hit · settle 不 charge ── +def test_reserve_then_settle_charges_actual(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 300, budget=10_000) + assert mod.get_pending_total("t1") == 300 + mod.settle("t1", "req_1", 120) + assert mod.get_used("t1") == 120 + assert mod.get_pending_total("t1") == 0 -def test_cache_hit_settle_no_charge(tmp_path): - _write_profile(tmp_path, daily=10000) - on_pre_api_request(api_request_id="req_cache", approx_input_tokens=100, max_tokens=200) - # response 带 cache_meta.hit=True · settle 不 charge - on_post_api_request( - api_request_id="req_cache", - response={"cache_meta": {"hit": True}}, - usage={"prompt_tokens": 80, "completion_tokens": 40}, - ) - assert get_daily_used("t_test") == 0 +def test_release_removes_reservation_no_charge(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 200, budget=10_000) + mod.release("t1", "req_x") + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +# ── reserve 超预算 · raise BudgetExceeded ──────────────────────── + + +def test_reserve_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="daily budget exhausted"): + mod.reserve("t1", "req_over", 200, budget=100) + # 不能 leak reservation + assert mod.get_pending_total("t1") == 0 + + +def test_used_plus_pending_plus_new_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_a", 60, budget=200) + mod.settle("t1", "r_a", 60) # used=60 + mod.reserve("t1", "r_b", 100, budget=200) # pending=100 · used=60 · sum=160 + with pytest.raises(mod.BudgetExceeded): + mod.reserve("t1", "r_c", 50, budget=200) # 60+100+50=210 > 200 + + +def test_budget_zero_is_unlimited(tmp_path, monkeypatch): + """budget=0 显式无限 · 大额 reserve 仍成功 · used 仍累加供审计。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_big", 10_000_000, budget=0) + mod.settle("t1", "r_big", 5_000_000) + assert mod.get_used("t1") == 5_000_000 + + +# ── api_request_id 唯一性 ───────────────────────────────────────── + + +def test_duplicate_reserve_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_dup", 50, budget=1000) + with pytest.raises(mod.BudgetExceeded, match="duplicate"): + mod.reserve("t1", "req_dup", 50, budget=1000) -# ── retry · reservation 独立 ── +def test_empty_api_request_id_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="anonymous"): + mod.reserve("t1", "", 50, budget=1000) -def test_multiple_api_request_ids_independent(tmp_path): - _write_profile(tmp_path, daily=10000) - on_pre_api_request(api_request_id="attempt_1", approx_input_tokens=50, max_tokens=50) - on_pre_api_request(api_request_id="attempt_2", approx_input_tokens=100, max_tokens=100) - on_post_api_request(api_request_id="attempt_1", usage={"prompt_tokens": 40, "completion_tokens": 30}) - on_post_api_request(api_request_id="attempt_2", usage={"prompt_tokens": 80, "completion_tokens": 60}) - assert get_daily_used("t_test") == 210 # 70 + 140 +# ── UTC daily bucket 隔离 ────────────────────────────────────────── -# ── env not enabled · no-op ── +def test_multiple_tenants_isolated(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("tA", "r1", 40, budget=100) + mod.settle("tA", "r1", 40) + mod.reserve("tB", "r2", 90, budget=100) + mod.settle("tB", "r2", 90) + assert mod.get_used("tA") == 40 + assert mod.get_used("tB") == 90 -def test_disabled_no_op(monkeypatch, tmp_path): - monkeypatch.setenv("SPIKE_P3_M0_BUDGET_ENABLED", "0") - import plugins.spike_p3_m0_budget as mod +# ── 并发 reserve/settle · threading.Lock 正确 ───────────────────── - monkeypatch.setattr(mod, "_ENABLED", False) - _write_profile(tmp_path, daily=100) - # 即使 profile.yaml 有 tenant + budget · disabled 状态不 raise - on_pre_api_request(api_request_id="req_off", approx_input_tokens=1000, max_tokens=1000) - on_post_api_request(api_request_id="req_off", usage={"prompt_tokens": 500}) - assert get_daily_used("t_test") == 0 + +def test_concurrent_reserve_settle_thread_safe(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + N = 40 + barrier = threading.Barrier(N) + + def worker(i): + barrier.wait() + mod.reserve("t_shared", f"req_{i}", 10, budget=10_000) + mod.settle("t_shared", f"req_{i}", 5) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert mod.get_used("t_shared") == N * 5 + assert mod.get_pending_total("t_shared") == 0 + + +# ── settle/release 幂等 · 未知 req_id 不 raise ─────────────────── + + +def test_settle_unknown_id_still_accumulates_used(tmp_path, monkeypatch): + """settle 允许在没有 reservation 的 req_id 上调用 · 仅累加 used(observer 语义)。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.settle("t1", "unknown_req", 25) + assert mod.get_used("t1") == 25 + + +def test_release_unknown_id_no_op(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.release("t1", "unknown_req") # 不 raise + assert mod.get_used("t1") == 0 + + +# ── observer hooks 不 enforce(§I 契约) ───────────────────────── + + +def test_pre_api_request_observer_never_enforces(tmp_path, monkeypatch): + """`pre_api_request` observer hook 恒 no-op · 不 raise · 不改 state。 + + Codex §I 强制:budget enforcement 已经搬进 llm_execution middleware(见 + spike_p3_m0_cache)· 本 observer 只做 audit trigger point。 + """ + from hermes_cli.plugins import invoke_hook + + _, mod = _enable_budget(tmp_path, monkeypatch) + # 无 tenant context · 无 config · 直接调 observer · 不 raise · state 不变 + results = invoke_hook( + "pre_api_request", + api_request_id="r1", + approx_input_tokens=999999, + max_tokens=999999, + ) + # observer 返回 None(_on_pre_api_request_observer)· invoke_hook 过滤 None → 空 list + assert results == [] + assert mod.get_used("t_any") == 0 + assert mod.get_pending_total("t_any") == 0 diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index 34fc2f784db20..df2d430b12dee 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -1,92 +1,165 @@ -"""P3-M0 Spike · tenant-safe cache prototype(middleware seam · fail-closed). - -**范围严守**(见 Hermes_AI 侧 `docs/fork/cache-layer.md`): - -- Seam:`run_llm_execution_middleware` @ `hermes_cli/middleware.py:187`(byte-identical baseline→fresh) -- **fail-closed 无 tenant**:context 缺失 → 走 `next_call(request)` 不 lookup 不 insert -- 只缓存 non-streaming + tool-free + successful terminal · 其他一律不缓存 -- **cache key schema v1**:tenant_id / principal_id / permission_scope_version / provider / model / - system_prompt_version / tool_schema_version / profile_version / knowledge_collection / - knowledge_collection_version / normalized_semantic_messages / cache_contract_version -- **usage 拆分**:origin_usage(元数据 · 不进 totals)· billable_usage=0 · saved_usage · cache_lookup_cost -- **cache hit 仍触发**:post_api_request hook · 独立 rate limit · audit `cache.hit` metric - -**M0 首轮 spike 实装**: -- `_get_tenant_context()` = 从 `HERMES_HOME/profile.yaml` 读(T2 · 单进程单 tenant) -- `_is_cacheable_request/response` = §1 严格布尔 -- `_build_cache_key` = §2 schema · sha256 -- `_cache` = 内存 LRU dict(**不 persistent · 不 cross-session · 不 cross-process**) -- `_attach_cache_meta` = 附 cache_meta 字段 - -**M0 首轮 spike 不做**: -- 生产接线到 fork core middleware(需 fork core PR · 独立)· spike 只提供 middleware - callback 可挂式接口 + 完整 fixture 测试 -- persistent / cross-session cache -- streaming 缓冲 codec +"""P3-M0 Spike · tenant-safe cache + orchestrator(llm_execution middleware). + +**Codex §I / §III / §IV 契约**: + +1. 注册 `llm_execution` middleware(`ctx.register_middleware("llm_execution", ...)`)· + 经过 `run_llm_execution_middleware`(`hermes_cli/middleware.py:187`)真正生产走位· + fail-open on callback(fork middleware 语义 · 见 middleware.py:_run_execution_chain) +2. 显式 orchestrate order: + cache lookup + ├─ hit → 返回 cached + 顶层 usage=0 · cache_meta.hit=True · **不 reserve budget** + └─ miss → budget reserve + → next_call(request) + ├─ success → cache insert(if cacheable) + budget settle(actual) + └─ exception → budget release · re-raise +3. 严格 tenant fail-closed:tenant_id/principal_id/permission_scope_version/ + profile_version 任一缺失/空/非法 → 走 next_call(不 lookup 不 insert)· log context_invalid +4. **只**支持 `api_mode == "chat_completions"`;其他 3 mode fail-closed pass-through +5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` +6. 并发:threading.Lock 保护 LRU + insert 序 +7. 配置:从 `hermes_cli.config.load_config` 读 `plugins.spike_p3_m0_cache.enabled` 等 + · **不**用 SPIKE_* env · **不**读 profile.yaml """ from __future__ import annotations import hashlib +import importlib import json import logging -import os +import re +import sys +import threading from collections import OrderedDict -from pathlib import Path from typing import Any, Callable, Dict, Optional logger = logging.getLogger(__name__) + +def _budget_module(): + """Locate the budget sibling module regardless of parent package. + + hermes_cli.plugins loads bundled plugins under ``hermes_plugins.*``; running + the plugins directly from a checkout may see ``plugins.*`` instead. Lazy + lookup avoids a hard ``from .. import`` that only works under one layout. + """ + parent = __name__.rsplit(".", 1)[0] if "." in __name__ else "" + if parent: + candidate = f"{parent}.spike_p3_m0_budget" + if candidate in sys.modules: + return sys.modules[candidate] + try: + return importlib.import_module(candidate) + except ImportError: + pass + for name, mod in list(sys.modules.items()): + if name.endswith(".spike_p3_m0_budget") or name == "spike_p3_m0_budget": + return mod + raise ImportError( + "spike_p3_m0_budget module not loaded; enable it in plugins.enabled" + ) + CACHE_CONTRACT_VERSION = "v1" +CODEC_VERSION = "codec_v1" CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent -# ── module-level state · spike scope · fork core middleware 接线由后续 PR ── +# 允许 tenant_id/principal_id/permission_scope_version/profile_version 的字符白名单 +# (类比 Hermes_AI `_validate_tenant_id` · 保守集合) +_ID_ALLOWED_RE = re.compile(r"^[a-zA-Z0-9_\-]+$") +_ID_MAX_LEN = 128 + +# ── Module state · thread-safe LRU ──────────────────────────────────── + +_lock = threading.RLock() _cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() -_ENABLED = os.environ.get("SPIKE_P3_M0_CACHE_ENABLED", "0") == "1" # 默认 OFF · config.yaml 可以覆盖 · 见 spec -# ── Tenant context helper(T2 · profile.yaml)── +# ── Config loading(fork config.yaml · 不用 env) ───────────────────── + +def _load_config() -> Dict[str, Any]: + """Load fork config via `hermes_cli.config.load_config` · returns full config dict.""" + try: + from hermes_cli.config import load_config + + return load_config() or {} + except Exception as exc: # noqa: BLE001 + logger.warning("spike-cache: load_config failed · treated as empty: %s", exc) + return {} + + +def _get_plugin_config() -> Dict[str, Any]: + """Return `plugins.spike_p3_m0_cache` sub-config(dict · empty if未配置).""" + cfg = _load_config() + plugins_cfg = cfg.get("plugins") or {} + return plugins_cfg.get("spike_p3_m0_cache") or {} + + +def _is_enabled() -> bool: + """`plugins.spike_p3_m0_cache.enabled` · 默认 False(opt-in)。""" + return bool(_get_plugin_config().get("enabled")) + +def _get_tenant_context() -> Optional[Dict[str, str]]: + """严格 fail-closed tenant context · 从 config `plugins.spike_p3_m0_cache.tenant` 读。 -def _get_tenant_context() -> Optional[Dict[str, Any]]: - """从 HERMES_HOME/profile.yaml 读 tenant metadata(T2 · 单进程单 tenant)。 + 所有 4 字段(tenant_id/principal_id/permission_scope_version/profile_version)必需· + 非空 · 类型 str · 字符白名单 · 任一违反 → 返回 None(fail-closed · log context_invalid). - 返回 dict 含:tenant_id / principal_id / permission_scope_version / - profile_version(全部 required · 缺一 fail-closed)。 + **禁**默认填 "system"/"0" 等安全值(Codex §III 明令)。 - **T1(单进程多租户)未启用** · host session metadata 通道待 M2+。 + T2 首选:由 provisioning 写入该 tenant 独立 HERMES_HOME 的 config.yaml · fork + 进程仅信 config · 不信 prompt/tool_args。 """ - hermes_home_env = os.environ.get("HERMES_HOME") - if not hermes_home_env: - return None - profile_path = Path(hermes_home_env) / "profile.yaml" - if not profile_path.exists(): - return None + tenant_block = _get_plugin_config().get("tenant") or {} + required = ("tenant_id", "principal_id", "permission_scope_version", "profile_version") + resolved: Dict[str, str] = {} + for field in required: + v = tenant_block.get(field) + if v is None: + logger.info("spike-cache: tenant context invalid · missing field %s", field) + return None + if not isinstance(v, str): + logger.info("spike-cache: tenant context invalid · field %s type=%s(need str)", field, type(v).__name__) + return None + if not v.strip(): + logger.info("spike-cache: tenant context invalid · field %s empty/whitespace", field) + return None + if len(v) > _ID_MAX_LEN: + logger.info("spike-cache: tenant context invalid · field %s len=%d > %d", field, len(v), _ID_MAX_LEN) + return None + if not _ID_ALLOWED_RE.match(v): + logger.info("spike-cache: tenant context invalid · field %s non-whitelist chars", field) + return None + resolved[field] = v + return resolved + + +def _get_budget_for_tenant() -> Optional[int]: + """`plugins.spike_p3_m0_cache.tenant.daily_budget_tokens` · 4 态归一化。 + + 非法值(bool/负/str/float)→ 记录 audit · 返回 None(fail-open · 不 enforce) + """ + raw = (_get_plugin_config().get("tenant") or {}).get("daily_budget_tokens") + budget_mod = _budget_module() try: - import yaml # type: ignore - - with profile_path.open("r", encoding="utf-8") as f: - raw = yaml.safe_load(f) or {} - except Exception as exc: # noqa: BLE001 - logger.warning("spike-cache: profile.yaml read failed: %s", exc) - return None - tenant_block = raw.get("tenant") or {} - tid = tenant_block.get("tenant_id") - if not tid: + return budget_mod.validate_daily_budget(raw) + except budget_mod.BudgetConfigError as exc: + logger.warning("spike-cache: daily_budget_tokens config invalid · not enforced: %s", exc) return None - return { - "tenant_id": str(tid), - "principal_id": str(tenant_block.get("principal_id") or "system"), - "permission_scope_version": str(tenant_block.get("permission_scope_version") or "0"), - "profile_version": str(tenant_block.get("profile_version") or "0"), - } -# ── Cacheability guards(§1 严格集合)── +# ── Cacheability guards(§4)── 只支持 chat_completions ──────────── +def _api_mode_supported(api_mode: str) -> bool: + """M0 只支持 `chat_completions` · 其他 fail-closed pass-through。""" + return api_mode == "chat_completions" -def is_cacheable_request(request: Dict[str, Any]) -> bool: - """仅接收 non-streaming + tool-free 请求。""" + +def is_cacheable_request(request: Any, *, api_mode: str = "chat_completions") -> bool: + if not _api_mode_supported(api_mode): + return False + if not isinstance(request, dict): + return False if request.get("stream"): return False tools = request.get("tools") or [] @@ -96,9 +169,12 @@ def is_cacheable_request(request: Dict[str, Any]) -> bool: def is_cacheable_response(response: Any) -> bool: - """仅接收 successful terminal response(无 tool_calls · finish=stop)。""" + """仅 finish=stop + 无 tool_calls · 支持 SDK 对象和 dict 形态。""" try: - choices = response.get("choices") if isinstance(response, dict) else getattr(response, "choices", None) + if isinstance(response, dict): + choices = response.get("choices") + else: + choices = getattr(response, "choices", None) if not choices: return False first = choices[0] @@ -109,39 +185,41 @@ def is_cacheable_response(response: Any) -> bool: if tool_calls: return False finish = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None) - if finish not in ("stop",): - return False - return True + return finish == "stop" except Exception: # noqa: BLE001 return False -# ── Cache key builder(§2 schema)── - +# ── Cache key builder(§2 schema v1)────────────────────────────── def build_cache_key( request: Dict[str, Any], - tenant_ctx: Dict[str, Any], + tenant_ctx: Dict[str, str], *, model: str, provider: str, ) -> str: - """SHA-256 hex of the deterministic key components.""" - messages_bytes = json.dumps(request.get("messages") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") - system_prompt_hash = hashlib.sha256(messages_bytes[:1024]).hexdigest()[:16] # 简 hash · M0 spike + """SHA-256 of deterministic key components(12 字段 schema v1).""" + messages = request.get("messages") or [] + messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") - tools_hash = hashlib.sha256(tools_bytes).hexdigest()[:16] + # system prompt hash · 从 messages 首 system 消息内容 + sys_msg_content = "" + for m in messages: + if isinstance(m, dict) and m.get("role") == "system": + sys_msg_content = json.dumps(m.get("content"), sort_keys=True, ensure_ascii=False) + break parts = { "tenant_id": tenant_ctx["tenant_id"], "principal_id": tenant_ctx["principal_id"], "permission_scope_version": tenant_ctx["permission_scope_version"], - "provider": provider, - "model": model, - "system_prompt_version": system_prompt_hash, - "tool_schema_version": tools_hash, + "provider": provider or "", + "model": model or "", + "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], + "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], "profile_version": tenant_ctx["profile_version"], - "knowledge_collection": request.get("_kb_collection") or "", - "knowledge_collection_version": request.get("_kb_version") or "", + "knowledge_collection": str(request.get("_kb_collection") or ""), + "knowledge_collection_version": str(request.get("_kb_version") or ""), "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), "cache_contract_version": CACHE_CONTRACT_VERSION, } @@ -149,102 +227,144 @@ def build_cache_key( return hashlib.sha256(canonical).hexdigest() -# ── Cache middleware(可挂载到 run_llm_execution_middleware · fork core 接线待另 PR)── +# ── Response归一化 · codec_v1(仅 chat_completions dict 形态)────── +def _extract_cacheable_response(response: Any) -> Dict[str, Any]: + """Normalize provider response → cacheable dict(codec_v1)· 只 chat_completions dict.""" + def _get(obj, k, default=None): + return obj.get(k, default) if isinstance(obj, dict) else getattr(obj, k, default) + origin_usage_obj = _get(response, "usage") or {} + if not isinstance(origin_usage_obj, dict): + origin_usage_obj = { + "prompt_tokens": getattr(origin_usage_obj, "prompt_tokens", 0) or 0, + "completion_tokens": getattr(origin_usage_obj, "completion_tokens", 0) or 0, + } + choices = _get(response, "choices") + return { + "model": _get(response, "model"), + "choices": choices, + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, # 顶层 usage = billable_usage · 全 0 + "codec_version": CODEC_VERSION, + "cache_meta": { + "hit": True, + "cache_contract_version": CACHE_CONTRACT_VERSION, + "origin_usage": origin_usage_obj, + "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "saved_usage": origin_usage_obj, + "cache_lookup_cost": None, # in-memory · 通常 0 + }, + } -def cache_middleware( - request: Dict[str, Any], + +# ── Orchestrator middleware(cache + budget 显式组合)──────────── + +def cache_and_budget_middleware( + request: Any, next_call: Callable[[Any], Any], **ctx: Any, ) -> Any: - """Middleware callback for `run_llm_execution_middleware`(spike 骨架)。 - - fail-closed · fail-open · pass-through 决策: - 1. `_ENABLED` False → pass-through - 2. tenant context 缺失 → pass-through(fail-closed 语义:不 lookup / 不 insert) - 3. request 不可缓存 → pass-through - 4. lookup 命中 → 返回 cached response + cache_meta(不调 next_call) - 5. lookup miss → 调 next_call · response 可缓存则 insert · 返回原 response + """`llm_execution` middleware · 严格 Codex §I 顺序: + + cache lookup → hit(返回 · 不 reserve)/ miss → reserve → next_call → + success(cache insert + settle actual)/ exception(release + re-raise) + + fail-closed 无 tenant · fail-closed 非 chat_completions · fail-open 非 dict. """ - if not _ENABLED: + api_request_id = ctx.get("api_request_id") or "" + api_mode = ctx.get("api_mode") or "chat_completions" + provider = ctx.get("provider") or "" + model = ctx.get("model") or "" + + # Fast pass-through · 未启用 / 不支持 mode / 非 dict request + if not _is_enabled() or not _api_mode_supported(api_mode) or not isinstance(request, dict): return next_call(request) tenant_ctx = _get_tenant_context() if not tenant_ctx: - return next_call(request) # fail-closed + # 严格 fail-closed · 无 tenant → 走 next_call(不 lookup 不 insert · 不 reserve) + return next_call(request) - if not is_cacheable_request(request): + if not is_cacheable_request(request, api_mode=api_mode): return next_call(request) - provider = ctx.get("provider", "") - model = ctx.get("model", "") key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + tenant_id = tenant_ctx["tenant_id"] - hit = _cache.get(key) + # ── 1. Cache lookup ── + with _lock: + hit = _cache.get(key) + if hit is not None: + _cache.move_to_end(key) # LRU touch if hit is not None: - # LRU touch - _cache.move_to_end(key) - return _attach_cache_meta(hit, hit_meta={"key": key, "tenant_id": tenant_ctx["tenant_id"]}) + # Hit · 返回 cached · **不 reserve budget** · 顶层 usage=0 + return dict(hit) # shallow copy · 避免调用方 mutate + + # ── 2. Miss · budget reserve(basedon config budget) ── + budget_value = _get_budget_for_tenant() + approx_input = int(ctx.get("approx_input_tokens") or 0) + max_tokens = int(request.get("max_tokens") or ctx.get("max_tokens") or 0) + tokens_needed = approx_input + max_tokens - response = next_call(request) + budget_mod = _budget_module() + try: + budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) + except budget_mod.BudgetExceeded: + # 预算超支 · 明确阻断 · 不调 next_call · 返回结构化 error response + logger.warning("spike-cache-orchestrator: budget exceeded · tenant=%s api_request_id=%s", tenant_id, api_request_id) + return { + "model": model, + "choices": [{"message": {"content": ""}, "finish_reason": "content_filter"}], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "cache_meta": {"hit": False, "budget_exceeded": True, "tenant_id": tenant_id}, + } + + # ── 3. next_call · success settle actual / exception release ── + try: + response = next_call(request) + except Exception: + budget_mod.release(tenant_id, api_request_id) + raise + + # ── 4. Cache insert(if cacheable) + budget settle actual ── + actual = _extract_actual_usage(response) + budget_mod.settle(tenant_id, api_request_id, actual) if is_cacheable_response(response): - _cache[key] = _extract_cacheable_response(response) - _cache.move_to_end(key) - while len(_cache) > CACHE_MAX_SIZE: - _cache.popitem(last=False) + cached = _extract_cacheable_response(response) + with _lock: + _cache[key] = cached + _cache.move_to_end(key) + while len(_cache) > CACHE_MAX_SIZE: + _cache.popitem(last=False) return response -def _extract_cacheable_response(response: Any) -> Dict[str, Any]: - """归一 provider response 为可 codec 化 dict(§5 codec_v1)。""" - # 简化 spike · 只取 chat_completions dict-like 分支 - if isinstance(response, dict): - return { - "model": response.get("model"), - "choices": response.get("choices"), - "usage": response.get("usage"), - "codec_version": "codec_v1", - } - # 非 dict(SDK 对象)· 提取属性 - return { - "model": getattr(response, "model", None), - "choices": getattr(response, "choices", None), - "usage": getattr(response, "usage", None), - "codec_version": "codec_v1", - } +def _extract_actual_usage(response: Any) -> int: + """从 response.usage 抽 prompt+completion tokens.""" + try: + if isinstance(response, dict): + usage = response.get("usage") or {} + else: + usage = getattr(response, "usage", None) or {} + if isinstance(usage, dict): + return int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) + return int(getattr(usage, "prompt_tokens", 0) or 0) + int(getattr(usage, "completion_tokens", 0) or 0) + except Exception: # noqa: BLE001 + return 0 -def _attach_cache_meta(cached: Dict[str, Any], *, hit_meta: Dict[str, Any]) -> Dict[str, Any]: - """附 cache_meta(usage 拆分 · §4)。""" - origin_usage = cached.get("usage") or {} - if hasattr(origin_usage, "__dict__"): - origin_usage = {k: getattr(origin_usage, k, None) for k in ("prompt_tokens", "completion_tokens")} - result = dict(cached) - result["cache_meta"] = { - "hit": True, - "cache_contract_version": CACHE_CONTRACT_VERSION, - "origin_usage": origin_usage, - "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, - "saved_usage": origin_usage, - "cache_lookup_cost": None, # in-memory 通常 0 - "key": hit_meta["key"], - } - return result +def clear_cache() -> None: + """测试用 · 清空 LRU + reset budget bucket。""" + with _lock: + _cache.clear() -def clear_cache() -> None: - """spike helper · 测试用。""" - _cache.clear() +def cache_size() -> int: + with _lock: + return len(_cache) def register(ctx: Any) -> None: - """Spike registration · 目前不注册任何 hook · fork core middleware 接线由独立 PR。 - - 通过 module-level API(`cache_middleware`, `is_cacheable_request`, ...)供 - 上游 middleware 挂载。fork core 侧的实际挂载(在 - `hermes_cli/middleware.py::LLM_EXECUTION_MIDDLEWARE` list 上加一项)不属于 - plugin ctx 范围 · 需 fork core PR。 - """ - return # M0 spike 不注册 hook · 仅 export API + """PluginContext register · 挂 llm_execution middleware(orchestrator).""" + ctx.register_middleware("llm_execution", cache_and_budget_middleware) diff --git a/plugins/spike_p3_m0_cache/plugin.yaml b/plugins/spike_p3_m0_cache/plugin.yaml index 24451b8a16a68..c659f446c8898 100644 --- a/plugins/spike_p3_m0_cache/plugin.yaml +++ b/plugins/spike_p3_m0_cache/plugin.yaml @@ -1,4 +1,6 @@ name: spike-p3-m0-cache version: "0.0.1" -description: "P3-M0 Spike · tenant-safe cache prototype · run_llm_execution_middleware seam · memory-only LRU · fail-closed without tenant context. non-streaming + tool-free + successful terminal responses only. usage split into origin/billable/saved. WIP · off by default." +description: "P3-M0 Spike · tenant-safe cache + budget orchestrator · registers llm_execution middleware (single seam) · calls sibling spike-p3-m0-budget service · memory-only LRU · fail-closed without tenant context · non-streaming + tool-free + successful terminal responses only · usage split into origin/billable/saved · off by default (opt-in via plugins.enabled)." author: "Hermes_AI × NousResearch/hermes-agent Spike" +provides_hooks: + - "llm_execution (middleware)" diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index f972d10d6a898..fdb342ef02250 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -1,199 +1,409 @@ -"""P3-M0 Spike · cache prototype 测试。 - -覆盖: -- 无 tenant context → fail-closed(pass-through · 不 lookup · 不 insert) -- streaming request → 不 lookup -- tools request → 不 lookup -- response 含 tool_calls → 不 insert -- response finish=length → 不 insert -- lookup hit → 返回 cached + cache_meta(usage 拆分) -- LRU cap · SHA-256 key 稳定 · fallback 后 provider 变 key 变 - -**未接线 fork core middleware**(需独立 PR)· 本测试直接调 `cache_middleware()` -callback 验证语义。 +"""P3-M0 Spike · cache + budget orchestrator middleware 测试(**真实生产走位**). + +**Codex §I / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 走真实 `PluginManager.discover_and_load` +- `has_middleware("llm_execution") == True` +- 通过 `run_llm_execution_middleware` 触发 · **不**直接调 `cache_and_budget_middleware` +- **不用** `SPIKE_*` env(fork AGENTS.md red-line 3)· **不污染**全局 manager +- 覆盖: + * cache miss → budget reserve/settle 各 1 次 + * cache hit → budget reserve 0 次(顶层 usage=0) + * budget 超支 → next_call 0 次(structured error response · budget_exceeded) + * provider exception → reservation 释放 · exception 传播 + * tenant context 缺失/非法 → fail-closed(pass-through · 不 lookup 不 reserve) + * streaming / tools request → pass-through """ from __future__ import annotations -import os -from pathlib import Path -from typing import Any, Dict - import pytest -from plugins.spike_p3_m0_cache import ( - CACHE_CONTRACT_VERSION, - build_cache_key, - cache_middleware, - clear_cache, - is_cacheable_request, - is_cacheable_response, +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + make_run_llm_execution, + write_config, ) -@pytest.fixture(autouse=True) -def _cache_isolation(monkeypatch, tmp_path): - """Enable spike cache + isolate HERMES_HOME per test.""" - monkeypatch.setenv("SPIKE_P3_M0_CACHE_ENABLED", "1") - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - # Reload the module-level flag(_ENABLED 是 import-time 常量 · 直接 monkeypatch 模块属性) - import plugins.spike_p3_m0_cache as mod +CACHE_KEY = "spike-p3-m0-cache" +BUDGET_KEY = "spike-p3-m0-budget" - monkeypatch.setattr(mod, "_ENABLED", True) - clear_cache() - yield - clear_cache() +# ── shared valid tenant block ───────────────────────────────────────── -def _write_profile(tmp_path: Path, **overrides) -> None: - import yaml # type: ignore +_VALID_TENANT = { + "tenant_id": "t_alpha", + "principal_id": "p_alpha", + "permission_scope_version": "1", + "profile_version": "1", +} - profile = { - "tenant": { - "tenant_id": "t_test", - "principal_id": "p_test", - "permission_scope_version": "1", - "profile_version": "1", - **overrides, - } - } - (tmp_path / "profile.yaml").write_text(yaml.safe_dump(profile), encoding="utf-8") +def _enable_cache( + tmp_path, + monkeypatch, + *, + daily_budget=None, + tenant=_VALID_TENANT, +): + tenant_cfg = dict(tenant) + if daily_budget is not None: + tenant_cfg["daily_budget_tokens"] = daily_budget + write_config( + tmp_path, + [CACHE_KEY, BUDGET_KEY], + spike_p3_m0_cache={ + "enabled": True, + "tenant": tenant_cfg, + }, + ) + manager = install_fresh_manager(monkeypatch, tmp_path) + cache_mod = get_discovered_module(manager, CACHE_KEY) + budget_mod = get_discovered_module(manager, BUDGET_KEY) + cache_mod.clear_cache() + budget_mod.reset_state() + return manager, cache_mod, budget_mod -# ── Cacheability guards ── +# ── Registration invariants(§I 强要求) ───────────────────────────── -def test_streaming_request_not_cacheable(): - assert is_cacheable_request({"stream": True, "messages": []}) is False +def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): + """PluginContext.register_middleware("llm_execution", ...) 生效 · fork middleware.py 契约。""" + from hermes_cli.plugins import has_middleware -def test_tools_request_not_cacheable(): - assert is_cacheable_request({"messages": [], "tools": [{"name": "x"}]}) is False + manager, _, _ = _enable_cache(tmp_path, monkeypatch) + assert has_middleware("llm_execution") is True + assert len(manager._middleware.get("llm_execution", [])) == 1 -def test_normal_request_cacheable(): - assert is_cacheable_request({"messages": [{"role": "user", "content": "hi"}]}) is True +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + """`plugins.enabled` 不含 spike-p3-m0-cache · discover 不 load · 无 middleware。""" + from hermes_cli.plugins import has_middleware + write_config(tmp_path, []) # 空 enabled · 无插件加载 + install_fresh_manager(monkeypatch, tmp_path) + assert has_middleware("llm_execution") is False -def test_response_with_tool_calls_not_cacheable(): - resp = {"choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}]} - assert is_cacheable_response(resp) is False +# ── Cache miss → budget reserve+settle 各 1 次(§I 强要求) ───────── -def test_response_finish_not_stop_not_cacheable(): - resp = {"choices": [{"message": {"content": "x"}, "finish_reason": "length"}]} - assert is_cacheable_response(resp) is False +def test_miss_reserves_and_settles_once(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) -def test_response_stop_cacheable(): - resp = {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 5}} - assert is_cacheable_response(resp) is True + call_count = {"next": 0} + def next_call(_req): + call_count["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 50}, + } -# ── fail-closed 无 tenant ── + run = make_run_llm_execution( + api_request_id="req_miss_1", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert call_count["next"] == 1 + assert resp["choices"][0]["message"]["content"] == "hi" + # settle 累加 actual usage = 150 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 150 + # 无 pending reservations + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 -def test_no_tenant_context_pass_through(tmp_path): - """无 profile.yaml → pass-through · 不 lookup 不 insert。""" - # HERMES_HOME set but profile 未写 - calls = [] - def next_call(_req): - calls.append(1) - return {"choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], "usage": {}} +# ── Cache hit → budget reserve 0 次 · 顶层 usage=0(§I 强要求) ───── - request = {"messages": [{"role": "user", "content": "hi"}]} - resp = cache_middleware(request, next_call, provider="openai", model="gpt-4o") - assert calls == [1] # 必调 next - assert "cache_meta" not in resp +def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) -# ── 有 tenant · hit / miss / codec ── + counters = {"next": 0} + def next_call(_req): + counters["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 200, "completion_tokens": 80}, + } -def test_miss_then_hit(tmp_path): - _write_profile(tmp_path) - calls = [] + run = make_run_llm_execution( + api_request_id="req_a", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + ) + req = {"messages": [{"role": "user", "content": "same"}]} + + # 首次 miss · settle 280 + run(req, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 + + # 二次 hit · next_call 0 次 · used 不变(reserve 0 次) + run2 = make_run_llm_execution( + api_request_id="req_b", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + ) + resp = run2(req, next_call) + assert counters["next"] == 1 # 未再调 next + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 # 未 charge + # 顶层 usage = billable_usage = 全 0 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + # cache_meta 拆分正确 + assert resp["cache_meta"]["hit"] is True + assert resp["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp["cache_meta"]["origin_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["saved_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["cache_contract_version"] == cache_mod.CACHE_CONTRACT_VERSION + # 二次调用不该有 pending reservation(hit 直接返回) + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Budget 超支 → next_call 0 次(§I 强要求) ──────────────────────── + + +def test_budget_exceeded_blocks_next_call(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100 # 极小 + ) + + counters = {"next": 0} def next_call(_req): - calls.append(1) + counters["next"] += 1 return { - "model": "gpt-4o", - "choices": [{"message": {"content": "cached_answer"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 100, "completion_tokens": 50}, + "model": "m", + "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, } - request = {"messages": [{"role": "user", "content": "hi"}]} + run = make_run_llm_execution( + api_request_id="req_over", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=500, # 已远超 budget 100 + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, + next_call, + ) + # 超支明确 short-circuit · next 未调 + assert counters["next"] == 0 + # 结构化 error response · budget_exceeded=True + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["tenant_id"] == _VALID_TENANT["tenant_id"] + # 无 pending reservation(reserve 抛异常 · 未记账) + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Provider exception → reservation 释放 + 传播(§I 强要求) ─────── + + +def test_provider_exception_releases_reservation(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + class ProviderBoom(RuntimeError): + pass - # 首次 miss - resp1 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") - assert calls == [1] - assert resp1["choices"][0]["message"]["content"] == "cached_answer" - assert "cache_meta" not in resp1 # miss 无 cache_meta + def next_call(_req): + raise ProviderBoom("upstream 500") + + run = make_run_llm_execution( + api_request_id="req_boom", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=100, + ) + with pytest.raises(ProviderBoom, match="upstream 500"): + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) + + # reservation 已释放 · used 未增 + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None + + +# ── Tenant context fail-closed(§III 严格) ───────────────────────── + + +@pytest.mark.parametrize( + "bad_tenant", + [ + {"tenant_id": "", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, # 缺 profile_version + {"tenant_id": "t/../evil", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "a" * 200, "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + ], + ids=["empty-tenant", "empty-principal", "missing-profile-version", "non-whitelist-chars", "over-max-len"], +) +def test_invalid_tenant_pass_through(tmp_path, monkeypatch, bad_tenant): + """任一 tenant 字段违反 · pass-through(不 lookup 不 reserve)· fail-closed。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, tenant=bad_tenant + ) - # 二次 hit - resp2 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") - assert calls == [1] # next 未再调 - assert resp2["cache_meta"]["hit"] is True - assert resp2["cache_meta"]["cache_contract_version"] == CACHE_CONTRACT_VERSION - assert resp2["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} - assert resp2["cache_meta"]["origin_usage"] == {"prompt_tokens": 100, "completion_tokens": 50} - assert resp2["cache_meta"]["saved_usage"] == {"prompt_tokens": 100, "completion_tokens": 50} + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, # 若 reserve 生效必超支 · 用于反证 fail-closed 走 next + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + assert counters["next"] == 1 # 走 next(未 reserve, 否则会 budget_exceeded) + assert "cache_meta" not in resp -def test_provider_change_invalidates_key(tmp_path): - """Fallback 后 provider 变 · key 变 · miss。""" - _write_profile(tmp_path) - call_providers = [] +def test_no_tenant_config_pass_through(tmp_path, monkeypatch): + """config.yaml 无 tenant 段 · fail-closed pass-through。""" + write_config( + tmp_path, + [CACHE_KEY, BUDGET_KEY], + spike_p3_m0_cache={"enabled": True}, # 无 tenant 段 + ) + manager = install_fresh_manager(monkeypatch, tmp_path) + + counters = {"next": 0} def next_call(_req): - return { - "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], - "usage": {}, - } + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 1 + assert "cache_meta" not in resp + - request = {"messages": [{"role": "user", "content": "same"}]} - r1 = cache_middleware(request, next_call, provider="openai", model="gpt-4o") - r2 = cache_middleware(request, next_call, provider="anthropic", model="gpt-4o") - # 两次都 miss(不同 provider)· 均无 cache_meta - assert "cache_meta" not in r1 - assert "cache_meta" not in r2 +# ── Streaming / tools request pass-through(§4 mode 限定) ─────────── -def test_tool_calls_response_not_inserted(tmp_path): - """tool_calls 响应不 insert · 下次同 request 仍 miss。""" - _write_profile(tmp_path) - calls = [] +def test_streaming_request_pass_through(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} def next_call(_req): - calls.append(1) - return { - "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], - "usage": {}, - } + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + # 首次 · streaming=True → 不 lookup 不 insert + run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + # 二次 · streaming=True · 仍 miss → next 再调 + run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 2 + + +def test_tools_request_pass_through(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + req = {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}]} + run(req, next_call) + run(req, next_call) + assert counters["next"] == 2 # 两次 miss(不 insert) + + +def test_non_chat_completions_mode_pass_through(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - request = {"messages": [{"role": "user", "content": "hi"}]} - cache_middleware(request, next_call, provider="p", model="m") - cache_middleware(request, next_call, provider="p", model="m") - assert calls == [1, 1] # 两次都调 next(未 insert) + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="responses", # 非 chat_completions + ) + req = {"messages": [{"role": "user", "content": "x"}]} + run(req, next_call) + run(req, next_call) + assert counters["next"] == 2 # 两次都 miss(不 insert) -# ── key schema ── +# ── Cache key schema stability(§2 v1 schema) ────────────────────── -def test_key_schema_stable(tmp_path): - _write_profile(tmp_path) - request = {"messages": [{"role": "user", "content": "hi"}]} - ctx = { - "tenant_id": "t1", "principal_id": "p1", - "permission_scope_version": "1", "profile_version": "1", - } - k1 = build_cache_key(request, ctx, model="gpt-4o", provider="openai") - k2 = build_cache_key(request, ctx, model="gpt-4o", provider="openai") - assert k1 == k2 and len(k1) == 64 # sha256 hex +def test_cache_key_stable_and_deterministic(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": [{"role": "user", "content": "hi"}]} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + assert k1 == k2 and len(k1) == 64 -def test_key_changes_on_tenant(tmp_path): - request = {"messages": []} - c1 = {"tenant_id": "A", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"} - c2 = {"tenant_id": "B", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"} - assert build_cache_key(request, c1, model="m", provider="p") != build_cache_key(request, c2, model="m", provider="p") +def test_cache_key_changes_on_provider(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="m", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") + assert k1 != k2 + + +def test_cache_key_changes_on_tenant(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + a = dict(_VALID_TENANT, tenant_id="tA") + b = dict(_VALID_TENANT, tenant_id="tB") + ka = cache_mod.build_cache_key(req, a, model="m", provider="p") + kb = cache_mod.build_cache_key(req, b, model="m", provider="p") + assert ka != kb + + +# ── Tool-call response 不 insert · 下次仍 miss ──────────────────── + + +def test_tool_calls_response_not_inserted(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + + run1 = make_run_llm_execution(api_request_id="r1", provider="p", model="m", api_mode="chat_completions") + run2 = make_run_llm_execution(api_request_id="r2", provider="p", model="m", api_mode="chat_completions") + req = {"messages": [{"role": "user", "content": "call tool"}]} + run1(req, next_call) + run2(req, next_call) + assert counters["next"] == 2 # 两次都 miss · tool_calls 不 insert diff --git a/plugins/spike_p3_m0_fixtures/citation_cases.py b/plugins/spike_p3_m0_fixtures/citation_cases.py index 08738cb0fc9c3..23917bea9da3d 100644 --- a/plugins/spike_p3_m0_fixtures/citation_cases.py +++ b/plugins/spike_p3_m0_fixtures/citation_cases.py @@ -1,24 +1,33 @@ """P3-M0 · Shared strip-sources fixture(冻结 · Hermes_AI + fork 双仓共享). -**跨仓 SHA-256 校验**:两仓 CI / spike 测试 read 同一份文件内容 · sha256 匹配 +**跨仓 SHA-256 校验**:两仓 pytest read 同一份文件内容 · sha256 匹配 才允许消费(见双仓 PR body 互链字段 "fixture SHA-256")· 任一侧修改 · 双仓 PR 必须同步更新 hash · 否则跨仓不一致视为 BLOCKER。 +**向后兼容基线**:Hermes_AI `hermes_devices/ops/audit.py::strip_sources` +(charter C4 · 已合入 main)· fork 侧 sanitize_presentation 至少覆盖该 +基线所有正例 + 加严 negative cases。 + **覆盖面**: -1. 三态最终响应(normal / with citation / mixed) -2. Unicode 边界(多字节 · surrogate pair · emoji · CJK) -3. 任意字节切分点(streaming S1 每一 byte boundary 都 sanitize 正确) -4. 多来源标记形态(inline `[1]` · fenced `` · yaml frontmatter · html tag) -5. interrupt 未闭合标记(partial 到一半 · sanitize 结果不留半括号) -6. Tool result 含来源(transform_tool_result 挂点候选) -7. History replay 转录(session.list/resume presentation 边界) - -**M0 首轮 strip prototype 覆盖出口** ①(final) + ④(history);其余 ②③⑤ 由 -fixture 冻结契约 · 不实装。 +1. 正例(必须 strip):[source:...] · [来源:...] · [source：...] · [来源：...] · + 独立行 source: / 来源: · 全角/半角括号 · 嵌套中文括号 +2. Negative(必须原样保留):bare [1] / [2][3] · Markdown footnote reference · + JSON 数组 [] · 数组下标 arr[0] · 年份 [2024] · 版本 [v1.2] · 代码块内 · + 非 source/来源 关键字的 [xxx] +3. Unicode 边界:CJK · emoji · surrogate pair · RTL +4. Streaming 字节切分点(S1 完整缓冲策略验证)· 每 1..N-1 byte 分片 +5. Interrupt 未闭合标记 · 不吞下一行 +6. Presentation vs raw transcript 边界(canonical 保留 · presentation strip) +7. Tool result 含来源 +8. History replay 转录 + +**M0 首轮 strip prototype 只实装 mode="final"** · 其他 mode(streaming/history/ +interim/tool)是 **skeleton pass-through**(fixture present · implementation +pending)。 """ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Callable @@ -27,193 +36,356 @@ class StripCase: """一个 strip 测试样本。 Attributes: - name: 测试用例唯一 id(供 pytest parametrize) - raw: 未 strip 的原始字符串(可含 citation) - expected: strip 后期望的字符串 - mode: 触发出口 · "final"/"history"/"streaming"/"interim"/"tool" - notes: 说明(不参与断言 · 供 review) + name: 测试用例唯一 id(供 pytest parametrize)· 全 fixture 唯一 + raw: 未 strip 的原始字符串 + expected: strip 后期望的字符串(必须严格 == · 不允许"包含") + mode: 触发出口 · "final" | "history" | "streaming" | "interim" | "tool" + marker_type: 声称触及的 marker 分类 · 供 pytest 校验实际存在 + "source_inline" | "source_line" | "unclosed" | + "unicode" | "streaming_split" | "negative" | + "history_replay" | "tool_result_json" + notes: 说明(不参与断言) """ name: str raw: str expected: str mode: str + marker_type: str notes: str = "" -# ===== 基础 3 态最终响应 ===== -NORMAL_FINAL: list[StripCase] = [ +# ============================================================================ +# 正例(必须 strip · 与 Hermes_AI ops/audit.py::strip_sources 向后兼容) +# ============================================================================ + +INLINE_SOURCE_CASES: list[StripCase] = [ + StripCase( + name="inline_source_ascii", + raw="Answer is X [source: doc_abc].", + expected="Answer is X.", + mode="final", + marker_type="source_inline", + notes="ASCII `[source: X]` · Hermes_AI baseline · 保末尾句点", + ), + StripCase( + name="inline_source_zh_colon", + raw="答案是 X [source:doc_abc]。", + expected="答案是 X。", + mode="final", + marker_type="source_inline", + notes="全角句点 + 半角冒号", + ), StripCase( - name="no_citation_pass_through", - raw="Hello world. This is a normal response.", - expected="Hello world. This is a normal response.", + name="inline_lai_yuan_half_width", + raw="答案是 X [来源: doc_abc]。", + expected="答案是 X。", mode="final", - notes="baseline · 无 citation · 原样透传", + marker_type="source_inline", + notes="中文 `[来源: X]` · 半角括号", ), StripCase( - name="inline_bracket_citation", - raw="The answer is 42 [1]. Reference: doc_abc.", - expected="The answer is 42. Reference: doc_abc.", + name="inline_lai_yuan_full_width", + raw="答案是 X 【来源:doc_abc】。", + expected="答案是 X。", mode="final", - notes="inline `[N]` 形态 · 最常见 · strip 移除包括前置空格", + marker_type="source_inline", + notes="全角方括号 【】", ), StripCase( - name="multiple_inline_citations", - raw="Fact one [1]. Fact two [2]. Fact three [3][4].", - expected="Fact one. Fact two. Fact three.", + name="inline_lai_yuan_full_width_colon", + raw="答案是 X 【来源:doc_abc】。", + expected="答案是 X。", mode="final", - notes="多来源标记 · 连续 [3][4] 也须移除", + marker_type="source_inline", + notes="全角冒号 · 与半角冒号等效处理", ), StripCase( - name="fenced_source_block", - raw="Answer is X.\n\n\n- doc_abc\n- doc_xyz\n", + name="inline_lai_yuan_paren_nested_zh", + raw="答案 【来源:FAQ(2026版)】。", + expected="答案。", + mode="final", + marker_type="source_inline", + notes="嵌套中文括号(2026版)· 一层嵌套 · 与 Hermes_AI baseline 兼容", + ), + StripCase( + name="inline_multiple_sources", + raw="点 A [source: a]。点 B [来源: b]。点 C [source:c]。", + expected="点 A。点 B。点 C。", + mode="final", + marker_type="source_inline", + notes="多个 inline source · 全部移除", + ), +] + +LINE_SOURCE_CASES: list[StripCase] = [ + StripCase( + name="line_source_ascii", + raw="Answer is X.\nsource: doc_abc", expected="Answer is X.", mode="final", - notes="fenced ... 块 · 整块移除 · 保留末尾空行处理由 sanitizer 决定", + marker_type="source_line", + notes="独立行 `source: X` · 整行移除", ), StripCase( - name="mixed_inline_and_fenced", - raw="Answer [1].\n\n\n[1] doc_abc\n", + name="line_lai_yuan_zh", + raw="答案是 X。\n来源:doc_abc", + expected="答案是 X。", + mode="final", + marker_type="source_line", + notes="独立行 `来源: X`", + ), + StripCase( + name="line_source_case_insensitive", + raw="Answer.\nSource: doc_abc", expected="Answer.", mode="final", - notes="inline + fenced 组合 · 两者都移除", + marker_type="source_line", + notes="`Source:`(首字母大写)· case-insensitive", ), ] -# ===== Unicode / CJK / emoji 边界 ===== -UNICODE_CASES: list[StripCase] = [ +# ============================================================================ +# Negative cases(必须原样保留 · Codex §VI 明令) +# ============================================================================ + +NEGATIVE_CASES: list[StripCase] = [ StripCase( - name="cjk_with_citation", - raw="发货时间是 48 小时 [1]。", - expected="发货时间是 48 小时。", + name="neg_bare_bracket_number", + raw="See [1] and [2] for details.", + expected="See [1] and [2] for details.", mode="final", - notes="CJK 中文 + `[N]` inline citation · 中文标点保留", + marker_type="negative", + notes="裸 `[N]` 无结构化 sources metadata · **不得**无条件 strip", ), StripCase( - name="emoji_with_citation", - raw="🚀 Launched! [1] 🎉", - expected="🚀 Launched! 🎉", + name="neg_markdown_footnote", + raw="A footnote reference[^1] and definition below.", + expected="A footnote reference[^1] and definition below.", mode="final", - notes="Emoji 是多字节 UTF-8 · sanitizer 不得断字符", + marker_type="negative", + notes="Markdown footnote `[^N]` · 原样保留", ), StripCase( - name="surrogate_pair_preserved", - raw="𝐇𝐞𝐥𝐥𝐨 [1] 𝕎𝕠𝕣𝕝𝕕", - expected="𝐇𝐞𝐥𝐥𝐨 𝕎𝕠𝕣𝕝𝕕", + name="neg_json_array", + raw='Data: {"items": [1, 2, 3]}.', + expected='Data: {"items": [1, 2, 3]}.', mode="final", - notes="astral plane 字符(surrogate pair)· strip 不切裂", + marker_type="negative", + notes="JSON array · 括号不得 strip", ), StripCase( - name="rtl_with_citation", - raw="مرحبا [1] بالعالم", - expected="مرحبا بالعالم", + name="neg_array_index", + raw="arr[0] and arr[10] access.", + expected="arr[0] and arr[10] access.", mode="final", - notes="RTL 阿拉伯文 · sanitizer 不打乱方向", + marker_type="negative", + notes="数组下标 · 原样保留", + ), + StripCase( + name="neg_year_bracket", + raw="Published in [2024]. Version [v1.2].", + expected="Published in [2024]. Version [v1.2].", + mode="final", + marker_type="negative", + notes="年份/版本号 [2024] [v1.2] · 无 source 关键字 · 原样保留", + ), + StripCase( + name="neg_code_block_brackets", + raw="Code: `arr[i] = arr[j]` swap.", + expected="Code: `arr[i] = arr[j]` swap.", + mode="final", + marker_type="negative", + notes="代码块内的括号 · 原样保留", + ), + StripCase( + name="neg_non_source_keyword_bracket", + raw="Note [important]: check this.", + expected="Note [important]: check this.", + mode="final", + marker_type="negative", + notes="`[important]` 非 source/来源 · 原样保留", ), ] -# ===== Interrupt / partial 未闭合 ===== -INTERRUPT_PARTIAL: list[StripCase] = [ +# ============================================================================ +# 未闭合标记(interrupt · 不吞下一行) +# ============================================================================ + +UNCLOSED_CASES: list[StripCase] = [ StripCase( - name="partial_open_bracket", - raw="The answer is 42 [", - expected="The answer is 42", - mode="interim", - notes="interrupt 时未闭合 `[` · 移除半个开括号", + name="unclosed_open_bracket_only", + raw="Answer is X [source: partial\nNext line intact.", + expected="Answer is X [source: partial\nNext line intact.", + mode="final", + marker_type="unclosed", + notes="未闭合 `[source:` · **保留下一行** · 不吞", ), StripCase( - name="partial_open_source_block", - raw="Answer.\n\n\n[1] doc", - expected="Answer.", + name="unclosed_zh_bracket_at_end", + raw="答案 【来源:doc", + expected="答案 【来源:doc", mode="interim", - notes="interrupt 时 `` 未闭合 · 从 tag 起截断", + marker_type="unclosed", + notes="流式 interrupt · 未闭合中括 · 保留(等待更多 chunk)", ), +] + + +# ============================================================================ +# Unicode 边界 +# ============================================================================ + +UNICODE_CASES: list[StripCase] = [ StripCase( - name="partial_open_bracket_with_number", - raw="Fact [1", - expected="Fact", - mode="interim", - notes="`[1` 未闭合 · 移除", + name="unicode_cjk_full", + raw="发货时间是 48 小时 [source: policy]。", + expected="发货时间是 48 小时。", + mode="final", + marker_type="unicode", + notes="CJK + inline source", + ), + StripCase( + name="unicode_emoji_with_source", + raw="🚀 Launched! [source: press_release] 🎉", + expected="🚀 Launched! 🎉", + mode="final", + marker_type="unicode", + notes="Emoji 多字节 · source strip 不断字符", + ), + StripCase( + name="unicode_surrogate_pair", + raw="𝐇𝐞𝐥𝐥𝐨 [source: mathml] 𝕎𝕠𝕣𝕝𝕕", + expected="𝐇𝐞𝐥𝐥𝐨 𝕎𝕠𝕣𝕝𝕕", + mode="final", + marker_type="unicode", + notes="Astral plane 字符 · surrogate pair 不切裂", + ), + StripCase( + name="unicode_rtl_arabic", + raw="مرحبا [source: greeting] بالعالم", + expected="مرحبا بالعالم", + mode="final", + marker_type="unicode", + notes="RTL 阿拉伯文 · 方向保持", ), ] -# ===== Streaming 字节切分点(S1 完整缓冲策略验证) ===== -# 每个 case: 完整字符串 raw + 期望 stripped · 测试时将 raw 按 1..len(raw) 每一 byte -# 切分点分片(chunks) · 送 buffered sanitizer · 最终 emit 必须等于 expected -STREAMING_BYTE_SPLIT: list[StripCase] = [ +# ============================================================================ +# Streaming 字节切分点(S1 完整缓冲策略验证) +# ============================================================================ + +STREAMING_BYTE_SPLIT_CASES: list[StripCase] = [ StripCase( - name="stream_split_short", - raw="A [1]. B [2].", - expected="A. B.", + name="stream_split_ascii", + raw="A [source: a] B [source: b] end.", + expected="A B end.", mode="streaming", - notes="短字符串 · 每字节 boundary 全测", + marker_type="streaming_split", + notes="ASCII · 每字节 boundary 全测(见 enumerate_byte_split_chunks)", ), StripCase( name="stream_split_cjk", - raw="答案是 42 [1]。", - expected="答案是 42。", + raw="答案 [source: doc] 是 42。", + expected="答案 是 42。", mode="streaming", - notes="CJK · UTF-8 多字节 · 切分点可能落在字符中间", + marker_type="streaming_split", + notes="CJK · UTF-8 多字节 · 切分点可能落在字符中间 · 缓冲重组", ), StripCase( name="stream_split_emoji_boundary", - raw="Done 🎉 [1] end", + raw="Done 🎉 [source: r] end", expected="Done 🎉 end", mode="streaming", - notes="emoji 字节边界 · sanitizer 需 utf-8 aware buffering", + marker_type="streaming_split", + notes="Emoji 字节边界 · UTF-8 aware buffering", ), ] -# ===== Tool result 含来源(transform_tool_result 挂点) ===== +# ============================================================================ +# Tool result 含来源 +# ============================================================================ + TOOL_RESULT_CASES: list[StripCase] = [ StripCase( name="tool_result_with_inline_source", - raw='{"answer": "X [1]", "sources": ["doc_a"]}', + raw='{"answer": "X [source: doc_a]", "sources": ["doc_a"]}', expected='{"answer": "X", "sources": ["doc_a"]}', mode="tool", - notes="JSON tool_result 内含 inline citation · 只 strip answer 里的 · 不改 sources 数组", + marker_type="tool_result_json", + notes="JSON tool_result 内 answer 含 inline source · strip answer · 保留 sources 数组", ), ] -# ===== History replay / session.resume 转录 ===== +# ============================================================================ +# History replay(session.list/resume 转录 · presentation boundary) +# ============================================================================ + HISTORY_REPLAY_CASES: list[StripCase] = [ StripCase( - name="history_assistant_msg_with_citation", - raw="Previous answer: The result is 42 [1]. [2] concurs.", - expected="Previous answer: The result is 42. concurs.", + name="history_assistant_msg_source", + raw="Previous: The result is 42 [source: analysis]. [来源: report] concurs.", + expected="Previous: The result is 42. concurs.", mode="history", - notes="从 session.resume 拉出的历史 assistant message · 走 presentation sanitizer · raw canonical 不改", + marker_type="history_replay", + notes="History replay 出口 · presentation sanitize · raw canonical 不改", ), ] -# ===== 汇总(供 pytest parametrize) ===== +# ============================================================================ +# 汇总(供 pytest parametrize) +# ============================================================================ + ALL_CASES: list[StripCase] = ( - NORMAL_FINAL + INLINE_SOURCE_CASES + + LINE_SOURCE_CASES + + NEGATIVE_CASES + + UNCLOSED_CASES + UNICODE_CASES - + INTERRUPT_PARTIAL - + STREAMING_BYTE_SPLIT + + STREAMING_BYTE_SPLIT_CASES + TOOL_RESULT_CASES + HISTORY_REPLAY_CASES ) +VALID_MARKER_TYPES: frozenset[str] = frozenset({ + "source_inline", + "source_line", + "negative", + "unclosed", + "unicode", + "streaming_split", + "tool_result_json", + "history_replay", +}) + +VALID_MODES: frozenset[str] = frozenset({ + "final", "history", "streaming", "interim", "tool", +}) + + def cases_by_mode(mode: str) -> list[StripCase]: """按出口模式筛选 cases · 供各 hook prototype 测试消费。""" return [c for c in ALL_CASES if c.mode == mode] +def cases_by_marker(marker_type: str) -> list[StripCase]: + """按 marker 分类筛选 · 供分层断言。""" + return [c for c in ALL_CASES if c.marker_type == marker_type] + + def enumerate_byte_split_chunks(raw: str) -> list[tuple[bytes, bytes]]: """将 raw 转 UTF-8 后 · 按每 1..N-1 字节切分点返回 (head, tail) 对。 专供 streaming S1 缓冲策略测试:每一切分点送 buffered sanitizer · 最终 emit 必须等于 sanitize(raw)。切在多字节字符中间(如 UTF-8 continuation - byte)· sanitizer 必须**等待更多字节**,不得强解码为 replacement char。 + byte)· sanitizer **必须等待更多字节** · 不得强解码为 replacement char。 """ encoded = raw.encode("utf-8") n = len(encoded) diff --git a/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py new file mode 100644 index 0000000000000..cfdc5d840df9e --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py @@ -0,0 +1,98 @@ +"""P3-M0 Spike · shared test helper · fresh PluginManager + tmp HERMES_HOME. + +**Codex §VIII 契约**:所有 3 spike 插件测试必须走真实 `PluginManager.discover_and_load` ++ 临时 `config.yaml` 启用 · `has_middleware("llm_execution") == True` ++ 通过 `run_llm_execution_middleware` / `invoke_hook` 触发 · **不用** `SPIKE_*` env ++ **不污染**全局 manager(fixture teardown 严格还原)。 + +**为什么用 fresh PluginManager 而非复用全局**: +- 全局 `_plugin_manager` 单例 · 一次 discover 后所有插件都 cached +- 测试需要:临时 config 启用 → discover → 验证 → teardown 还原 +- 用 fresh instance + monkeypatch global · 严格隔离 · zero pollution +""" +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +import yaml + + +def write_config(hermes_home: Path, enabled: List[str], **plugin_cfg: Any) -> None: + """Write ``config.yaml`` under ``hermes_home`` enabling the given plugins. + + ``plugin_cfg`` keyword args become entries under ``plugins.``. Example: + write_config(tmp, ["spike-p3-m0-cache"], spike_p3_m0_cache={"enabled": True, ...}) + """ + hermes_home.mkdir(parents=True, exist_ok=True) + cfg: Dict[str, Any] = { + "plugins": { + "enabled": list(enabled), + **plugin_cfg, + }, + } + (hermes_home / "config.yaml").write_text( + yaml.safe_dump(cfg, sort_keys=True, allow_unicode=True), encoding="utf-8" + ) + + +def install_fresh_manager(monkeypatch, hermes_home: Path): + """Replace the global PluginManager with a fresh instance for this test. + + Steps: + 1. Set ``HERMES_HOME`` env → temp dir. + 2. Force ``hermes_cli.config._load_config_impl`` cache clear. + 3. Create fresh :class:`hermes_cli.plugins.PluginManager`. + 4. Monkeypatch ``hermes_cli.plugins._plugin_manager`` → fresh instance. + 5. Call ``discover_and_load(force=True)``. + + Fixture teardown (monkeypatch scope) restores env + module attr. + Returns the fresh manager. + """ + from hermes_cli import plugins as plugins_mod + + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + # 强制 config cache 失效 · load_config 内部 mtime_ns 键 · 换 HERMES_HOME 自然 miss + try: + from hermes_cli import config as config_mod + + if hasattr(config_mod, "_CONFIG_CACHE"): + monkeypatch.setattr(config_mod, "_CONFIG_CACHE", {}, raising=False) + except Exception: + pass + + fresh = plugins_mod.PluginManager() + monkeypatch.setattr(plugins_mod, "_plugin_manager", fresh) + + # 卸载 hermes_plugins.* 命名空间(避免上次 discover 的旧 module 状态污染) + for name in [k for k in list(sys.modules) if k.startswith("hermes_plugins")]: + monkeypatch.delitem(sys.modules, name, raising=False) + + fresh.discover_and_load(force=True) + return fresh + + +def get_discovered_module(manager, plugin_key: str): + """Return the imported module object of a plugin by its plugin.yaml `name`.""" + loaded = manager._plugins.get(plugin_key) + if loaded is None: + raise KeyError(f"plugin {plugin_key!r} not discovered · loaded keys: {list(manager._plugins)}") + if loaded.module is None: + raise RuntimeError(f"plugin {plugin_key!r} discovered but module is None · error={loaded.error!r}") + return loaded.module + + +def make_run_llm_execution(**default_ctx): + """Return a helper that invokes `run_llm_execution_middleware` with default ctx. + + Simplifies tests: ``run = make_run_llm_execution(api_request_id="r1", ...)`` + then ``run(request, next_call, provider="p")``. + """ + from hermes_cli.middleware import run_llm_execution_middleware + + def _run(request, next_call, **override_ctx): + ctx = {**default_ctx, **override_ctx} + return run_llm_execution_middleware(request, next_call, **ctx) + + return _run diff --git a/plugins/spike_p3_m0_strip/__init__.py b/plugins/spike_p3_m0_strip/__init__.py index 1fc958fa73a1e..d2630ffdd8464 100644 --- a/plugins/spike_p3_m0_strip/__init__.py +++ b/plugins/spike_p3_m0_strip/__init__.py @@ -1,27 +1,31 @@ """P3-M0 Spike · strip-sources prototype(exit ① 普通最终响应). -**范围严守**(见 `../spike-p3-m0-fixtures/README.md` 与 Hermes_AI 侧 -`docs/fork/strip-contract.md`): - -- 仅覆盖出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` -- streaming/interim/interrupt/history-replay/tool-result 各出口 fixture 冻结 - · **本 spike 不实装** -- `_sanitize_surrogates`(fresh baseline 已在 turn_finalizer 内)独立于本 hook - · 顺序上 sanitize 后于 transform_llm_output · 二者协同不干扰 -- raw canonical transcript 完全不改(finalizer 已在 :317/:333 写入原文) -- 无第三方 plugin 冲突:hook 返回 non-None str 首个胜出 · 本 plugin 无 None 短路 - -**不是** C4 单点方案(只覆盖 5 出口之 1)。M0 首轮证据 + 契约冻结 · 不称 C4 兑现。 +**Codex §VI fail-closed 契约**: +- sanitizer 异常 → 返回**安全占位** · 记录 audit · **不**返回 None(否则原文透出) +- 无改动 · 返回 None(标准 hook "first non-None str wins" 契约) + +**范围**: +- 仅出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` +- streaming/interim/interrupt/history-replay/tool-result 五出口 fixture 冻结 + · **本 spike 不实装**(fixture present · implementation pending) +- `_sanitize_surrogates`(fresh baseline turn_finalizer 已加)独立 · 不冲突 + +**插件启用**:通过 fork `config.yaml` 的 `plugins.enabled` 列表 · **不**用 env。 """ from __future__ import annotations import logging from typing import Any, Optional -from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-open 测试 +from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-closed 测试 logger = logging.getLogger(__name__) +# fail-closed 占位 · 出现 sanitizer 崩时返回 · 提示用户重试 +FAIL_CLOSED_PLACEHOLDER = ( + "[response filtered · sanitizer error · please retry]" +) + def on_transform_llm_output( *, @@ -31,23 +35,49 @@ def on_transform_llm_output( platform: str = "", **_kwargs: Any, ) -> Optional[str]: - """Return stripped text · None if unchanged. + """`transform_llm_output` hook · fail-closed on sanitizer exception. + + Contract(`hermes_cli/plugins.py::VALID_HOOKS`): + - 返回 non-empty str → 覆盖 `final_response`(first-wins 跨 plugins) + - 返回 None / "" → 保留 `final_response` 不变 - Hook contract(`hermes_cli/plugins.py::VALID_HOOKS`): - - First non-empty str return wins across all plugins consuming this hook. - - None / empty str leaves `final_response` unchanged. - - Fired at `agent/turn_finalizer.py:561`. + Codex §VI · fail-closed 补丁: + - 无异常 · 无改动 → 返回 None + - 无异常 · 有改动 → 返回 stripped str + - 异常 → 返回 `FAIL_CLOSED_PLACEHOLDER`(**不返回原文**)· 记录 audit """ + text = response_text or "" try: - stripped = _sanitize_mod.sanitize_presentation(response_text or "", mode="final") - except Exception as exc: # noqa: BLE001 — fail-open · never break chat - logger.warning("spike-p3-m0-strip on_transform_llm_output failed: %s", exc) - return None - if stripped == (response_text or ""): - return None # 无改动 · 保留 + stripped = _sanitize_mod.sanitize_presentation(text, mode="final") + except Exception as exc: # noqa: BLE001 — fail-closed · 拒原文泄漏 + logger.error( + "spike-p3-m0-strip: sanitizer FAILED · fail-closed (session_id=%s model=%s platform=%s): %s", + session_id, model, platform, exc, + ) + # audit event · 便于监控告警 + _emit_audit( + event="strip.sanitizer_failed", + session_id=session_id, + model=model, + platform=platform, + error_class=type(exc).__name__, + ) + return FAIL_CLOSED_PLACEHOLDER + if stripped == text: + return None # 无改动 return stripped +def _emit_audit(**payload: Any) -> None: + """轻量 audit hook · 通过 lifecycle invoke_hook 触发观察者(如 langfuse/nemo_relay)。""" + try: + from hermes_cli.lifecycle import invoke_hook + + invoke_hook("api_request_error", **payload) # 复用现有 API error 通道 + except Exception: # noqa: BLE001 — audit never breaks hook + pass + + def register(ctx: Any) -> None: """PluginContext hook · called at plugin discovery.""" ctx.register_hook("transform_llm_output", on_transform_llm_output) diff --git a/plugins/spike_p3_m0_strip/plugin.yaml b/plugins/spike_p3_m0_strip/plugin.yaml index 93c3ba1bb50ef..740bf16ed47fa 100644 --- a/plugins/spike_p3_m0_strip/plugin.yaml +++ b/plugins/spike_p3_m0_strip/plugin.yaml @@ -1,6 +1,6 @@ name: spike-p3-m0-strip version: "0.0.1" -description: "P3-M0 Spike · strip-sources prototype · transform_llm_output hook · covers exit ① (normal final response) only. Streaming ② / interim ③ / tool ⑤ / history-replay ④ are fixture-frozen only, not implemented. WIP · do not enable in production." +description: "P3-M0 Spike · strip-sources prototype · transform_llm_output hook · fail-closed on sanitizer exception (returns safe placeholder, NOT original text). Covers exit ① normal final response only. Streaming ② / interim ③ / history-replay ④ / tool-result ⑤ exits are fixture-frozen (pending). Configured via plugins.enabled in config.yaml (no env vars). WIP · do not enable in production." author: "Hermes_AI × NousResearch/hermes-agent Spike" -hooks: +provides_hooks: - transform_llm_output diff --git a/plugins/spike_p3_m0_strip/sanitize.py b/plugins/spike_p3_m0_strip/sanitize.py index ac23efeaf5f2f..848549b28f2e4 100644 --- a/plugins/spike_p3_m0_strip/sanitize.py +++ b/plugins/spike_p3_m0_strip/sanitize.py @@ -1,43 +1,55 @@ -"""Presentation-boundary sanitizer(spike 骨架 · 只实装 mode="final")。 - -**冻结契约**(见 Hermes_AI 侧 `docs/fork/strip-contract.md` §6): - -``` -sanitize_presentation(text: str, *, mode: str) -> str - mode: - - "final" : 出口 ① `transform_llm_output`(**M0 首轮实装**) - - "history" : 出口 ④ session.resume/list(fixture 冻结 · 不实装) - - "streaming" : 出口 ② 缓冲后 sanitize(fixture 冻结 · 不实装) - - "interim" : 出口 ③ interim_assistant_callback wrap(fixture 冻结) - - "tool" : 出口 ⑤ transform_tool_result(fixture 冻结) -``` - -**策略**(见 strip-contract §4): -- streaming 默认完整缓冲后 sanitize(牺牲实时性 · 保零泄漏)· 有状态 parser 是 - M2+ 优化项 · 必须通过每字节切分点 + Unicode 边界 + interrupt 未闭合测试 -- final mode 只针对完整字符串 · 无跨 chunk 复杂性 +"""Presentation-boundary sanitizer(spike · mode="final" 实装 · 其他 skeleton). + +**向后兼容基线**:Hermes_AI `hermes_devices/ops/audit.py::strip_sources` +(charter C4 · fixture SHA-256 双仓锁死) + +**Codex §VI 契约**: +- 只 strip 明确的 `[source|来源: ...]` / `【来源:...】` / 独立行 `source:` `来源:` +- 允许一层嵌套中文括号(与 Hermes_AI 一致) +- **不得**无条件 strip 裸 `[N]`(需结构化 sources metadata 配对 · 本 M0 spike 不做) +- 未闭合标记不吞下一行 +- **fail-closed** on sanitizer exception:返回安全占位 · **不**返回原文 + +**冻结 · pending 出口**: +- mode="streaming":skeleton pass-through(pending · S1 完整缓冲 · M1+) +- mode="history":skeleton pass-through(pending · presentation boundary · M1+) +- mode="interim":skeleton pass-through(pending · M1+) +- mode="tool":skeleton pass-through(pending · M1+) """ from __future__ import annotations import re -# ── 3 种常见 citation 形态(见 fixture citation_cases.py NORMAL_FINAL) ── - -# 1. inline `[N]` `[N][M]` 形态 · 移除包括前置单空格 -_INLINE_CITATION_RE = re.compile(r" ?\[\d+\](?:\[\d+\])*") +# ── 向后兼容 Hermes_AI ops/audit.py::_INLINE_RE ────────────────────────── +# 匹配 [source: ...] / [来源: ...] / 【来源:...】 / (source: ...) 等 · +# 允许一层嵌套中文/半角括号(如 [来源: FAQ(2026版)]). +_INLINE_SOURCE_RE = re.compile( + r"\s?[\[【（(]\s*(?:source|来源)\s*[:：]" + r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】）)]", + re.IGNORECASE, +) -# 2. fenced `...` 块 · 整块移除(含前后空行) -_FENCED_SOURCES_RE = re.compile( - r"\n{0,2}[\s\S]*?\n?", - re.MULTILINE, +# ── 独立行 `source:` / `来源:` · 整行(含换行)移除 ────────────────────── +_LINE_SOURCE_RE = re.compile( + r"^[ \t]*(?:source|来源)\s*[:：].*$", + re.IGNORECASE | re.MULTILINE, ) +# ── Trailing whitespace / 连续空行清理(sub 完的后处理)────────────── +_TRAILING_WS_RE = re.compile(r"[ \t]+(?=\n|$)") +_MULTI_NEWLINE_RE = re.compile(r"\n{3,}") + + +class SanitizerError(RuntimeError): + """Sanitizer 内部错误 · 触发 fail-closed(见 hook `__init__.py`)。""" + def sanitize_presentation(text: str, *, mode: str) -> str: - """Strip citation / source markers · 按 mode 返回 sanitized text. + """Strip source markers · 按 mode 返回 sanitized text. - M0 首轮:仅实装 mode="final"。其他 mode 返回原字符串(不改)· 契约冻结 · 待 - M1+ 分阶段实装。 + M0 首轮:仅实装 mode="final"。其他 mode 返回原字符串(skeleton pass-through · + fixture present · implementation pending)。 """ if mode != "final": return text @@ -45,11 +57,13 @@ def sanitize_presentation(text: str, *, mode: str) -> str: def _sanitize_final(text: str) -> str: - """出口 ① · 普通最终响应 sanitize。""" + """出口 ① · 普通最终响应 sanitize · Hermes_AI ops/audit.py 兼容。""" if not text: return text - # 先移 fenced · 避免 fenced 内 `[N]` 被 inline 规则误命中(应整块删) - stripped = _FENCED_SOURCES_RE.sub("", text) - stripped = _INLINE_CITATION_RE.sub("", stripped) - # 收尾 trailing whitespace(移 citation 后可能留) - return stripped.rstrip() + out = _INLINE_SOURCE_RE.sub("", text) + out = _LINE_SOURCE_RE.sub("", out) + # 清理 sub 留下的行尾空格 · 3+ 连续空行合并为 2 + out = _TRAILING_WS_RE.sub("", out) + out = _MULTI_NEWLINE_RE.sub("\n\n", out) + # 清理结果末尾空行(与 Hermes_AI baseline 一致) + return out.rstrip() diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_final.py b/plugins/spike_p3_m0_strip/tests/test_strip_final.py index eb0963acb197b..11feb80376785 100644 --- a/plugins/spike_p3_m0_strip/tests/test_strip_final.py +++ b/plugins/spike_p3_m0_strip/tests/test_strip_final.py @@ -1,10 +1,15 @@ -"""P3-M0 Spike · strip prototype 出口 ① 测试。 - -**Fixture 双仓共享**: -- Hermes_AI: `tests/fixtures/strip_sources/citation_cases.py` -- fork: `plugins/spike_p3_m0_fixtures/citation_cases.py` -- SHA-256: `0cd6ac14035292103ecf574aa6434cf3fcc4d0bdcbe9f21f32a0ba7e9ae26212` -- 跨仓校验:PR body 互链字段 · 任一侧修改双仓 PR 必须同步更新 hash +"""P3-M0 Spike · strip prototype 出口 ① 测试(**真实 PluginManager · invoke_hook**). + +**Codex §VI / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 真实 `PluginManager.discover_and_load` +- 通过 `hermes_cli.plugins.invoke_hook("transform_llm_output", ...)` 触发 +- 覆盖: + * fixture SHA-256 双仓一致(cross-repo lock) + * final-mode 全 fixture case parametrize · sanitize_presentation 直调 + * 有改动 hook 返回 stripped str · 无改动返回 None + * sanitizer 异常 → hook 返回 FAIL_CLOSED_PLACEHOLDER(**fail-closed** · 不返回原文) + * negative case:bare [N] / JSON / footnote ref 等**必须**原样保留 + * streaming byte-split · UTF-8 缓冲 · sanitize 结果与整体一致 """ from __future__ import annotations @@ -16,36 +21,72 @@ from plugins.spike_p3_m0_fixtures.citation_cases import ( ALL_CASES, StripCase, + cases_by_marker, cases_by_mode, + enumerate_byte_split_chunks, +) +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + write_config, ) -from plugins.spike_p3_m0_strip import on_transform_llm_output -from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + +STRIP_KEY = "spike-p3-m0-strip" _HERE = Path(__file__).parent _FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" +# 与 Hermes_AI `tests/test_p3_m0_fixture_gate.py::EXPECTED_SHA256` 完全一致 +EXPECTED_FIXTURE_SHA256 = ( + "47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666" +) -# ── Fixture SHA-256 校验(跨仓一致性 门槛)── -EXPECTED_FIXTURE_SHA256 = "0cd6ac14035292103ecf574aa6434cf3fcc4d0bdcbe9f21f32a0ba7e9ae26212" +# ── Fixture SHA-256 校验(跨仓一致性 门槛) ───────────────────── def test_fixture_sha256_matches_hermes_ai(): - """跨仓 fixture 内容一致性 · SHA-256 必须匹配。""" fixture_path = _FIXTURE_DIR / "citation_cases.py" actual = hashlib.sha256(fixture_path.read_bytes()).hexdigest() assert actual == EXPECTED_FIXTURE_SHA256, ( f"fixture SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " - f"actual: {actual}\n跨仓 fixture 不一致 · 需同步" + f"actual: {actual}\n跨仓 fixture 不一致 · 需双仓同步更新" ) -# ── 出口 ① 普通最终响应 sanitize(M0 首轮 spike 覆盖)── +# ── Discovery / registration(§VIII) ──────────────────────────── + + +def _enable_strip(tmp_path, monkeypatch): + write_config(tmp_path, [STRIP_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, STRIP_KEY) + return manager, mod + + +def test_discover_registers_transform_llm_output_hook(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + manager, mod = _enable_strip(tmp_path, monkeypatch) + assert has_hook("transform_llm_output") is True + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_hook("transform_llm_output") is False + + +# ── final-mode sanitize_presentation 全 fixture parametrize ──── @pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) def test_sanitize_final_mode(case: StripCase): - """全 final-mode fixture case · sanitize_presentation(mode='final') 正确。""" + """全 final mode fixture case · sanitize_presentation(mode='final') 正确。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + got = sanitize_presentation(case.raw, mode="final") assert got == case.expected, ( f"case={case.name}\n raw: {case.raw!r}\n " @@ -53,53 +94,117 @@ def test_sanitize_final_mode(case: StripCase): ) +# ── Negative case:bare [N] / JSON / footnote 必须原样保留(§VI 硬约束) ── + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_preserved(case: StripCase): + """negative case:sanitize 不得吞掉 · 原样返回。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + got = sanitize_presentation(case.raw, mode="final") + assert got == case.raw, ( + f"NEGATIVE case={case.name} 被误 strip · raw={case.raw!r} got={got!r}" + ) + + +# ── Hook 契约(通过 invoke_hook · 真实 PluginManager 走位) ──────── + + @pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) -def test_transform_llm_output_hook(case: StripCase): - """`on_transform_llm_output` hook 契约: - - 有改动 · 返回 stripped str - - 无改动 · 返回 None(第一个 non-None str 胜出)""" - result = on_transform_llm_output(response_text=case.raw) +def test_transform_llm_output_hook_via_invoke_hook(tmp_path, monkeypatch, case: StripCase): + from hermes_cli.plugins import invoke_hook + + _enable_strip(tmp_path, monkeypatch) + results = invoke_hook( + "transform_llm_output", + response_text=case.raw, + session_id="s_test", + model="m", + platform="cli", + ) if case.raw == case.expected: - assert result is None, f"unchanged input should return None · case={case.name}" + assert results == [], f"unchanged input · hook 应返回 None(过滤后空 list)· case={case.name}" else: - assert result == case.expected, f"case={case.name}: got {result!r}" + assert results == [case.expected], ( + f"case={case.name} · expected [{case.expected!r}] · got {results!r}" + ) + + +def test_hook_empty_input_returns_none(tmp_path, monkeypatch): + from hermes_cli.plugins import invoke_hook + _enable_strip(tmp_path, monkeypatch) + assert invoke_hook("transform_llm_output", response_text="") == [] -def test_hook_none_returns_none(): - """空输入 · 返回 None(与"不改动"契约一致)· 不 crash。""" - assert on_transform_llm_output(response_text="") is None - assert on_transform_llm_output(response_text=None) is None # type: ignore[arg-type] +# ── fail-CLOSED on sanitizer exception(§VI 硬约束) ───────────── -def test_hook_fail_open_on_exception(monkeypatch): - """sanitizer 崩 · hook 返回 None(fail-open · 不阻断 chat)。""" - from plugins.spike_p3_m0_strip import sanitize as sanitize_mod - def boom(*args, **kwargs): +def test_hook_fail_closed_on_sanitizer_exception(tmp_path, monkeypatch): + """sanitizer 崩 · hook 返回 FAIL_CLOSED_PLACEHOLDER · **不**返回原文。 + + Codex §VI:sanitizer 失败必须拒原文透出 · 返回安全占位符 · 记录 audit。 + """ + from hermes_cli.plugins import invoke_hook + + _, strip_mod = _enable_strip(tmp_path, monkeypatch) + from plugins.spike_p3_m0_fixtures.pluginmgr_helper import get_discovered_module + # 从 discover 后的 module 修改 sanitize 依赖(fresh manager 下的 module) + sanitize_dep = strip_mod._sanitize_mod + + def boom(*_args, **_kwargs): raise RuntimeError("simulated sanitizer crash") - monkeypatch.setattr(sanitize_mod, "sanitize_presentation", boom) - # hook 内 try/except 保护 · 返回 None(final_response 未被覆盖) - result = on_transform_llm_output(response_text="Answer [1].") - assert result is None + monkeypatch.setattr(sanitize_dep, "sanitize_presentation", boom) + + dangerous_text = "SECRET INTERNAL DATA · SHOULD NOT LEAK · [source: internal]" + results = invoke_hook( + "transform_llm_output", + response_text=dangerous_text, + session_id="s_test", + model="m", + platform="cli", + ) + # 关键断言:不返回原文 · 返回占位符 + assert len(results) == 1 + assert results[0] == strip_mod.FAIL_CLOSED_PLACEHOLDER + assert dangerous_text not in results[0], "fail-closed 泄漏原文 · 违 §VI 硬约束" -# ── 其他 mode 冻结契约(M0 首轮不实装 · 只 skeleton)── +# ── 其他 mode 契约冻结(skeleton pass-through · M1+ 实装) ────── @pytest.mark.parametrize("mode", ["history", "streaming", "interim", "tool"]) -def test_other_modes_skeleton_pass_through(mode: str): - """非 final mode · 当前 spike skeleton 原样返回(契约冻结 · M1+ 分阶段实装)。""" - text = "Answer [1]. Reference doc_abc." +def test_non_final_modes_skeleton_pass_through(mode: str): + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + text = "Answer with citation [source: kb_2024]. More text." got = sanitize_presentation(text, mode=mode) - assert got == text, f"mode={mode} should pass-through in M0 spike" + assert got == text, f"mode={mode} 应 pass-through(M0 skeleton · 见 §VI 冻结)" -# ── 冻结契约 · 出口 ②/③/④/⑤ 待 M1+(fixture 已覆盖 · 实装待议) ── +# ── Streaming byte-split · S1 完整缓冲后 sanitize 与整体一致 ─── -@pytest.mark.parametrize("mode", ["streaming", "interim", "history", "tool"]) -def test_fixture_frozen_for_non_final_modes(mode: str): - """fixture 有非 final mode case · 供后续实装参照。""" - cases = cases_by_mode(mode) - assert len(cases) >= 1, f"fixture 应含 {mode} mode case" +@pytest.mark.parametrize("case", cases_by_marker("streaming_split"), ids=lambda c: c.name) +def test_streaming_full_buffer_matches_final(case: StripCase): + """S1 策略验证:任意字节切分点分片 → 缓冲拼回 → UTF-8 decode → final sanitize + → 结果必须等于对整体 raw 直接 final sanitize 的结果(与 case.expected 一致)。 + """ + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + expected = sanitize_presentation(case.raw, mode="final") + assert expected == case.expected, ( + f"streaming_split case={case.name} · 直接 final sanitize 与 fixture expected 不一致 · " + f"expected={case.expected!r} got={expected!r}" + ) + # 每个切分点:两段拼接 · decode · sanitize · 与 expected 一致 + for head, tail in enumerate_byte_split_chunks(case.raw): + buf = head + tail + text = buf.decode("utf-8") + got = sanitize_presentation(text, mode="final") + assert got == expected, ( + f"streaming byte-split 破坏 sanitize · case={case.name} " + f"head_len={len(head)} · got={got!r} expected={expected!r}" + ) From a600bf37a593da363d259584dc816446b5a5fa67 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 19:32:38 +0000 Subject: [PATCH 03/19] =?UTF-8?q?spike(p3-m0):=20third-round=20Codex=20REQ?= =?UTF-8?q?UEST=20CHANGES=20=C2=B7=20decouple=20cache+budget=20=C2=B7=20fa?= =?UTF-8?q?il-CLOSED=20=C2=B7=20Windows=20CRLF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second Codex re-review on fork PR #1 · all 7 blockers addressed: §I Windows CRLF fixture SHA — .gitattributes now pins the shared strip fixture to LF; SHA gate reads canonical LF bytes so Windows autocrlf checkouts pass. Adds test_fixture_sha256_crlf_input_normalizes to assert the CRLF → LF normalization keeps the frozen SHA (47155c24…) intact. §II cache + budget control-flow decoupled — cache eligibility no longer short-circuits budget enforcement. New order in cache_and_budget_middleware: disabled → next_call budget module missing → fail-CLOSED (never call provider) unsupported api_mode → fail-CLOSED (M0 narrows to chat_completions) non-dict request → fail-CLOSED tenant context invalid → fail-CLOSED (next_call == 0) budget config invalid → fail-CLOSED (no silent None) missing api_request_id → fail-CLOSED cacheable hit → return cached, no reserve cacheable miss OR non-cacheable → reserve → next_call → success settle / exception release All fail-CLOSED paths return a non-empty assistant message with finish_reason=content_filter (Codex §V: safe, displayable, no retry). §III Strict config typing + plugin dep gate — _is_enabled is now `v is True` (rejects "false", 1, [], {}, YAML-string coercion). BudgetConfigError never becomes None; orchestrator fail-CLOSES. Removed the loose sys.modules fallback in _budget_module so cache cannot silently satisfy the dependency when the budget plugin is disabled — new test_cache_alone_without_budget_fail_closed proves it. §IV Budget ledger strict idempotency — new BudgetSettleError raised on settle without a live reservation (unknown / double / post-release); the old "settle unknown accumulates used" contract was a laundering hole and is removed. release() returns bool for idempotency. New tests: - test_concurrent_reserve_precise_success_and_exceeded_counts: 40 threads at barrier, budget = 15 × per-req → exactly 15 succeed, 25 raise BudgetExceeded, no overspend - test_utc_day_rollover_isolates_bucket: monkeypatch _today_utc to simulate Day2 checkout; yesterday's used cannot poison today. §V Production adjacency — _estimate_needed_tokens no longer silently returns 0. It prefers ctx.approx_input_tokens (>0), else estimates from messages content chars / 4 (documented M0 approximation, with an M1+ pending contract for a real tokenizer). New tests prove: streaming + over-budget → next_call == 0 tools + over-budget → next_call == 0 non-cacheable req → still reserves+settles provider exception → releases reservation, re-raises char-count fallback → catches unbudgeted large prompts §VI Strip differential + hook order risk + sys.modules snapshot — New plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py vendors Hermes_AI hermes_devices/ops/audit.py::strip_sources verbatim with a byte-check test locking the pattern strings. New tests/test_strip_differential_vs_hermes_ai.py (27 tests) locks the two intentional UX divergences (\s? leading, .rstrip() vs .strip()) and proves coverage inclusion + negative parity. pluginmgr_helper adds snapshot_plugin_modules / addfinalizer cleanup; new leak test compares names + object ids after teardown. sanitize.py docstring pins the UX diffs explicitly. §VII delivered via scripts/run_tests.sh; full CI-faithful regression run reported in PR body. Test results (scripts/run_tests.sh, 8 workers, deterministic env): - test_strip_differential_vs_hermes_ai.py — 27 passed, 0.5s - test_strip_final.py — 65 passed, 5.3s - test_budget_reserve.py — 28 passed, 5.3s - test_cache_middleware.py — 35 passed, 6.4s Total: 155 passed / 0 failed in 6.4s Non-plugin footprint: only .gitattributes gained one line for the LF rule on the shared fixture. All other changes stay inside opt-in plugin directories. Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01FiKVo54oX4Uv2qksAHjYsZ --- .gitattributes | 4 + plugins/spike_p3_m0_budget/__init__.py | 40 +- .../tests/test_budget_reserve.py | 174 +++++++-- plugins/spike_p3_m0_cache/__init__.py | 337 ++++++++++++----- .../tests/test_cache_middleware.py | 349 ++++++++++++++---- .../hermes_ai_baseline.py | 58 +++ .../spike_p3_m0_fixtures/pluginmgr_helper.py | 46 ++- plugins/spike_p3_m0_strip/sanitize.py | 32 +- .../test_strip_differential_vs_hermes_ai.py | 153 ++++++++ .../tests/test_strip_final.py | 82 +++- 10 files changed, 1055 insertions(+), 220 deletions(-) create mode 100644 plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py create mode 100644 plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py diff --git a/.gitattributes b/.gitattributes index 553e3cd21b3cf..6cedf3280c476 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,7 @@ web/package-lock.json linguist-generated=true Dockerfile text eol=lf *.dockerfile text eol=lf docker/entrypoint.sh text eol=lf + +# P3-M0 shared strip fixture — SHA-256 is a cross-repo lock; must stay LF-normalized +# on every checkout so the double-repo fixture SHA gate matches on Windows too. +plugins/spike_p3_m0_fixtures/citation_cases.py text eol=lf diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py index 5dfc67ea01421..c83ae2d7f8ef7 100644 --- a/plugins/spike_p3_m0_budget/__init__.py +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -36,7 +36,15 @@ class BudgetExceeded(RuntimeError): class BudgetConfigError(ValueError): - """budget 配置字段类型/值非法 · 视 as fail-open(不 enforce · 记录 audit)。""" + """budget 配置字段类型/值非法 · 由 orchestrator 硬 fail-CLOSED(**不**再静默转 None)。""" + + +class BudgetSettleError(RuntimeError): + """settle/release 在无 reservation 上被调用 · **不改** used(严格幂等)。 + + Codex 第三轮 §IV:settle 必须仅结算仍存在的 reservation;unknown/double settle + 不得增加 used。抛此异常供 orchestrator 记 audit 但不阻断。 + """ # ── Module state · 单进程 · 非 persistent ───────────────────────────── @@ -119,20 +127,40 @@ def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: - """结算实际用量 · 移除 reservation · 累加 used。""" + """结算实际用量 · **仅结算仍存在的 reservation** · 累加 used。 + + Codex 第三轮 §IV(**严格幂等**): + - 若 reservation 存在 · 移除并累加 used + - 若 reservation 不存在(未 reserve / double settle / release 后 settle): + **不改** used · raise BudgetSettleError · 由 orchestrator 记 audit + """ if actual_tokens < 0: actual_tokens = 0 with _lock: b = _get_or_create_bucket(tenant_id) - b["reservations"].pop(api_request_id, None) + if api_request_id not in b["reservations"]: + raise BudgetSettleError( + f"settle: no reservation for api_request_id={api_request_id!r} · " + "double settle or missing reserve" + ) + b["reservations"].pop(api_request_id) b["used"] = b["used"] + actual_tokens -def release(tenant_id: str, api_request_id: str) -> None: - """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。""" +def release(tenant_id: str, api_request_id: str) -> bool: + """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。 + + Codex 第三轮 §IV(**幂等**): + - 若 reservation 存在 · 移除 · 返回 True + - 若 reservation 不存在 · **no-op** · 返回 False(便于 orchestrator 检测 double release) + - 任何情况下 · used 不变 + """ with _lock: b = _get_or_create_bucket(tenant_id) - b["reservations"].pop(api_request_id, None) + if api_request_id in b["reservations"]: + b["reservations"].pop(api_request_id) + return True + return False # ── 观察 helpers · 测试用 ───────────────────────────────────────────── diff --git a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py index 23a392dc857b1..7182d9789645e 100644 --- a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py +++ b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py @@ -1,10 +1,11 @@ -"""P3-M0 Spike · budget service 测试(观察 hook · 无 enforcement). - -**Codex §I 契约调整**: -- Budget 不再由 `pre_api_request` hook 抛 BudgetExceeded(生产调用点吞异常) -- 现由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 调用 budget service -- 本文件测 service 函数 · orchestration 端到端由 `spike_p3_m0_cache` 测试覆盖 -- Discovery 侧仍通过真实 PluginManager 验证:hook 挂上、observer 只做 audit +"""P3-M0 Spike · budget service 测试(observer hook · service API). + +**Codex 第三轮 §IV 契约(修订)**: +- settle 仅结算**仍存在**的 reservation · unknown/double settle → raise BudgetSettleError · **不改** used +- release 幂等 · 返回 True(存在)/ False(不存在)· 不改 used +- 40 线程 barrier 精确超限测试:budget = N * per-req · 超过 N 个必 raise +- UTC 跨日 rollover 真测试(monkeypatch `_today_utc`) +- observer hooks 恒 no-op(无 enforcement) """ from __future__ import annotations @@ -34,14 +35,11 @@ def _enable_budget(tmp_path, monkeypatch): def test_discovery_registers_observer_hooks(tmp_path, monkeypatch): - """PluginContext.register_hook 生效 · pre/post_api_request 均挂上。""" from hermes_cli.plugins import has_hook manager, mod = _enable_budget(tmp_path, monkeypatch) - # observer 挂上(无 enforcement · 见模块 docstring) assert has_hook("pre_api_request") is True assert has_hook("post_api_request") is True - # 不 register llm_execution middleware(orchestration 是 cache 的责任) assert manager._middleware.get("llm_execution") in (None, []) @@ -50,7 +48,6 @@ def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): write_config(tmp_path, []) install_fresh_manager(monkeypatch, tmp_path) - # 无 hook 挂上 assert has_hook("pre_api_request") is False @@ -60,7 +57,7 @@ def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): def test_validate_daily_budget_states(tmp_path, monkeypatch): _, mod = _enable_budget(tmp_path, monkeypatch) assert mod.validate_daily_budget(None) is None - assert mod.validate_daily_budget(0) == 0 # 显式无限 + assert mod.validate_daily_budget(0) == 0 assert mod.validate_daily_budget(1000) == 1000 @@ -79,7 +76,6 @@ def test_validate_daily_budget_rejects(tmp_path, monkeypatch, bad_value): def test_reserve_none_budget_no_op(tmp_path, monkeypatch): - """budget=None(未启用)· reserve no-op · 不 raise。""" _, mod = _enable_budget(tmp_path, monkeypatch) mod.reserve("t1", "req_a", 500, budget=None) assert mod.get_pending_total("t1") == 0 @@ -95,14 +91,32 @@ def test_reserve_then_settle_charges_actual(tmp_path, monkeypatch): assert mod.get_pending_total("t1") == 0 -def test_release_removes_reservation_no_charge(tmp_path, monkeypatch): +def test_release_removes_reservation_returns_true(tmp_path, monkeypatch): + """release 存在的 reservation · 返回 True · used 不变。""" _, mod = _enable_budget(tmp_path, monkeypatch) mod.reserve("t1", "req_x", 200, budget=10_000) - mod.release("t1", "req_x") + assert mod.release("t1", "req_x") is True + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +def test_release_unknown_returns_false_no_op(tmp_path, monkeypatch): + """Codex §IV:release unknown reservation · 返回 False · no-op(幂等)。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.release("t1", "never_reserved") is False assert mod.get_used("t1") == 0 assert mod.get_pending_total("t1") == 0 +def test_release_after_release_returns_false(tmp_path, monkeypatch): + """double release · 第二次返回 False · 无副作用。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 100, budget=1000) + assert mod.release("t1", "req_x") is True + assert mod.release("t1", "req_x") is False + assert mod.get_used("t1") == 0 + + # ── reserve 超预算 · raise BudgetExceeded ──────────────────────── @@ -110,21 +124,19 @@ def test_reserve_over_budget_raises(tmp_path, monkeypatch): _, mod = _enable_budget(tmp_path, monkeypatch) with pytest.raises(mod.BudgetExceeded, match="daily budget exhausted"): mod.reserve("t1", "req_over", 200, budget=100) - # 不能 leak reservation assert mod.get_pending_total("t1") == 0 def test_used_plus_pending_plus_new_over_budget_raises(tmp_path, monkeypatch): _, mod = _enable_budget(tmp_path, monkeypatch) mod.reserve("t1", "r_a", 60, budget=200) - mod.settle("t1", "r_a", 60) # used=60 - mod.reserve("t1", "r_b", 100, budget=200) # pending=100 · used=60 · sum=160 + mod.settle("t1", "r_a", 60) + mod.reserve("t1", "r_b", 100, budget=200) with pytest.raises(mod.BudgetExceeded): - mod.reserve("t1", "r_c", 50, budget=200) # 60+100+50=210 > 200 + mod.reserve("t1", "r_c", 50, budget=200) def test_budget_zero_is_unlimited(tmp_path, monkeypatch): - """budget=0 显式无限 · 大额 reserve 仍成功 · used 仍累加供审计。""" _, mod = _enable_budget(tmp_path, monkeypatch) mod.reserve("t1", "r_big", 10_000_000, budget=0) mod.settle("t1", "r_big", 5_000_000) @@ -147,7 +159,42 @@ def test_empty_api_request_id_raises(tmp_path, monkeypatch): mod.reserve("t1", "", 50, budget=1000) -# ── UTC daily bucket 隔离 ────────────────────────────────────────── +# ── settle 严格幂等(Codex §IV 修订)───────────────────────────── + + +def test_settle_unknown_id_raises_and_does_not_charge(tmp_path, monkeypatch): + """Codex §IV 修订:settle 无 reservation · **raise BudgetSettleError · 不改 used**。 + + 旧契约"unknown settle 累加 used"是**错误**的 · 会导致 unknown id 洗钱。 + """ + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetSettleError, match="no reservation"): + mod.settle("t1", "unknown_req", 25) + assert mod.get_used("t1") == 0 + + +def test_double_settle_raises_after_first_ok(tmp_path, monkeypatch): + """double settle · 第二次 raise · used 不再加。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.settle("t1", "req_1", 50) # 首次 OK + assert mod.get_used("t1") == 50 + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 999) # 第二次 · reservation 已消 · raise + assert mod.get_used("t1") == 50 # 未增 + + +def test_settle_after_release_raises(tmp_path, monkeypatch): + """release 后 settle · reservation 不存在 · raise · used 不变。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.release("t1", "req_1") + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 25) + assert mod.get_used("t1") == 0 + + +# ── 多 tenant 隔离 ────────────────────────────────────────────────── def test_multiple_tenants_isolated(tmp_path, monkeypatch): @@ -160,7 +207,7 @@ def test_multiple_tenants_isolated(tmp_path, monkeypatch): assert mod.get_used("tB") == 90 -# ── 并发 reserve/settle · threading.Lock 正确 ───────────────────── +# ── 并发 reserve/settle · 无 race ──────────────────────────────── def test_concurrent_reserve_settle_thread_safe(tmp_path, monkeypatch): @@ -183,42 +230,95 @@ def worker(i): assert mod.get_pending_total("t_shared") == 0 -# ── settle/release 幂等 · 未知 req_id 不 raise ─────────────────── +# ── 真并发超限:Codex §IV 强要求 · 精确 N 成功 / rest 失败 ───────── + +def test_concurrent_reserve_precise_success_and_exceeded_counts(tmp_path, monkeypatch): + """40 线程 barrier reserve · budget 只允许 N 个成功 · 精确 N 成功 · 40-N 失败。 -def test_settle_unknown_id_still_accumulates_used(tmp_path, monkeypatch): - """settle 允许在没有 reservation 的 req_id 上调用 · 仅累加 used(observer 语义)。""" + Codex §IV:证明无超扣、无 race race-double-count。 + """ _, mod = _enable_budget(tmp_path, monkeypatch) - mod.settle("t1", "unknown_req", 25) - assert mod.get_used("t1") == 25 + N_THREADS = 40 + PER_REQ = 10 + N_SUCCESS = 15 # 只允许 15 个成功 + BUDGET = N_SUCCESS * PER_REQ # 150 + barrier = threading.Barrier(N_THREADS) + successes = [] + exceeded = [] + success_lock = threading.Lock() + + def worker(i): + barrier.wait() + try: + mod.reserve("t_precise", f"r{i}", PER_REQ, budget=BUDGET) + with success_lock: + successes.append(i) + except mod.BudgetExceeded: + with success_lock: + exceeded.append(i) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(successes) == N_SUCCESS, ( + f"expected exactly {N_SUCCESS} successes · got {len(successes)}" + ) + assert len(exceeded) == N_THREADS - N_SUCCESS + assert mod.get_pending_total("t_precise") == BUDGET # 全部成功者仍 pending -def test_release_unknown_id_no_op(tmp_path, monkeypatch): + # settle 全部成功者 · 无超扣 + for i in successes: + mod.settle("t_precise", f"r{i}", PER_REQ) + assert mod.get_used("t_precise") == BUDGET + assert mod.get_pending_total("t_precise") == 0 + + +# ── UTC 跨日 rollover 真测试(Codex §IV) ──────────────────────── + + +def test_utc_day_rollover_isolates_bucket(tmp_path, monkeypatch): + """monkeypatch `_today_utc` 模拟跨日 · 昨日 used 不影响今日 budget。""" _, mod = _enable_budget(tmp_path, monkeypatch) - mod.release("t1", "unknown_req") # 不 raise - assert mod.get_used("t1") == 0 + + day1 = "2026-08-09" + day2 = "2026-08-10" + + # Day 1 · 用满 budget=100 的 80 + monkeypatch.setattr(mod, "_today_utc", lambda: day1) + mod.reserve("t_roll", "r_d1", 80, budget=100) + mod.settle("t_roll", "r_d1", 80) + assert mod.get_used("t_roll") == 80 + + # Day 2 · 昨日 used 不算入今日 · 今日仍可 reserve 满 100 + monkeypatch.setattr(mod, "_today_utc", lambda: day2) + # 新 bucket · used 归 0 + assert mod.get_used("t_roll") == 0 + mod.reserve("t_roll", "r_d2", 100, budget=100) + mod.settle("t_roll", "r_d2", 100) + assert mod.get_used("t_roll") == 100 + + # 回到 Day 1 · 昨日 bucket 仍 80(未被今日污染) + monkeypatch.setattr(mod, "_today_utc", lambda: day1) + assert mod.get_used("t_roll") == 80 # ── observer hooks 不 enforce(§I 契约) ───────────────────────── def test_pre_api_request_observer_never_enforces(tmp_path, monkeypatch): - """`pre_api_request` observer hook 恒 no-op · 不 raise · 不改 state。 - - Codex §I 强制:budget enforcement 已经搬进 llm_execution middleware(见 - spike_p3_m0_cache)· 本 observer 只做 audit trigger point。 - """ from hermes_cli.plugins import invoke_hook _, mod = _enable_budget(tmp_path, monkeypatch) - # 无 tenant context · 无 config · 直接调 observer · 不 raise · state 不变 results = invoke_hook( "pre_api_request", api_request_id="r1", approx_input_tokens=999999, max_tokens=999999, ) - # observer 返回 None(_on_pre_api_request_observer)· invoke_hook 过滤 None → 空 list assert results == [] assert mod.get_used("t_any") == 0 assert mod.get_pending_total("t_any") == 0 diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index df2d430b12dee..52c2d37780c14 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -1,24 +1,54 @@ """P3-M0 Spike · tenant-safe cache + orchestrator(llm_execution middleware). -**Codex §I / §III / §IV 契约**: +**Codex 第三轮 §I / §II / §III / §V 契约(修订)**: 1. 注册 `llm_execution` middleware(`ctx.register_middleware("llm_execution", ...)`)· 经过 `run_llm_execution_middleware`(`hermes_cli/middleware.py:187`)真正生产走位· fail-open on callback(fork middleware 语义 · 见 middleware.py:_run_execution_chain) -2. 显式 orchestrate order: - cache lookup - ├─ hit → 返回 cached + 顶层 usage=0 · cache_meta.hit=True · **不 reserve budget** - └─ miss → budget reserve - → next_call(request) - ├─ success → cache insert(if cacheable) + budget settle(actual) - └─ exception → budget release · re-raise -3. 严格 tenant fail-closed:tenant_id/principal_id/permission_scope_version/ - profile_version 任一缺失/空/非法 → 走 next_call(不 lookup 不 insert)· log context_invalid -4. **只**支持 `api_mode == "chat_completions"`;其他 3 mode fail-closed pass-through + +2. **控制流严格解耦 · cache eligibility ≠ budget enforcement**: + + 插件禁用 → next_call(不 lookup 不 reserve) + 插件启用 → + ├─ 硬依赖门:budget module 缺失 → fail-CLOSED(结构化 error response · 不 next_call) + ├─ tenant context 4 字段严格校验(缺/空/非白/超长)→ fail-CLOSED + ├─ budget config 校验(BudgetConfigError → fail-CLOSED · 不静默 None) + ├─ 非 chat_completions api_mode → fail-CLOSED(M0 明确缩小支持范围 · 见 §II) + ├─ 非 dict request → fail-CLOSED + │ + ├─ 判 cacheable_request(streaming/tools 非 cacheable · **但仍 reserve/settle**) + │ + ├─ cacheable_request → cache lookup + │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) + │ + ├─ (MISS 或 非 cacheable_request)→ budget.reserve + │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) + │ └─ 成功 → next_call + │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) + │ └─ 异常 → budget.release + re-raise + +3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / + profile_version 任一缺失/空/非法 → **不 next_call** · 返回结构化 fail-closed response + · **禁**默认填 "system"/"0" 等安全值(Codex §III) + +4. **只**支持 `api_mode == "chat_completions"`:其他 3 mode(anthropic_messages / + bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· + M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 + 5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` -6. 并发:threading.Lock 保护 LRU + insert 序 -7. 配置:从 `hermes_cli.config.load_config` 读 `plugins.spike_p3_m0_cache.enabled` 等 - · **不**用 SPIKE_* env · **不**读 profile.yaml + +6. 并发:threading.RLock 保护 LRU + insert 序 + +7. 配置: + - `plugins.spike_p3_m0_cache.enabled` **严格 bool**(v is True · 拒 "false"/1/等) + - `plugins.spike_p3_m0_cache.tenant.*`(4 字段 · str · 白名单 · len≤128) + - `plugins.spike_p3_m0_cache.tenant.daily_budget_tokens`(4 态 · BudgetConfigError 硬 fail-CLOSED) + - **不用** SPIKE_* env · **不**读 profile.yaml · **不**读 HERMES_DAILY_TOKEN_BUDGET + +8. **input token 估算**(Codex §V):不得静默 0 后宣称预算护栏成立。 + - 优先信 ctx.approx_input_tokens(若 int/float > 0) + - 否则从 request.messages content 字符数 / 4(粗略英文 heuristic · M0 approximation · 见文档) + - 加上 request.max_tokens 上限 · 得 tokens_needed """ from __future__ import annotations @@ -41,6 +71,9 @@ def _budget_module(): hermes_cli.plugins loads bundled plugins under ``hermes_plugins.*``; running the plugins directly from a checkout may see ``plugins.*`` instead. Lazy lookup avoids a hard ``from .. import`` that only works under one layout. + + **Codex §III**:budget 模块缺失时 middleware **不**得 fail-open · 由调用方(orchestrator) + 捕获 ImportError 走 fail-CLOSED response · 保持"cache 单独启用"场景可测。 """ parent = __name__.rsplit(".", 1)[0] if "." in __name__ else "" if parent: @@ -51,13 +84,11 @@ def _budget_module(): return importlib.import_module(candidate) except ImportError: pass - for name, mod in list(sys.modules.items()): - if name.endswith(".spike_p3_m0_budget") or name == "spike_p3_m0_budget": - return mod raise ImportError( - "spike_p3_m0_budget module not loaded; enable it in plugins.enabled" + "spike_p3_m0_budget module not loaded; enable spike-p3-m0-budget in plugins.enabled" ) + CACHE_CONTRACT_VERSION = "v1" CODEC_VERSION = "codec_v1" CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent @@ -67,6 +98,14 @@ def _budget_module(): _ID_ALLOWED_RE = re.compile(r"^[a-zA-Z0-9_\-]+$") _ID_MAX_LEN = 128 +# fail-closed 结构化 response 消息(非空 · 可展示 · finish_reason=content_filter 抑制 retry) +_BUDGET_EXCEEDED_MESSAGE = ( + "[请求已被拒绝 · tenant 每日预算已达上限 · 请稍后重试或联系管理员]" +) +_BUDGET_FAIL_CLOSED_MESSAGE = ( + "[请求已被拒绝 · 预算护栏未就绪 · 请联系管理员]" +) + # ── Module state · thread-safe LRU ──────────────────────────────────── @@ -95,8 +134,12 @@ def _get_plugin_config() -> Dict[str, Any]: def _is_enabled() -> bool: - """`plugins.spike_p3_m0_cache.enabled` · 默认 False(opt-in)。""" - return bool(_get_plugin_config().get("enabled")) + """`plugins.spike_p3_m0_cache.enabled` **严格 bool** · 只接受 literal `True`。 + + Codex §III 硬约束:拒 `bool("false") == True`(str 转 bool 陷阱)· 拒 int/list/dict。 + True → 启用;其他(False / None / "true" / 1 / [] / {} / ...)→ 视 as 禁用。 + """ + return _get_plugin_config().get("enabled") is True def _get_tenant_context() -> Optional[Dict[str, str]]: @@ -111,6 +154,9 @@ def _get_tenant_context() -> Optional[Dict[str, str]]: 进程仅信 config · 不信 prompt/tool_args。 """ tenant_block = _get_plugin_config().get("tenant") or {} + if not isinstance(tenant_block, dict): + logger.info("spike-cache: tenant context invalid · not a mapping") + return None required = ("tenant_id", "principal_id", "permission_scope_version", "profile_version") resolved: Dict[str, str] = {} for field in required: @@ -134,28 +180,18 @@ def _get_tenant_context() -> Optional[Dict[str, str]]: return resolved -def _get_budget_for_tenant() -> Optional[int]: - """`plugins.spike_p3_m0_cache.tenant.daily_budget_tokens` · 4 态归一化。 - - 非法值(bool/负/str/float)→ 记录 audit · 返回 None(fail-open · 不 enforce) - """ - raw = (_get_plugin_config().get("tenant") or {}).get("daily_budget_tokens") - budget_mod = _budget_module() - try: - return budget_mod.validate_daily_budget(raw) - except budget_mod.BudgetConfigError as exc: - logger.warning("spike-cache: daily_budget_tokens config invalid · not enforced: %s", exc) - return None - - -# ── Cacheability guards(§4)── 只支持 chat_completions ──────────── +# ── Cacheability guards ───────────────────────────────────────────── def _api_mode_supported(api_mode: str) -> bool: - """M0 只支持 `chat_completions` · 其他 fail-closed pass-through。""" + """M0 只支持 `chat_completions`(见 §II)· 其他 mode fail-CLOSED · 不 next_call。""" return api_mode == "chat_completions" def is_cacheable_request(request: Any, *, api_mode: str = "chat_completions") -> bool: + """判 request 是否可 lookup/insert cache(**与 budget 决策解耦**)。 + + Codex §II:streaming / tools 可以不缓存 · 但**不能不计预算**。 + """ if not _api_mode_supported(api_mode): return False if not isinstance(request, dict): @@ -256,84 +292,213 @@ def _get(obj, k, default=None): } -# ── Orchestrator middleware(cache + budget 显式组合)──────────── +# ── Input token 估算(Codex §V:不得静默 0) ────────────────────── + +def _estimate_input_tokens_from_messages(request: Dict[str, Any]) -> int: + """从 messages content 字符数估算 input tokens · **M0 approximation**。 + + heuristic:总字符数 ÷ 4(rough 英文 char-per-token)· 中文/emoji 会低估 · + 但对 M0 预算护栏"防明显超支"目标足够 · 生产 M1+ 需接 tiktoken/tokenizer 精算 + (out-of-band 契约字段 · 见 docs/fork/budget-hook.md §5b)。 + + 保底:若 messages 全空/无 content · 返回 0 · 由调用方结合 max_tokens 判断。 + """ + if not isinstance(request, dict): + return 0 + msgs = request.get("messages") or [] + if not isinstance(msgs, list): + return 0 + total_chars = 0 + for m in msgs: + if not isinstance(m, dict): + continue + content = m.get("content") + if isinstance(content, str): + total_chars += len(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict): + t = part.get("text") + if isinstance(t, str): + total_chars += len(t) + return total_chars // 4 + + +def _estimate_needed_tokens(request: Any, ctx: Dict[str, Any]) -> int: + """总预留 = input 估算 + max_tokens · **不静默 0**(Codex §V)。 + + 优先信 ctx.approx_input_tokens(若 int/float > 0);否则从 messages 估算 · + 加上 request.max_tokens(或 ctx.max_tokens)· 得 tokens_needed。 + """ + explicit = ctx.get("approx_input_tokens") + if isinstance(explicit, bool): + explicit = None + if isinstance(explicit, (int, float)) and explicit > 0: + approx_input = int(explicit) + else: + approx_input = _estimate_input_tokens_from_messages(request if isinstance(request, dict) else {}) + max_tokens_raw = None + if isinstance(request, dict): + max_tokens_raw = request.get("max_tokens") + if max_tokens_raw is None: + max_tokens_raw = ctx.get("max_tokens") + try: + max_tokens = int(max_tokens_raw or 0) + except (TypeError, ValueError): + max_tokens = 0 + if max_tokens < 0: + max_tokens = 0 + return approx_input + max_tokens + + +# ── Structured error responses(Codex §V:非空、安全、可展示) ───── + +def _fail_closed_response(model: str, *, reason_code: str, **extras: Any) -> Dict[str, Any]: + """结构化 fail-closed response · finish_reason=content_filter 抑制 retry/fallback。 + + Codex §V:BudgetExceeded / 预算护栏未就绪 时返回必须**非空** · 安全可展示 · + 不触发 retry。上层 orchestrator 若追加 retry 决策 · 需读 `cache_meta.reason_code` + 做过滤(M1+ 契约:orchestrator 遇 content_filter + cache_meta.reason_code 明确 · 不 retry)。 + """ + return { + "model": model or "", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": _BUDGET_EXCEEDED_MESSAGE if reason_code == "budget_exceeded" else _BUDGET_FAIL_CLOSED_MESSAGE, + }, + "finish_reason": "content_filter", + }], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "cache_meta": { + "hit": False, + "budget_exceeded": reason_code == "budget_exceeded", + "budget_fail_closed": reason_code != "budget_exceeded", + "reason_code": reason_code, + **extras, + }, + } + + +# ── Orchestrator middleware(cache + budget 显式组合 · 解耦) ───── def cache_and_budget_middleware( request: Any, next_call: Callable[[Any], Any], **ctx: Any, ) -> Any: - """`llm_execution` middleware · 严格 Codex §I 顺序: + """`llm_execution` middleware · 严格 Codex 第三轮 §II 控制流(解耦 cache 与 budget)。 - cache lookup → hit(返回 · 不 reserve)/ miss → reserve → next_call → - success(cache insert + settle actual)/ exception(release + re-raise) - - fail-closed 无 tenant · fail-closed 非 chat_completions · fail-open 非 dict. + 见模块 docstring §2 完整控制流。 """ - api_request_id = ctx.get("api_request_id") or "" - api_mode = ctx.get("api_mode") or "chat_completions" - provider = ctx.get("provider") or "" - model = ctx.get("model") or "" + api_request_id = str(ctx.get("api_request_id") or "") + api_mode = str(ctx.get("api_mode") or "chat_completions") + provider = str(ctx.get("provider") or "") + model = str(ctx.get("model") or "") - # Fast pass-through · 未启用 / 不支持 mode / 非 dict request - if not _is_enabled() or not _api_mode_supported(api_mode) or not isinstance(request, dict): + # 1. Plugin disabled → next_call 直通(Codex §II step 1) + if not _is_enabled(): return next_call(request) - tenant_ctx = _get_tenant_context() - if not tenant_ctx: - # 严格 fail-closed · 无 tenant → 走 next_call(不 lookup 不 insert · 不 reserve) - return next_call(request) + # 2. Budget module 硬依赖门(Codex §III:cache 单独启用不得因 budget 缺失 fail-open) + try: + budget_mod = _budget_module() + except ImportError as exc: + logger.error( + "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED: %s", + exc, + ) + return _fail_closed_response(model, reason_code="budget_module_missing") + + # 3. api_mode 支持范围收紧(Codex §II:非 chat_completions fail-CLOSED · 不静默 pass-through) + if not _api_mode_supported(api_mode): + logger.warning( + "spike-cache: unsupported api_mode=%s · M0 only supports chat_completions · fail-CLOSED", + api_mode, + ) + return _fail_closed_response(model, reason_code="unsupported_api_mode", api_mode=api_mode) - if not is_cacheable_request(request, api_mode=api_mode): - return next_call(request) + # 4. request 类型门(non-dict 无法 estimate token · fail-CLOSED) + if not isinstance(request, dict): + logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) + return _fail_closed_response(model, reason_code="non_dict_request") - key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + # 5. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) + tenant_ctx = _get_tenant_context() + if tenant_ctx is None: + return _fail_closed_response(model, reason_code="tenant_context_invalid") tenant_id = tenant_ctx["tenant_id"] - # ── 1. Cache lookup ── - with _lock: - hit = _cache.get(key) + # 6. Budget config 校验 · BudgetConfigError 硬 fail-CLOSED(Codex §III:不转 None 静默继续) + raw_budget = (_get_plugin_config().get("tenant") or {}).get("daily_budget_tokens") + try: + budget_value = budget_mod.validate_daily_budget(raw_budget) + except budget_mod.BudgetConfigError as exc: + logger.error("spike-cache: daily_budget_tokens config invalid · fail-CLOSED: %s", exc) + return _fail_closed_response(model, reason_code="budget_config_invalid") + + # 7. api_request_id 门(reserve 需唯一 id · 空则无法 settle) + if not api_request_id: + logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") + return _fail_closed_response(model, reason_code="missing_api_request_id") + + # 8. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) + cacheable_req = is_cacheable_request(request, api_mode=api_mode) + + # 9. 若可缓存 → lookup · HIT 直接返回(不 reserve) + cache_key: Optional[str] = None + if cacheable_req: + cache_key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + with _lock: + hit = _cache.get(cache_key) + if hit is not None: + _cache.move_to_end(cache_key) if hit is not None: - _cache.move_to_end(key) # LRU touch - if hit is not None: - # Hit · 返回 cached · **不 reserve budget** · 顶层 usage=0 - return dict(hit) # shallow copy · 避免调用方 mutate - - # ── 2. Miss · budget reserve(basedon config budget) ── - budget_value = _get_budget_for_tenant() - approx_input = int(ctx.get("approx_input_tokens") or 0) - max_tokens = int(request.get("max_tokens") or ctx.get("max_tokens") or 0) - tokens_needed = approx_input + max_tokens - - budget_mod = _budget_module() + # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate + return dict(hit) + + # 10. MISS 或非 cacheable · 都必须 reserve(Codex §II step 6) + tokens_needed = _estimate_needed_tokens(request, ctx) try: budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) except budget_mod.BudgetExceeded: - # 预算超支 · 明确阻断 · 不调 next_call · 返回结构化 error response - logger.warning("spike-cache-orchestrator: budget exceeded · tenant=%s api_request_id=%s", tenant_id, api_request_id) - return { - "model": model, - "choices": [{"message": {"content": ""}, "finish_reason": "content_filter"}], - "usage": {"prompt_tokens": 0, "completion_tokens": 0}, - "cache_meta": {"hit": False, "budget_exceeded": True, "tenant_id": tenant_id}, - } - - # ── 3. next_call · success settle actual / exception release ── + logger.warning( + "spike-cache: budget exceeded · tenant=%s api_request_id=%s tokens_needed=%s budget=%s", + tenant_id, api_request_id, tokens_needed, budget_value, + ) + return _fail_closed_response( + model, + reason_code="budget_exceeded", + tenant_id=tenant_id, + tokens_needed=tokens_needed, + budget=budget_value, + ) + + # 11. next_call · success settle actual / exception release try: response = next_call(request) except Exception: budget_mod.release(tenant_id, api_request_id) raise - # ── 4. Cache insert(if cacheable) + budget settle actual ── + # 12. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) actual = _extract_actual_usage(response) - budget_mod.settle(tenant_id, api_request_id, actual) - - if is_cacheable_response(response): + try: + budget_mod.settle(tenant_id, api_request_id, actual) + except budget_mod.BudgetSettleError as exc: + # 理论不该发生(reserve 刚成功)· 记 audit · 不阻断 response + logger.error( + "spike-cache: settle raised BudgetSettleError post-reserve · api_request_id=%s: %s", + api_request_id, exc, + ) + + # 13. 只有 req + resp 皆 cacheable 才 insert + if cacheable_req and cache_key is not None and is_cacheable_response(response): cached = _extract_cacheable_response(response) with _lock: - _cache[key] = cached - _cache.move_to_end(key) + _cache[cache_key] = cached + _cache.move_to_end(cache_key) while len(_cache) > CACHE_MAX_SIZE: _cache.popitem(last=False) diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index fdb342ef02250..6f2c39bc3c38c 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -1,17 +1,18 @@ """P3-M0 Spike · cache + budget orchestrator middleware 测试(**真实生产走位**). -**Codex §I / §VIII 契约**: +**Codex 第三轮 §I / §II / §III / §V / §VIII 契约**: - 通过临时 `config.yaml` 启用插件 · 走真实 `PluginManager.discover_and_load` - `has_middleware("llm_execution") == True` - 通过 `run_llm_execution_middleware` 触发 · **不**直接调 `cache_and_budget_middleware` - **不用** `SPIKE_*` env(fork AGENTS.md red-line 3)· **不污染**全局 manager -- 覆盖: - * cache miss → budget reserve/settle 各 1 次 - * cache hit → budget reserve 0 次(顶层 usage=0) - * budget 超支 → next_call 0 次(structured error response · budget_exceeded) - * provider exception → reservation 释放 · exception 传播 - * tenant context 缺失/非法 → fail-closed(pass-through · 不 lookup 不 reserve) - * streaming / tools request → pass-through +- **cache eligibility ≠ budget enforcement 严格解耦**: + * streaming / tools 非 cacheable · **但仍 reserve/settle** + * 非 chat_completions api_mode → fail-CLOSED(next_call == 0)· M0 缩小支持范围 + * budget 模块缺失 / tenant context 缺失 / budget config 非法 → fail-CLOSED · next_call == 0 +- cache miss + cacheable req → reserve/settle 各 1 次 +- cache hit → 不 reserve · 顶层 usage=0 +- provider exception → release 且原样传播 +- 严格 bool `enabled`(bool("false")==True 陷阱阻断) """ from __future__ import annotations @@ -45,31 +46,45 @@ def _enable_cache( *, daily_budget=None, tenant=_VALID_TENANT, + enabled=True, + include_budget=True, ): tenant_cfg = dict(tenant) if daily_budget is not None: tenant_cfg["daily_budget_tokens"] = daily_budget + enabled_list = [CACHE_KEY] + if include_budget: + enabled_list.append(BUDGET_KEY) write_config( tmp_path, - [CACHE_KEY, BUDGET_KEY], + enabled_list, spike_p3_m0_cache={ - "enabled": True, + "enabled": enabled, "tenant": tenant_cfg, }, ) manager = install_fresh_manager(monkeypatch, tmp_path) cache_mod = get_discovered_module(manager, CACHE_KEY) - budget_mod = get_discovered_module(manager, BUDGET_KEY) cache_mod.clear_cache() - budget_mod.reset_state() + budget_mod = None + if include_budget: + budget_mod = get_discovered_module(manager, BUDGET_KEY) + budget_mod.reset_state() return manager, cache_mod, budget_mod +def _is_fail_closed(resp): + """Structured fail-closed response 特征:cache_meta 含 budget_exceeded 或 budget_fail_closed。""" + if not isinstance(resp, dict): + return False + meta = resp.get("cache_meta") or {} + return bool(meta.get("budget_exceeded") or meta.get("budget_fail_closed")) + + # ── Registration invariants(§I 强要求) ───────────────────────────── def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): - """PluginContext.register_middleware("llm_execution", ...) 生效 · fork middleware.py 契约。""" from hermes_cli.plugins import has_middleware manager, _, _ = _enable_cache(tmp_path, monkeypatch) @@ -78,15 +93,56 @@ def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): - """`plugins.enabled` 不含 spike-p3-m0-cache · discover 不 load · 无 middleware。""" from hermes_cli.plugins import has_middleware - write_config(tmp_path, []) # 空 enabled · 无插件加载 + write_config(tmp_path, []) install_fresh_manager(monkeypatch, tmp_path) assert has_middleware("llm_execution") is False -# ── Cache miss → budget reserve+settle 各 1 次(§I 强要求) ───────── +def test_enabled_bool_strict_true_only(tmp_path, monkeypatch): + """Codex §III:`enabled` 严格 bool True · 拒 str/int/其他(bool('false')==True 陷阱)。""" + # enabled = "false"(str)· 应视 as 禁用 · 直通 next_call + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled="false", + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + # 未启用 → 直通 next · 未 reserve + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +@pytest.mark.parametrize("enabled_value", [1, "true", "yes", [1], {"x": 1}]) +def test_enabled_non_bool_treated_as_disabled(tmp_path, monkeypatch, enabled_value): + """任何非 literal-True 值(int / str / list / dict)· 视 as 禁用 · 直通 next。""" + _, _, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled=enabled_value, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +# ── Cache miss(cacheable request) → reserve+settle 各 1 次 ───── def test_miss_reserves_and_settles_once(tmp_path, monkeypatch): @@ -109,6 +165,7 @@ def next_call(_req): provider="openai", model="gpt-4o", api_mode="chat_completions", + approx_input_tokens=200, ) resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) @@ -116,7 +173,6 @@ def next_call(_req): assert resp["choices"][0]["message"]["content"] == "hi" # settle 累加 actual usage = 150 assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 150 - # 无 pending reservations assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 @@ -143,6 +199,7 @@ def next_call(_req): provider="openai", model="gpt-4o", api_mode="chat_completions", + approx_input_tokens=300, ) req = {"messages": [{"role": "user", "content": "same"}]} @@ -159,44 +216,34 @@ def next_call(_req): api_mode="chat_completions", ) resp = run2(req, next_call) - assert counters["next"] == 1 # 未再调 next - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 # 未 charge - # 顶层 usage = billable_usage = 全 0 + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} - # cache_meta 拆分正确 assert resp["cache_meta"]["hit"] is True assert resp["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} assert resp["cache_meta"]["origin_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} assert resp["cache_meta"]["saved_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} assert resp["cache_meta"]["cache_contract_version"] == cache_mod.CACHE_CONTRACT_VERSION - # 二次调用不该有 pending reservation(hit 直接返回) assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 -# ── Budget 超支 → next_call 0 次(§I 强要求) ──────────────────────── +# ── Budget 超支 → next_call 0 次(§I / §V 强要求 · 非空结构化响应) ─ -def test_budget_exceeded_blocks_next_call(tmp_path, monkeypatch): +def test_budget_exceeded_blocks_next_call_returns_safe_response(tmp_path, monkeypatch): manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=100 # 极小 + tmp_path, monkeypatch, daily_budget=100 ) counters = {"next": 0} def next_call(_req): counters["next"] += 1 - return { - "model": "m", - "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 0, "completion_tokens": 0}, - } + return {"model": "m", "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} run = make_run_llm_execution( - api_request_id="req_over", - provider="openai", - model="m", - api_mode="chat_completions", - approx_input_tokens=500, # 已远超 budget 100 + api_request_id="req_over", provider="openai", model="m", api_mode="chat_completions", + approx_input_tokens=500, ) resp = run( {"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, @@ -204,15 +251,99 @@ def next_call(_req): ) # 超支明确 short-circuit · next 未调 assert counters["next"] == 0 - # 结构化 error response · budget_exceeded=True + # 非空结构化 error response · 抑制 retry · Codex §V assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["reason_code"] == "budget_exceeded" assert resp["cache_meta"]["tenant_id"] == _VALID_TENANT["tenant_id"] + # 非空 assistant message(可展示给用户) + assert resp["choices"][0]["message"]["role"] == "assistant" + assert resp["choices"][0]["message"]["content"] # 非空字符串 + # finish_reason 抑制 retry(content_filter) + assert resp["choices"][0]["finish_reason"] == "content_filter" + # 顶层 usage=0 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} # 无 pending reservation(reserve 抛异常 · 未记账) assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 -# ── Provider exception → reservation 释放 + 传播(§I 强要求) ─────── +def test_budget_exceeded_on_streaming_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:streaming 请求 + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_stream", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "stream": True, "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 # streaming 也被 reserve 门阻断 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_budget_exceeded_on_tools_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:tools request + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_tools", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}], "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_non_cacheable_request_still_reserves_and_settles(tmp_path, monkeypatch): + """Codex §II:streaming/tools 非 cacheable · **但仍 reserve/settle**(解耦)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "streamed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 60, "completion_tokens": 40}} + + # streaming request(不缓存 · 但必须 reserve) + run = make_run_llm_execution( + api_request_id="r_s1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 100 # settle 60+40 + # 不 cache · 二次同请求仍 miss(需 fresh api_request_id) + run2 = make_run_llm_execution( + api_request_id="r_s2", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + run2({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 2 # 再调 next + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 # 再 settle 100 + + +# ── Provider exception → reservation 释放 + 传播 ──────────────────── def test_provider_exception_releases_reservation(tmp_path, monkeypatch): @@ -227,22 +358,18 @@ def next_call(_req): raise ProviderBoom("upstream 500") run = make_run_llm_execution( - api_request_id="req_boom", - provider="openai", - model="m", - api_mode="chat_completions", + api_request_id="req_boom", provider="openai", model="m", api_mode="chat_completions", approx_input_tokens=100, ) with pytest.raises(ProviderBoom, match="upstream 500"): run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) - # reservation 已释放 · used 未增 assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None -# ── Tenant context fail-closed(§III 严格) ───────────────────────── +# ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ @pytest.mark.parametrize( @@ -250,18 +377,17 @@ def next_call(_req): [ {"tenant_id": "", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, {"tenant_id": "t", "principal_id": "", "permission_scope_version": "1", "profile_version": "1"}, - {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, # 缺 profile_version + {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, {"tenant_id": "t/../evil", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, {"tenant_id": "a" * 200, "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, ], ids=["empty-tenant", "empty-principal", "missing-profile-version", "non-whitelist-chars", "over-max-len"], ) -def test_invalid_tenant_pass_through(tmp_path, monkeypatch, bad_tenant): - """任一 tenant 字段违反 · pass-through(不 lookup 不 reserve)· fail-closed。""" +def test_invalid_tenant_fail_closed_blocks_next_call(tmp_path, monkeypatch, bad_tenant): + """Codex §III:enforcement 开启时 · 缺失/非法 tenant · **next_call == 0** · 结构化 fail-closed。""" manager, cache_mod, budget_mod = _enable_cache( tmp_path, monkeypatch, daily_budget=100, tenant=bad_tenant ) - counters = {"next": 0} def next_call(_req): @@ -270,22 +396,24 @@ def next_call(_req): run = make_run_llm_execution( api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=500, # 若 reserve 生效必超支 · 用于反证 fail-closed 走 next + approx_input_tokens=500, ) resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) - assert counters["next"] == 1 # 走 next(未 reserve, 否则会 budget_exceeded) - assert "cache_meta" not in resp + assert counters["next"] == 0 # fail-CLOSED · 严格 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" + assert resp["choices"][0]["message"]["content"] # 非空可展示 error + assert resp["choices"][0]["finish_reason"] == "content_filter" -def test_no_tenant_config_pass_through(tmp_path, monkeypatch): - """config.yaml 无 tenant 段 · fail-closed pass-through。""" +def test_no_tenant_config_fail_closed(tmp_path, monkeypatch): + """config.yaml 无 tenant 段 · fail-CLOSED(不走 next_call)。""" write_config( tmp_path, [CACHE_KEY, BUDGET_KEY], spike_p3_m0_cache={"enabled": True}, # 无 tenant 段 ) - manager = install_fresh_manager(monkeypatch, tmp_path) - + install_fresh_manager(monkeypatch, tmp_path) counters = {"next": 0} def next_call(_req): @@ -296,33 +424,51 @@ def next_call(_req): api_request_id="r1", provider="p", model="m", api_mode="chat_completions", ) resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) - assert counters["next"] == 1 - assert "cache_meta" not in resp + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" -# ── Streaming / tools request pass-through(§4 mode 限定) ─────────── +def test_non_chat_completions_mode_fail_closed(tmp_path, monkeypatch): + """Codex §II:非 chat_completions api_mode 明确 fail-CLOSED(不静默 pass-through)。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="responses", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "unsupported_api_mode" + assert resp["cache_meta"]["api_mode"] == "responses" -def test_streaming_request_pass_through(tmp_path, monkeypatch): +def test_non_dict_request_fail_closed(tmp_path, monkeypatch): + """非 dict request 无法 estimate token · fail-CLOSED。""" manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) counters = {"next": 0} def next_call(_req): counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + return None run = make_run_llm_execution( api_request_id="r1", provider="p", model="m", api_mode="chat_completions", ) - # 首次 · streaming=True → 不 lookup 不 insert - run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) - # 二次 · streaming=True · 仍 miss → next 再调 - run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) - assert counters["next"] == 2 + resp = run(["not", "a", "dict"], next_call) # list · 非 dict + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "non_dict_request" -def test_tools_request_pass_through(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) +@pytest.mark.parametrize("bad_budget", [True, False, -1, "1000", 1.5]) +def test_invalid_budget_config_fail_closed(tmp_path, monkeypatch, bad_budget): + """Codex §III:BudgetConfigError 硬 fail-CLOSED · **不**转 None 静默继续。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=bad_budget + ) counters = {"next": 0} def next_call(_req): @@ -332,13 +478,13 @@ def next_call(_req): run = make_run_llm_execution( api_request_id="r1", provider="p", model="m", api_mode="chat_completions", ) - req = {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}]} - run(req, next_call) - run(req, next_call) - assert counters["next"] == 2 # 两次 miss(不 insert) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "budget_config_invalid" -def test_non_chat_completions_mode_pass_through(tmp_path, monkeypatch): +def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): + """空 api_request_id · reserve 无法追踪 · fail-CLOSED。""" manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) counters = {"next": 0} @@ -347,12 +493,36 @@ def next_call(_req): return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="responses", # 非 chat_completions + api_request_id="", provider="p", model="m", api_mode="chat_completions", ) - req = {"messages": [{"role": "user", "content": "x"}]} - run(req, next_call) - run(req, next_call) - assert counters["next"] == 2 # 两次都 miss(不 insert) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" + + +# ── Cache alone(budget disabled)· 硬依赖门 fail-CLOSED ───────────── + + +def test_cache_alone_without_budget_fail_closed(tmp_path, monkeypatch): + """Codex §III:cache 单独启用时 budget 缺失 · **必须 fail-CLOSED** · 不 pass-through。""" + # 只启用 cache · 不启用 budget · 但 config tenant 完整 + manager, cache_mod, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=1000, include_budget=False, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + # budget module 未加载 · 硬 fail-CLOSED + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "budget_module_missing" # ── Cache key schema stability(§2 v1 schema) ────────────────────── @@ -406,4 +576,29 @@ def next_call(_req): req = {"messages": [{"role": "user", "content": "call tool"}]} run1(req, next_call) run2(req, next_call) - assert counters["next"] == 2 # 两次都 miss · tool_calls 不 insert + assert counters["next"] == 2 + + +# ── Input token estimation (Codex §V:不静默 0) ──────────────────── + + +def test_input_tokens_estimated_from_messages_when_no_ctx(tmp_path, monkeypatch): + """无 ctx.approx_input_tokens · 从 messages char count / 4 估算 · 不静默 0。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2}} + + # 未传 approx_input_tokens · content 长约 4000 字符 → 估算 ~1000 tokens 触发超支 + long_content = "x" * 4000 + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) + # 估算 = 4000/4 + 100 = 1100 · budget=1000 · 超支 + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["tokens_needed"] >= 1000 diff --git a/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py b/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py new file mode 100644 index 0000000000000..50689c02f63ce --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py @@ -0,0 +1,58 @@ +"""P3-M0 · Vendored copy of Hermes_AI `hermes_devices/ops/audit.py::strip_sources`. + +**Purpose**:cross-repo import is not always available in fork CI; +this file mirrors the Hermes_AI baseline **verbatim** so the differential test +(`plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py`) can +compare fork sanitizer output against it and lock the exact divergence. + +**Source of truth**: + Repo: https://github.com/bri12afdsarker96-lgtm/Hermes_AI + File: `hermes_devices/ops/audit.py` + Symbols mirrored: `_INLINE_RE`, `_LINE_RE`, `strip_sources` + Baseline commit: PR #27 merge `aa5c3c4f4f862b6e7bf4a9bc97f5dfeb35489e55` + (P2-0.1 段① · charter C4) + +**Sync discipline**:when the Hermes_AI baseline changes, this file MUST be +resynced in the same PR, and both repos' PR bodies must reference the drift. +The differential test's `test_baseline_regex_verbatim` byte-checks the strings. +""" +from __future__ import annotations + +import re + + +# ── mirrored from Hermes_AI hermes_devices/ops/audit.py:74-91 (verbatim) ── + +# Citation markers we strip from user-bound text. Deliberately conservative: +# only bracketed/labelled "source/来源" fragments — ordinary brackets survive. +# The body class excludes \n (an unclosed marker must NOT swallow following +# lines of real reply text — that falls through to _LINE_RE instead) and +# allows one nested bracket pair (来源：FAQ（2026版）) without leaving residue. +_INLINE_RE = re.compile( + r"[\[【（(]\s*(?:source|来源)\s*[:：]" + r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】）)]", + re.IGNORECASE, +) +_LINE_RE = re.compile(r"^[ \t]*(?:source|来源)\s*[:：].*$", re.IGNORECASE | re.MULTILINE) + + +def strip_sources(text: str) -> str: + """Remove citation markers from a user-bound reply (charter C4).""" + if not text: + return text + out = _INLINE_RE.sub("", text) + out = _LINE_RE.sub("", out) + out = re.sub(r"[ \t]+(?=\n|$)", "", out) # trailing spaces left by removals + out = re.sub(r"\n{3,}", "\n\n", out) + return out.strip() + + +# Verbatim source strings — locked by test_baseline_regex_verbatim so any drift +# from the Hermes_AI baseline flips the differential test loudly. +BASELINE_INLINE_PATTERN = ( + r"[\[【（(]\s*(?:source|来源)\s*[:：]" + r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】）)]" +) +BASELINE_LINE_PATTERN = r"^[ \t]*(?:source|来源)\s*[:：].*$" diff --git a/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py index cfdc5d840df9e..e2eceed0b3599 100644 --- a/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py +++ b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py @@ -37,7 +37,7 @@ def write_config(hermes_home: Path, enabled: List[str], **plugin_cfg: Any) -> No ) -def install_fresh_manager(monkeypatch, hermes_home: Path): +def install_fresh_manager(monkeypatch, hermes_home: Path, request=None): """Replace the global PluginManager with a fresh instance for this test. Steps: @@ -49,6 +49,10 @@ def install_fresh_manager(monkeypatch, hermes_home: Path): Fixture teardown (monkeypatch scope) restores env + module attr. Returns the fresh manager. + + **Codex 第三轮 §VI**:测试可选调用 :func:`assert_no_plugin_module_leak` (见下) + 在 teardown 阶段快照对比 `hermes_plugins.*`,证明 discover 引入的模块在 + fixture 结束后全部清除,不污染后续测试。 """ from hermes_cli import plugins as plugins_mod @@ -65,14 +69,52 @@ def install_fresh_manager(monkeypatch, hermes_home: Path): fresh = plugins_mod.PluginManager() monkeypatch.setattr(plugins_mod, "_plugin_manager", fresh) + # 快照:进入 fixture 前 sys.modules 里已经存在的 hermes_plugins.* 集合 + pre_existing = {k for k in list(sys.modules) if k.startswith("hermes_plugins")} # 卸载 hermes_plugins.* 命名空间(避免上次 discover 的旧 module 状态污染) - for name in [k for k in list(sys.modules) if k.startswith("hermes_plugins")]: + for name in list(pre_existing): monkeypatch.delitem(sys.modules, name, raising=False) fresh.discover_and_load(force=True) + + # Codex 第三轮 §VI:teardown 显式清除 discover 引入的 hermes_plugins.* 新键, + # 确保 fixture 结束后 sys.modules 恢复到"进入 fixture 前"的精确状态。 + # 若 request 传入 · 用 pytest addfinalizer(monkeypatch 无此 API);否则记录 + # 一个显式 helper 供测试手工清理与验证。 + if request is not None: + def _cleanup_new_plugin_modules(): + for name in list(sys.modules): + if name.startswith("hermes_plugins") and name not in pre_existing: + sys.modules.pop(name, None) + request.addfinalizer(_cleanup_new_plugin_modules) + return fresh +def snapshot_plugin_modules() -> set: + """Snapshot of currently-loaded ``hermes_plugins.*`` module names. + + Used by :func:`assert_no_plugin_module_leak` and the standalone test + ``test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue`` to prove + that `install_fresh_manager` + monkeypatch teardown removes every module it + caused to be inserted. + """ + return {k for k in sys.modules if k.startswith("hermes_plugins")} + + +def assert_no_plugin_module_leak(before: set) -> None: + """Assert no new ``hermes_plugins.*`` entries remain in sys.modules. + + Call **after** monkeypatch fixture teardown to verify cleanup. + """ + after = snapshot_plugin_modules() + leaked = after - before + assert not leaked, ( + f"hermes_plugins.* module leak after fixture teardown: {sorted(leaked)} · " + "install_fresh_manager teardown did not clean up loaded plugin modules" + ) + + def get_discovered_module(manager, plugin_key: str): """Return the imported module object of a plugin by its plugin.yaml `name`.""" loaded = manager._plugins.get(plugin_key) diff --git a/plugins/spike_p3_m0_strip/sanitize.py b/plugins/spike_p3_m0_strip/sanitize.py index 848549b28f2e4..cd8a06d65a2f0 100644 --- a/plugins/spike_p3_m0_strip/sanitize.py +++ b/plugins/spike_p3_m0_strip/sanitize.py @@ -1,7 +1,7 @@ """Presentation-boundary sanitizer(spike · mode="final" 实装 · 其他 skeleton). -**向后兼容基线**:Hermes_AI `hermes_devices/ops/audit.py::strip_sources` -(charter C4 · fixture SHA-256 双仓锁死) +**向后兼容基线(有明确增强 · 见"差异说明")**: +- Hermes_AI `hermes_devices/ops/audit.py::strip_sources`(charter C4 · fixture SHA-256 双仓锁死) **Codex §VI 契约**: - 只 strip 明确的 `[source|来源: ...]` / `【来源:...】` / 独立行 `source:` `来源:` @@ -10,6 +10,21 @@ - 未闭合标记不吞下一行 - **fail-closed** on sanitizer exception:返回安全占位 · **不**返回原文 +**与 Hermes_AI baseline 的差异(Codex 第三轮 §VI · 明文锁定)**: + +fork sanitizer 相对 baseline 做了两处**用户体验增强**,由 differential test 锁死: + +| 差异 | Hermes_AI baseline | fork sanitizer | 用户可见效果 | +|---|---|---|---| +| `_INLINE_SOURCE_RE` 前导 | `[\[【（(]...`(不吃前导空白) | `\s?[\[【（(]...`(吃 1 个前导空白) | `"Answer [source:x]." → baseline: "Answer ." · fork: "Answer."` | +| 结果末尾清理 | `.strip()`(掐首尾空白) | `.rstrip()`(仅掐尾) | 保留首行 indentation(如 `" Answer"`)· 只去尾空 | + +**契约锁定**: +- baseline strip 的每个 source marker · fork 也必须 strip(**覆盖包含关系**) +- baseline 不 strip 的(negative case)· fork 也不 strip(**负例一致**) +- 差异**仅**在于两点 UX 增强(前导空白、尾空白)· 由 + `tests/test_strip_differential_vs_hermes_ai.py` 一一断言 + **冻结 · pending 出口**: - mode="streaming":skeleton pass-through(pending · S1 完整缓冲 · M1+) - mode="history":skeleton pass-through(pending · presentation boundary · M1+) @@ -20,9 +35,8 @@ import re -# ── 向后兼容 Hermes_AI ops/audit.py::_INLINE_RE ────────────────────────── -# 匹配 [source: ...] / [来源: ...] / 【来源:...】 / (source: ...) 等 · -# 允许一层嵌套中文/半角括号(如 [来源: FAQ(2026版)]). +# ── 与 Hermes_AI ops/audit.py::_INLINE_RE 差 1 个前导 `\s?` ───────────── +# body class 完全一致 · 允许一层嵌套中文/半角括号(如 [来源: FAQ(2026版)]). _INLINE_SOURCE_RE = re.compile( r"\s?[\[【（(]\s*(?:source|来源)\s*[:：]" r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" @@ -30,13 +44,13 @@ re.IGNORECASE, ) -# ── 独立行 `source:` / `来源:` · 整行(含换行)移除 ────────────────────── +# ── 独立行 `source:` / `来源:` · 整行(含换行)移除 · 与 baseline 完全一致 ─ _LINE_SOURCE_RE = re.compile( r"^[ \t]*(?:source|来源)\s*[:：].*$", re.IGNORECASE | re.MULTILINE, ) -# ── Trailing whitespace / 连续空行清理(sub 完的后处理)────────────── +# ── Trailing whitespace / 连续空行清理(sub 完的后处理 · 与 baseline 一致)─ _TRAILING_WS_RE = re.compile(r"[ \t]+(?=\n|$)") _MULTI_NEWLINE_RE = re.compile(r"\n{3,}") @@ -57,7 +71,7 @@ def sanitize_presentation(text: str, *, mode: str) -> str: def _sanitize_final(text: str) -> str: - """出口 ① · 普通最终响应 sanitize · Hermes_AI ops/audit.py 兼容。""" + """出口 ① · 普通最终响应 sanitize · Hermes_AI ops/audit.py 兼容 + UX 增强。""" if not text: return text out = _INLINE_SOURCE_RE.sub("", text) @@ -65,5 +79,5 @@ def _sanitize_final(text: str) -> str: # 清理 sub 留下的行尾空格 · 3+ 连续空行合并为 2 out = _TRAILING_WS_RE.sub("", out) out = _MULTI_NEWLINE_RE.sub("\n\n", out) - # 清理结果末尾空行(与 Hermes_AI baseline 一致) + # rstrip 而非 strip:保留首行 indentation(见模块 docstring "差异说明") return out.rstrip() diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py new file mode 100644 index 0000000000000..5f224bf418b2f --- /dev/null +++ b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py @@ -0,0 +1,153 @@ +"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. + +**Codex 第三轮 §VI 强制**: +- fork sanitizer 与 Hermes_AI `ops/audit.py::strip_sources` 差异**必须**明文锁定 +- 差异**仅**允许:①`\s?` 前导空白吸收 · ②`.rstrip()` 保留首行 indentation +- 覆盖包含关系:baseline strip 的 · fork 必 strip +- 负例一致:baseline 保留的 · fork 必保留 +- baseline 源字符串 verbatim byte-check(飘移即失败) +""" +from __future__ import annotations + +import pytest + +from plugins.spike_p3_m0_fixtures import hermes_ai_baseline +from plugins.spike_p3_m0_fixtures.citation_cases import ALL_CASES, cases_by_marker +from plugins.spike_p3_m0_strip.sanitize import ( + _INLINE_SOURCE_RE, + _LINE_SOURCE_RE, + sanitize_presentation, +) + + +# ── 1. baseline 源字符串 verbatim byte-check ──────────────────── + + +def test_baseline_regex_verbatim(): + """Vendored baseline 与 Hermes_AI 源必须 byte-identical(source of truth 门)。 + + 这两个模式串直接从 `hermes_devices/ops/audit.py::_INLINE_RE._LINE_RE.pattern` + 抄写 · 若上游修改 · 本测试**必须**先失败提醒同步 vendor。 + """ + expected_inline = ( + r"[\[【（(]\s*(?:source|来源)\s*[:：]" + r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】）)]" + ) + expected_line = r"^[ \t]*(?:source|来源)\s*[:：].*$" + assert hermes_ai_baseline.BASELINE_INLINE_PATTERN == expected_inline, ( + "vendored baseline INLINE pattern drifted · resync from Hermes_AI " + "hermes_devices/ops/audit.py::_INLINE_RE.pattern" + ) + assert hermes_ai_baseline.BASELINE_LINE_PATTERN == expected_line, ( + "vendored baseline LINE pattern drifted · resync from Hermes_AI " + "hermes_devices/ops/audit.py::_LINE_RE.pattern" + ) + + +# ── 2. Fork inline pattern = baseline + `\s?` 前导 ──────────── + + +def test_fork_inline_pattern_is_baseline_plus_leading_space(): + """fork `_INLINE_SOURCE_RE` = `\\s?` + baseline INLINE pattern(严格锁定)。""" + assert _INLINE_SOURCE_RE.pattern == r"\s?" + hermes_ai_baseline.BASELINE_INLINE_PATTERN + + +def test_fork_line_pattern_equals_baseline(): + """fork `_LINE_SOURCE_RE` = baseline LINE pattern(**完全一致**)。""" + assert _LINE_SOURCE_RE.pattern == hermes_ai_baseline.BASELINE_LINE_PATTERN + + +# ── 3. 覆盖包含关系:baseline strip 的 · fork 必 strip ───────── + + +_SOURCE_MARKER_LEFT_BRACKETS = ("[source", "[来源", "[SOURCE", "[Source", + "【source", "【来源", "（source", "(source", + "（来源", "(来源") + + +def _has_source_marker(text: str) -> bool: + """Detect any remaining `source:` / `来源:` inline marker(differential coverage).""" + lowered = text.lower() + for prefix in ("[source", "[来源", "【source", "【来源", "（source", "(source", + "（来源", "(来源"): + if prefix.lower() in lowered: + return True + return False + + +@pytest.mark.parametrize( + "case", + [c for c in ALL_CASES if c.mode == "final" and c.marker_type != "negative"], + ids=lambda c: c.name, +) +def test_fork_strips_everything_baseline_strips(case): + """Positive case:baseline 移除的每个 marker · fork 也必移除(**覆盖包含关系**)。 + + 契约锁定的两点断言(§VI): + A. 两侧输出**均无** source marker 残留(覆盖完整性) + B. 两侧输出去除**所有空白**后 byte-identical(差异仅在空白 · UX 增强范围内) + """ + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + + # A. 覆盖完整性(仅对 baseline 实际 strip 的 case 生效) + # 未闭合标记(§VI:不吞下一行)· baseline 与 fork 均保留 · 跳过此断言 + if baseline_out != case.raw: + assert not _has_source_marker(baseline_out), ( + f"case={case.name} · baseline 输出仍含 source marker · fixture 缺失或 baseline drift · " + f"baseline={baseline_out!r}" + ) + assert not _has_source_marker(fork_out), ( + f"case={case.name} · fork 输出仍含 source marker · 覆盖不完整 · fork={fork_out!r}" + ) + + # B. 差异仅在空白(核心保证:内容 byte-identical after \s+ 剥离) + import re as _re + baseline_nowhite = _re.sub(r"\s+", "", baseline_out) + fork_nowhite = _re.sub(r"\s+", "", fork_out) + assert baseline_nowhite == fork_nowhite, ( + f"case={case.name} · fork 与 baseline 在去除空白后仍不 byte-identical\n" + f" raw: {case.raw!r}\n" + f" baseline: {baseline_out!r}\n" + f" fork: {fork_out!r}\n" + f" baseline_nowhite: {baseline_nowhite!r}\n" + f" fork_nowhite: {fork_nowhite!r}\n" + "差异必须限于 §VI 声明的两点(前导空白、首尾空白)· " + "否则更新 sanitize.py 或 fixture · 双仓 PR body 同步" + ) + + +# ── 4. Negative case 一致性:baseline 不 strip · fork 也不 strip ─ + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_identical(case): + """Negative case:baseline 与 fork 都必须原样保留(strict equality)。""" + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + assert baseline_out == case.raw, ( + f"baseline unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} baseline={baseline_out!r}" + ) + assert fork_out == case.raw, ( + f"fork unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} fork={fork_out!r}" + ) + + +# ── 5. UX 差异边界样例(明文示例断言) ───────────────────────── + + +def test_ux_diff_inline_leading_space(): + """UX 差异 ①:`\\s?` 前导空白吸收 · 单空格样例。""" + raw = "Answer is X [source: doc]." + assert hermes_ai_baseline.strip_sources(raw) == "Answer is X ." + assert sanitize_presentation(raw, mode="final") == "Answer is X." + + +def test_ux_diff_preserves_leading_indent(): + """UX 差异 ②:`.rstrip()` 保留首行 indentation。""" + raw = " Answer with [source: x]" + assert hermes_ai_baseline.strip_sources(raw) == "Answer with" # baseline .strip() + assert sanitize_presentation(raw, mode="final") == " Answer with" # fork .rstrip() diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_final.py b/plugins/spike_p3_m0_strip/tests/test_strip_final.py index 11feb80376785..b97d323b2b182 100644 --- a/plugins/spike_p3_m0_strip/tests/test_strip_final.py +++ b/plugins/spike_p3_m0_strip/tests/test_strip_final.py @@ -26,8 +26,10 @@ enumerate_byte_split_chunks, ) from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + assert_no_plugin_module_leak, get_discovered_module, install_fresh_manager, + snapshot_plugin_modules, write_config, ) @@ -37,23 +39,48 @@ _FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" # 与 Hermes_AI `tests/test_p3_m0_fixture_gate.py::EXPECTED_SHA256` 完全一致 +# **规范 LF 字节** SHA-256(Git normalized · 见 fork .gitattributes 声明) EXPECTED_FIXTURE_SHA256 = ( "47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666" ) -# ── Fixture SHA-256 校验(跨仓一致性 门槛) ───────────────────── +def _canonical_lf_bytes(path: Path) -> bytes: + """Read file bytes with CRLF/CR line endings normalized to LF. + + Cross-repo SHA gate contract locks the **canonical LF bytes** (not disk + bytes) so Windows autocrlf checkout does not produce a false SHA mismatch. + Paired with fork `.gitattributes text eol=lf` rule for this file. + """ + raw = path.read_bytes() + return raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +# ── Fixture SHA-256 校验(跨仓一致性 门槛 · 规范 LF 字节) ───── def test_fixture_sha256_matches_hermes_ai(): fixture_path = _FIXTURE_DIR / "citation_cases.py" - actual = hashlib.sha256(fixture_path.read_bytes()).hexdigest() + actual = hashlib.sha256(_canonical_lf_bytes(fixture_path)).hexdigest() assert actual == EXPECTED_FIXTURE_SHA256, ( - f"fixture SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " + f"fixture canonical LF SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " f"actual: {actual}\n跨仓 fixture 不一致 · 需双仓同步更新" ) +def test_fixture_sha256_crlf_input_normalizes(): + """CRLF checkout(Windows autocrlf 模拟)仍匹配 SHA gate · 契约锁 LF 字节。""" + fixture_path = _FIXTURE_DIR / "citation_cases.py" + lf_bytes = fixture_path.read_bytes() + if b"\r\n" in lf_bytes: + pytest.skip("checkout 本就是 CRLF · 直接由 test_fixture_sha256_matches_hermes_ai 覆盖") + crlf_bytes = lf_bytes.replace(b"\n", b"\r\n") + normalized = crlf_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + assert hashlib.sha256(normalized).hexdigest() == EXPECTED_FIXTURE_SHA256, ( + "CRLF → LF 规范化后 SHA 与冻结值不符" + ) + + # ── Discovery / registration(§VIII) ──────────────────────────── @@ -71,6 +98,55 @@ def test_discover_registers_transform_llm_output_hook(tmp_path, monkeypatch): assert has_hook("transform_llm_output") is True +def test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue(tmp_path): + """Codex 第三轮 §VI:证明 fixture teardown 后 sys.modules 完全恢复(names + object ids)。 + + 比 name-only 快照更强的契约:每个 hermes_plugins.* 键的 module OBJECT 在 teardown + 后必须还是**同一实例**(delitem+monkeypatch.undo 恢复原对象)。若 discover 新建的 + module 对象泄漏 · id() 会不同 · 直接暴露污染。 + """ + import sys + import pytest as _pytest + + before_names = snapshot_plugin_modules() + before_ids = {k: id(sys.modules[k]) for k in before_names} + + mp = _pytest.MonkeyPatch() + finalizers = [] + + class _FakeRequest: + def addfinalizer(self, fn): + finalizers.append(fn) + + try: + write_config(tmp_path, [STRIP_KEY]) + install_fresh_manager(mp, tmp_path, request=_FakeRequest()) + # 验证 discover 确实加载了(要么新键 · 要么替换了 object id) + during_names = snapshot_plugin_modules() + during_ids = {k: id(sys.modules[k]) for k in during_names} + discover_loaded = (during_names - before_names) or { + k for k in before_names & during_names if during_ids.get(k) != before_ids.get(k) + } + assert discover_loaded, "install_fresh_manager 未加载 · 也未替换任何 hermes_plugins.*" + finally: + # 手工 teardown:先 finalizers · 再 monkeypatch undo + for fn in reversed(finalizers): + fn() + mp.undo() + + # 快照完全恢复:names 与 object ids 都必须与 before 一致 + after_names = snapshot_plugin_modules() + assert after_names == before_names, ( + f"name leak · new keys: {sorted(after_names - before_names)} · " + f"missing keys: {sorted(before_names - after_names)}" + ) + for k in before_names: + assert id(sys.modules[k]) == before_ids[k], ( + f"hermes_plugins module {k!r} object identity changed after teardown · " + "discover-inserted new object leaked past finalizer + monkeypatch.undo" + ) + + def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): from hermes_cli.plugins import has_hook From 8c19102ccf597fab3825e65c0c0aa77036e27cb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 19:47:48 +0000 Subject: [PATCH 04/19] =?UTF-8?q?spike(p3-m0):=20third-round=20=C2=A77=20s?= =?UTF-8?q?trip=20differential=20=E8=A1=A5=E8=AF=81=20=C2=B7=20fix=20`\s?= =?UTF-8?q?=3F`=20=E2=86=92=20`[=20\t]=3F`=20newline=20swallow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex third-round §7 identified a real bug in the fork's UX-enhanced strip regex: the leading `\s?` matches `\n` (Python `\s` includes newlines), so `"Line A\n[source: k]Next word"` collapsed into `"Line ANext word"` — the newline separating two lines was silently eaten by the marker match. Fix: - sanitize.py `_INLINE_SOURCE_RE`: `\s?` → `[ \t]?` (horizontal whitespace only) - Docstring updated to reflect the tighter constraint and add the newline safety proof to the diff table. - Differential test pattern-lock: `test_fork_inline_pattern_is_baseline_plus_leading_horizontal_space` now asserts `[ \t]?` rather than the previous `\s?`. New negative-proof tests (8) locking Codex §7 requirements: 1. test_leading_space_does_not_eat_newline_before_marker — `"Line A\n[source: k]Next"` → `"Line A\nNext"` (no line concatenation) 2. test_leading_space_does_not_eat_double_newline_before_marker — paragraph separator preserved 3. test_leading_space_only_absorbs_single_space_or_tab — parametrized: `" "` and `"\t"` absorbed; `"\n"` and `"\r"` preserved 4. test_whitespace_diff_does_not_widen_source_deletion_range — non-source `[normal]` bracket preserved next to `[source: k]` 5. test_trailing_whitespace_rstrip_scope_matches_baseline_content — trailing spaces / newlines both stripped, content byte-identical 6. test_leading_whitespace_only_diff_between_baseline_and_fork — leading indent preserved by fork, stripped by baseline; content matches 7. test_middle_line_source_marker_preserves_surrounding_newlines — `"Line 1\nLine 2 [source: k]\nLine 3"` → both `\n` preserved 8. test_marker_at_start_of_line_preserves_prev_line_terminator — with and without leading horizontal whitespace variants Also switched two docstrings to raw-string form (r"...") to silence Python 3.13 DeprecationWarning on `\s` / `\[` escapes. Test results (scripts/run_tests.sh, 8 workers): test_strip_differential_vs_hermes_ai.py — 35 passed (was 27, +8 §7 tests) test_strip_final.py — 65 passed test_budget_reserve.py — 28 passed test_cache_middleware.py — 35 passed Total: 163 passed / 0 failed in 3.6s The `\s?` → `[ \t]?` change is UX-neutral for existing fixture cases (no fixture case had `\n` immediately before a source marker) but removes the latent line-concatenation bug against real streaming/multiline provider output. Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01FiKVo54oX4Uv2qksAHjYsZ --- plugins/spike_p3_m0_strip/sanitize.py | 18 ++- .../test_strip_differential_vs_hermes_ai.py | 130 +++++++++++++++++- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/plugins/spike_p3_m0_strip/sanitize.py b/plugins/spike_p3_m0_strip/sanitize.py index cd8a06d65a2f0..dac512ba06912 100644 --- a/plugins/spike_p3_m0_strip/sanitize.py +++ b/plugins/spike_p3_m0_strip/sanitize.py @@ -1,4 +1,4 @@ -"""Presentation-boundary sanitizer(spike · mode="final" 实装 · 其他 skeleton). +r"""Presentation-boundary sanitizer(spike · mode="final" 实装 · 其他 skeleton). **向后兼容基线(有明确增强 · 见"差异说明")**: - Hermes_AI `hermes_devices/ops/audit.py::strip_sources`(charter C4 · fixture SHA-256 双仓锁死) @@ -10,19 +10,24 @@ - 未闭合标记不吞下一行 - **fail-closed** on sanitizer exception:返回安全占位 · **不**返回原文 -**与 Hermes_AI baseline 的差异(Codex 第三轮 §VI · 明文锁定)**: +**与 Hermes_AI baseline 的差异(Codex 第三轮 §VI · §7 明文锁定)**: fork sanitizer 相对 baseline 做了两处**用户体验增强**,由 differential test 锁死: | 差异 | Hermes_AI baseline | fork sanitizer | 用户可见效果 | |---|---|---|---| -| `_INLINE_SOURCE_RE` 前导 | `[\[【（(]...`(不吃前导空白) | `\s?[\[【（(]...`(吃 1 个前导空白) | `"Answer [source:x]." → baseline: "Answer ." · fork: "Answer."` | +| `_INLINE_SOURCE_RE` 前导 | `[\[【（(]...`(不吃前导空白) | `[ \t]?[\[【（(]...`(吃 1 个前导**水平**空白) | `"Answer [source:x]." → baseline: "Answer ." · fork: "Answer."` | | 结果末尾清理 | `.strip()`(掐首尾空白) | `.rstrip()`(仅掐尾) | 保留首行 indentation(如 `" Answer"`)· 只去尾空 | +**关键约束(Codex 第三轮 §7 补证)**: +- 前导消耗**只**取 `[ \t]`(空格 / tab)· **不**用 `\s?`(否则会吞 `\n` 拼接下一行) +- 若 `Line A\n[source: k]Next` · 期望 `Line A\nNext`(fork 与 baseline 均保留换行 · 无行拼接) +- 空白差异**不扩大** source 删除范围:非 source 括号 `[normal]` 一律保留 + **契约锁定**: - baseline strip 的每个 source marker · fork 也必须 strip(**覆盖包含关系**) - baseline 不 strip 的(negative case)· fork 也不 strip(**负例一致**) -- 差异**仅**在于两点 UX 增强(前导空白、尾空白)· 由 +- 差异**仅**在于两点 UX 增强(前导**水平**空白、首行 indent 保留 vs 全 strip)· 由 `tests/test_strip_differential_vs_hermes_ai.py` 一一断言 **冻结 · pending 出口**: @@ -35,10 +40,11 @@ import re -# ── 与 Hermes_AI ops/audit.py::_INLINE_RE 差 1 个前导 `\s?` ───────────── +# ── 与 Hermes_AI ops/audit.py::_INLINE_RE 差 1 个前导 `[ \t]?` ────────── +# 前导用 `[ \t]?` 而非 `\s?` · 避免吞掉 `\n` 拼接下一行(Codex §7 反例证明)。 # body class 完全一致 · 允许一层嵌套中文/半角括号(如 [来源: FAQ(2026版)]). _INLINE_SOURCE_RE = re.compile( - r"\s?[\[【（(]\s*(?:source|来源)\s*[:：]" + r"[ \t]?[\[【（(]\s*(?:source|来源)\s*[:：]" r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" r"[\]】）)]", re.IGNORECASE, diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py index 5f224bf418b2f..42c91411f507b 100644 --- a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py +++ b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py @@ -1,4 +1,4 @@ -"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. +r"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. **Codex 第三轮 §VI 强制**: - fork sanitizer 与 Hermes_AI `ops/audit.py::strip_sources` 差异**必须**明文锁定 @@ -45,12 +45,17 @@ def test_baseline_regex_verbatim(): ) -# ── 2. Fork inline pattern = baseline + `\s?` 前导 ──────────── +# ── 2. Fork inline pattern = baseline + `[ \t]?` 前导(**只水平空白**) ─ -def test_fork_inline_pattern_is_baseline_plus_leading_space(): - """fork `_INLINE_SOURCE_RE` = `\\s?` + baseline INLINE pattern(严格锁定)。""" - assert _INLINE_SOURCE_RE.pattern == r"\s?" + hermes_ai_baseline.BASELINE_INLINE_PATTERN +def test_fork_inline_pattern_is_baseline_plus_leading_horizontal_space(): + """fork `_INLINE_SOURCE_RE` = `[ \\t]?` + baseline INLINE pattern(严格锁定)。 + + Codex §7 反例证明:前导消耗必须是 `[ \\t]?`(空格 / tab)· **不**能是 `\\s?` + (`\\s` 包含 `\\n` · 会把行末换行吞掉,拼接下一行 · 见 + `test_leading_space_does_not_eat_newline_before_marker`)。 + """ + assert _INLINE_SOURCE_RE.pattern == r"[ \t]?" + hermes_ai_baseline.BASELINE_INLINE_PATTERN def test_fork_line_pattern_equals_baseline(): @@ -151,3 +156,118 @@ def test_ux_diff_preserves_leading_indent(): raw = " Answer with [source: x]" assert hermes_ai_baseline.strip_sources(raw) == "Answer with" # baseline .strip() assert sanitize_presentation(raw, mode="final") == " Answer with" # fork .rstrip() + + +# ── 6. Codex §7 反例证明:前导差异不吞换行、不扩大删除范围 ───── + + +def test_leading_space_does_not_eat_newline_before_marker(): + """前一行的 `\\n` **不得**被前导消耗吞掉 · 否则会拼接下一行(§7 反例)。 + + 这是当前实施 `[ \\t]?` 而非 `\\s?` 的核心理由。曾用的 `\\s?` 匹配 `\\n` · + `"Line A\\n[source: kb]Next"` → `"Line ANext"`(BUG)· 现改为 `[ \\t]?` · + 结果应保持换行结构:`"Line A\\nNext"`。 + """ + raw = "Line A\n[source: kb]Next word" + fork_out = sanitize_presentation(raw, mode="final") + # baseline 不吃前导 · 直接删 marker · 结果保持换行 · 后接 Next word + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 关键断言:换行未被吞 · 无行拼接 + assert "\n" in fork_out, ( + f"fork 前导消耗吞掉了 `\\n` · 拼接了下一行 · got={fork_out!r} · " + "前导 regex 必须是 `[ \\t]?` 不是 `\\s?`" + ) + assert fork_out == "Line A\nNext word", ( + f"fork 处理跨行 marker 错 · raw={raw!r} got={fork_out!r} expected='Line A\\nNext word'" + ) + # 同时 · fork 与 baseline 保持一致(此 case 前导不消耗 · 空白差异不体现) + assert fork_out == baseline_out + + +def test_leading_space_does_not_eat_double_newline_before_marker(): + """双换行边界同理 · 前导消耗**不吃** `\\n` · 段落分隔保留。""" + raw = "Paragraph A ends here.\n\n[source: kb]Paragraph B starts." + fork_out = sanitize_presentation(raw, mode="final") + # 双 \n 应保留(不吞任何 \n · 段落分隔完整) + assert "\n\n" in fork_out, ( + f"fork 吞掉了段落分隔 · got={fork_out!r}" + ) + assert fork_out == "Paragraph A ends here.\n\nParagraph B starts." + + +def test_leading_space_only_absorbs_single_space_or_tab(): + """前导 `[ \\t]?` 只吸收 1 个空格 / tab · 不吸收其他 whitespace。""" + # 空格 · 吸收 + assert sanitize_presentation("X [source: k]Y", mode="final") == "XY" + # tab · 吸收 + assert sanitize_presentation("X\t[source: k]Y", mode="final") == "XY" + # 换行 · 不吸收(§7 硬约束) + assert sanitize_presentation("X\n[source: k]Y", mode="final") == "X\nY" + # 回车 · 不吸收(避免 Windows 场景意外拼接) + assert sanitize_presentation("X\r[source: k]Y", mode="final") == "X\rY" + + +def test_whitespace_diff_does_not_widen_source_deletion_range(): + """空白差异**不**扩大 source 删除范围:非 source 括号一律保留。 + + Codex §7:确保 `\\s?` → `[ \\t]?` 与 `.rstrip()` 的两处 UX 增强· + 不会让 fork 意外删掉 baseline 保留的内容。 + """ + # 非 source 括号(`[normal]`)紧跟 source marker · 只 source 被删 + raw = "Result: [normal] [source: kb] end." + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + assert "[normal]" in fork_out, f"fork 误删了 [normal] · got={fork_out!r}" + assert "[normal]" in baseline_out + # source marker 已删 + assert "[source" not in fork_out.lower() + # 两侧内容(去空白)一致 + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) + + +def test_trailing_whitespace_rstrip_scope_matches_baseline_content(): + """`.rstrip()` 只掐尾空白 · 不改变文本主体 · 与 baseline `.strip()` 内容一致(仅首空白 diff)。""" + raw = "Answer here [source: k]\n\n " + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 两侧 trailing 皆掐 · 主体一致 + assert fork_out == "Answer here" + assert baseline_out == "Answer here" + + +def test_leading_whitespace_only_diff_between_baseline_and_fork(): + """首行 indentation 场景 · fork 保留 · baseline 掐 · 主体 byte-identical。""" + raw = " Answer [source: k]" + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # fork 保留首 3 空格 · baseline 全 strip + assert fork_out == " Answer" + assert baseline_out == "Answer" + # 去空白后主体 byte-identical + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) == "Answer" + + +def test_middle_line_source_marker_preserves_surrounding_newlines(): + """中间行的 inline source marker · 两侧换行完整保留 · 无行拼接。""" + raw = "Line 1\nLine 2 [source: k]\nLine 3" + fork_out = sanitize_presentation(raw, mode="final") + assert fork_out == "Line 1\nLine 2\nLine 3" + + +def test_marker_at_start_of_line_preserves_prev_line_terminator(): + """行首 marker 场景 · 前一行的 `\\n` 保留 · 只删 marker + 前导水平空白(若有)。""" + raw_no_lead = "Line 1\n[source: k]Line 2 content" + raw_with_lead = "Line 1\n \t[source: k]Line 2 content" + + out_no_lead = sanitize_presentation(raw_no_lead, mode="final") + out_with_lead = sanitize_presentation(raw_with_lead, mode="final") + + # 无 leading horizontal ws · 结果 "Line 1\nLine 2 content" + assert out_no_lead == "Line 1\nLine 2 content" + # 有 leading horizontal ws · marker 前导吸收 1 个 · 剩下 " \t"? + # 实际:`[ \t]?[source: k]` 匹配 `\t[source: k]` · 前面的 " " 保留 · 结果 "Line 1\n Line 2 content" + # 关键点:换行未被吞 · 有 \n + assert "\n" in out_with_lead + assert out_with_lead == "Line 1\n Line 2 content" From 3d7cb5497d555e2119384d069ad34926902fda21 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 04:45:44 +0800 Subject: [PATCH 05/19] fix(spike): fail closed before provider execution --- plugins/spike_p3_m0_cache/__init__.py | 1097 ++++++++++++++----------- 1 file changed, 611 insertions(+), 486 deletions(-) diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index 52c2d37780c14..a2f218a1a7b12 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -1,535 +1,660 @@ -"""P3-M0 Spike · tenant-safe cache + orchestrator(llm_execution middleware). - -**Codex 第三轮 §I / §II / §III / §V 契约(修订)**: - -1. 注册 `llm_execution` middleware(`ctx.register_middleware("llm_execution", ...)`)· - 经过 `run_llm_execution_middleware`(`hermes_cli/middleware.py:187`)真正生产走位· - fail-open on callback(fork middleware 语义 · 见 middleware.py:_run_execution_chain) - -2. **控制流严格解耦 · cache eligibility ≠ budget enforcement**: - - 插件禁用 → next_call(不 lookup 不 reserve) - 插件启用 → - ├─ 硬依赖门:budget module 缺失 → fail-CLOSED(结构化 error response · 不 next_call) - ├─ tenant context 4 字段严格校验(缺/空/非白/超长)→ fail-CLOSED - ├─ budget config 校验(BudgetConfigError → fail-CLOSED · 不静默 None) - ├─ 非 chat_completions api_mode → fail-CLOSED(M0 明确缩小支持范围 · 见 §II) - ├─ 非 dict request → fail-CLOSED - │ - ├─ 判 cacheable_request(streaming/tools 非 cacheable · **但仍 reserve/settle**) - │ - ├─ cacheable_request → cache lookup - │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) - │ - ├─ (MISS 或 非 cacheable_request)→ budget.reserve - │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) - │ └─ 成功 → next_call - │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) - │ └─ 异常 → budget.release + re-raise - -3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / - profile_version 任一缺失/空/非法 → **不 next_call** · 返回结构化 fail-closed response - · **禁**默认填 "system"/"0" 等安全值(Codex §III) - -4. **只**支持 `api_mode == "chat_completions"`:其他 3 mode(anthropic_messages / - bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· - M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 - -5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` - -6. 并发:threading.RLock 保护 LRU + insert 序 - -7. 配置: - - `plugins.spike_p3_m0_cache.enabled` **严格 bool**(v is True · 拒 "false"/1/等) - - `plugins.spike_p3_m0_cache.tenant.*`(4 字段 · str · 白名单 · len≤128) - - `plugins.spike_p3_m0_cache.tenant.daily_budget_tokens`(4 态 · BudgetConfigError 硬 fail-CLOSED) - - **不用** SPIKE_* env · **不**读 profile.yaml · **不**读 HERMES_DAILY_TOKEN_BUDGET - -8. **input token 估算**(Codex §V):不得静默 0 后宣称预算护栏成立。 - - 优先信 ctx.approx_input_tokens(若 int/float > 0) - - 否则从 request.messages content 字符数 / 4(粗略英文 heuristic · M0 approximation · 见文档) - - 加上 request.max_tokens 上限 · 得 tokens_needed -""" -from __future__ import annotations - -import hashlib -import importlib -import json -import logging -import re -import sys -import threading -from collections import OrderedDict -from typing import Any, Callable, Dict, Optional - +"""P3-M0 Spike · tenant-safe cache + orchestrator(llm_execution middleware). + +**Codex 第三轮 §I / §II / §III / §V 契约(修订)**: + +1. 注册 `llm_execution` middleware(`ctx.register_middleware("llm_execution", ...)`)· + 经过 `run_llm_execution_middleware`(`hermes_cli/middleware.py:187`)真正生产走位· + fail-open on callback(fork middleware 语义 · 见 middleware.py:_run_execution_chain) + +2. **控制流严格解耦 · cache eligibility ≠ budget enforcement**: + + 插件禁用 → next_call(不 lookup 不 reserve) + 插件启用 → + ├─ 硬依赖门:budget module 缺失 → fail-CLOSED(结构化 error response · 不 next_call) + ├─ tenant context 4 字段严格校验(缺/空/非白/超长)→ fail-CLOSED + ├─ budget config 校验(BudgetConfigError → fail-CLOSED · 不静默 None) + ├─ 非 chat_completions api_mode → fail-CLOSED(M0 明确缩小支持范围 · 见 §II) + ├─ 非 dict request → fail-CLOSED + │ + ├─ 判 cacheable_request(streaming/tools 非 cacheable · **但仍 reserve/settle**) + │ + ├─ cacheable_request → cache lookup + │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) + │ + ├─ (MISS 或 非 cacheable_request)→ budget.reserve + │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) + │ └─ 成功 → next_call + │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) + │ └─ 异常 → budget.release + re-raise + +3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / + profile_version 任一缺失/空/非法 → **不 next_call** · 返回结构化 fail-closed response + · **禁**默认填 "system"/"0" 等安全值(Codex §III) + +4. **只**支持 `api_mode == "chat_completions"`:其他 3 mode(anthropic_messages / + bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· + M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 + +5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` + +6. 并发:threading.RLock 保护 LRU + insert 序 + +7. 配置: + - `plugins.spike_p3_m0_cache.enabled` **严格 bool**(v is True · 拒 "false"/1/等) + - `plugins.spike_p3_m0_cache.tenant.*`(4 字段 · str · 白名单 · len≤128) + - `plugins.spike_p3_m0_cache.tenant.daily_budget_tokens`(4 态 · BudgetConfigError 硬 fail-CLOSED) + - **不用** SPIKE_* env · **不**读 profile.yaml · **不**读 HERMES_DAILY_TOKEN_BUDGET + +8. **input token 估算**(Codex §V):不得静默 0 后宣称预算护栏成立。 + - 优先信 ctx.approx_input_tokens(若 int/float > 0) + - 否则从 request.messages content 字符数 / 4(粗略英文 heuristic · M0 approximation · 见文档) + - 加上 request.max_tokens 上限 · 得 tokens_needed +""" +from __future__ import annotations + +import hashlib +import importlib +import json +import logging +import re +import sys +import threading +from collections import OrderedDict +from typing import Any, Callable, Dict, Optional + logger = logging.getLogger(__name__) -def _budget_module(): - """Locate the budget sibling module regardless of parent package. - - hermes_cli.plugins loads bundled plugins under ``hermes_plugins.*``; running - the plugins directly from a checkout may see ``plugins.*`` instead. Lazy - lookup avoids a hard ``from .. import`` that only works under one layout. +class ConfigLoadError(RuntimeError): + """The runtime config could not be read safely.""" + + +class ConfigShapeError(RuntimeError): + """The runtime config has an invalid shape for this plugin.""" + + +def _budget_module(): + """Locate the budget sibling module regardless of parent package. + + hermes_cli.plugins loads bundled plugins under ``hermes_plugins.*``; running + the plugins directly from a checkout may see ``plugins.*`` instead. Lazy + lookup avoids a hard ``from .. import`` that only works under one layout. + + **Codex §III**:budget 模块缺失时 middleware **不**得 fail-open · 由调用方(orchestrator) + 捕获 ImportError 走 fail-CLOSED response · 保持"cache 单独启用"场景可测。 + """ + parent = __name__.rsplit(".", 1)[0] if "." in __name__ else "" + if parent: + candidate = f"{parent}.spike_p3_m0_budget" + if candidate in sys.modules: + return sys.modules[candidate] + try: + return importlib.import_module(candidate) + except ImportError: + pass + raise ImportError( + "spike_p3_m0_budget module not loaded; enable spike-p3-m0-budget in plugins.enabled" + ) + + +CACHE_CONTRACT_VERSION = "v1" +CODEC_VERSION = "codec_v1" +CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent + +# 允许 tenant_id/principal_id/permission_scope_version/profile_version 的字符白名单 +# (类比 Hermes_AI `_validate_tenant_id` · 保守集合) +_ID_ALLOWED_RE = re.compile(r"^[a-zA-Z0-9_\-]+$") +_ID_MAX_LEN = 128 + +# fail-closed 结构化 response 消息(非空 · 可展示 · finish_reason=content_filter 抑制 retry) +_BUDGET_EXCEEDED_MESSAGE = ( + "[请求已被拒绝 · tenant 每日预算已达上限 · 请稍后重试或联系管理员]" +) +_BUDGET_FAIL_CLOSED_MESSAGE = ( + "[请求已被拒绝 · 预算护栏未就绪 · 请联系管理员]" +) + + +# ── Module state · thread-safe LRU ──────────────────────────────────── + +_lock = threading.RLock() +_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + + +# ── Config loading(fork config.yaml · 不用 env) ───────────────────── + +def _load_config() -> Dict[str, Any]: + """Load fork config via ``hermes_cli.config.load_config``. - **Codex §III**:budget 模块缺失时 middleware **不**得 fail-open · 由调用方(orchestrator) - 捕获 ImportError 走 fail-CLOSED response · 保持"cache 单独启用"场景可测。 + Once the middleware has been registered, a config read failure is a + security failure, not evidence that the plugin is disabled. Returning an + empty mapping here would make ``_is_enabled`` false and silently bypass + budget enforcement through ``next_call``. """ - parent = __name__.rsplit(".", 1)[0] if "." in __name__ else "" - if parent: - candidate = f"{parent}.spike_p3_m0_budget" - if candidate in sys.modules: - return sys.modules[candidate] - try: - return importlib.import_module(candidate) - except ImportError: - pass - raise ImportError( - "spike_p3_m0_budget module not loaded; enable spike-p3-m0-budget in plugins.enabled" - ) - - -CACHE_CONTRACT_VERSION = "v1" -CODEC_VERSION = "codec_v1" -CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent - -# 允许 tenant_id/principal_id/permission_scope_version/profile_version 的字符白名单 -# (类比 Hermes_AI `_validate_tenant_id` · 保守集合) -_ID_ALLOWED_RE = re.compile(r"^[a-zA-Z0-9_\-]+$") -_ID_MAX_LEN = 128 - -# fail-closed 结构化 response 消息(非空 · 可展示 · finish_reason=content_filter 抑制 retry) -_BUDGET_EXCEEDED_MESSAGE = ( - "[请求已被拒绝 · tenant 每日预算已达上限 · 请稍后重试或联系管理员]" -) -_BUDGET_FAIL_CLOSED_MESSAGE = ( - "[请求已被拒绝 · 预算护栏未就绪 · 请联系管理员]" -) - - -# ── Module state · thread-safe LRU ──────────────────────────────────── - -_lock = threading.RLock() -_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() - - -# ── Config loading(fork config.yaml · 不用 env) ───────────────────── - -def _load_config() -> Dict[str, Any]: - """Load fork config via `hermes_cli.config.load_config` · returns full config dict.""" try: from hermes_cli.config import load_config - return load_config() or {} + cfg = load_config() except Exception as exc: # noqa: BLE001 - logger.warning("spike-cache: load_config failed · treated as empty: %s", exc) + raise ConfigLoadError("spike cache config could not be loaded") from exc + if cfg is None: return {} - - + if not isinstance(cfg, dict): + raise ConfigShapeError("root config must be a mapping") + return cfg + + def _get_plugin_config() -> Dict[str, Any]: - """Return `plugins.spike_p3_m0_cache` sub-config(dict · empty if未配置).""" + """Return the validated ``plugins.spike_p3_m0_cache`` mapping.""" cfg = _load_config() - plugins_cfg = cfg.get("plugins") or {} - return plugins_cfg.get("spike_p3_m0_cache") or {} - - -def _is_enabled() -> bool: - """`plugins.spike_p3_m0_cache.enabled` **严格 bool** · 只接受 literal `True`。 - - Codex §III 硬约束:拒 `bool("false") == True`(str 转 bool 陷阱)· 拒 int/list/dict。 - True → 启用;其他(False / None / "true" / 1 / [] / {} / ...)→ 视 as 禁用。 - """ - return _get_plugin_config().get("enabled") is True - - -def _get_tenant_context() -> Optional[Dict[str, str]]: - """严格 fail-closed tenant context · 从 config `plugins.spike_p3_m0_cache.tenant` 读。 - - 所有 4 字段(tenant_id/principal_id/permission_scope_version/profile_version)必需· - 非空 · 类型 str · 字符白名单 · 任一违反 → 返回 None(fail-closed · log context_invalid). - - **禁**默认填 "system"/"0" 等安全值(Codex §III 明令)。 - - T2 首选:由 provisioning 写入该 tenant 独立 HERMES_HOME 的 config.yaml · fork - 进程仅信 config · 不信 prompt/tool_args。 - """ - tenant_block = _get_plugin_config().get("tenant") or {} - if not isinstance(tenant_block, dict): - logger.info("spike-cache: tenant context invalid · not a mapping") - return None - required = ("tenant_id", "principal_id", "permission_scope_version", "profile_version") - resolved: Dict[str, str] = {} - for field in required: - v = tenant_block.get(field) - if v is None: - logger.info("spike-cache: tenant context invalid · missing field %s", field) - return None - if not isinstance(v, str): - logger.info("spike-cache: tenant context invalid · field %s type=%s(need str)", field, type(v).__name__) - return None - if not v.strip(): - logger.info("spike-cache: tenant context invalid · field %s empty/whitespace", field) - return None - if len(v) > _ID_MAX_LEN: - logger.info("spike-cache: tenant context invalid · field %s len=%d > %d", field, len(v), _ID_MAX_LEN) - return None - if not _ID_ALLOWED_RE.match(v): - logger.info("spike-cache: tenant context invalid · field %s non-whitelist chars", field) - return None - resolved[field] = v - return resolved - - -# ── Cacheability guards ───────────────────────────────────────────── - -def _api_mode_supported(api_mode: str) -> bool: - """M0 只支持 `chat_completions`(见 §II)· 其他 mode fail-CLOSED · 不 next_call。""" - return api_mode == "chat_completions" - - -def is_cacheable_request(request: Any, *, api_mode: str = "chat_completions") -> bool: - """判 request 是否可 lookup/insert cache(**与 budget 决策解耦**)。 - - Codex §II:streaming / tools 可以不缓存 · 但**不能不计预算**。 - """ - if not _api_mode_supported(api_mode): - return False - if not isinstance(request, dict): - return False - if request.get("stream"): - return False - tools = request.get("tools") or [] - if tools: - return False - return True - - -def is_cacheable_response(response: Any) -> bool: - """仅 finish=stop + 无 tool_calls · 支持 SDK 对象和 dict 形态。""" - try: - if isinstance(response, dict): - choices = response.get("choices") - else: - choices = getattr(response, "choices", None) - if not choices: - return False - first = choices[0] - msg = first.get("message") if isinstance(first, dict) else getattr(first, "message", None) - if not msg: - return False - tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) - if tool_calls: - return False - finish = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None) - return finish == "stop" - except Exception: # noqa: BLE001 - return False - - -# ── Cache key builder(§2 schema v1)────────────────────────────── - -def build_cache_key( - request: Dict[str, Any], - tenant_ctx: Dict[str, str], - *, - model: str, - provider: str, -) -> str: - """SHA-256 of deterministic key components(12 字段 schema v1).""" - messages = request.get("messages") or [] - messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") - tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") - # system prompt hash · 从 messages 首 system 消息内容 - sys_msg_content = "" - for m in messages: - if isinstance(m, dict) and m.get("role") == "system": - sys_msg_content = json.dumps(m.get("content"), sort_keys=True, ensure_ascii=False) - break - parts = { - "tenant_id": tenant_ctx["tenant_id"], - "principal_id": tenant_ctx["principal_id"], - "permission_scope_version": tenant_ctx["permission_scope_version"], - "provider": provider or "", - "model": model or "", - "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], - "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], - "profile_version": tenant_ctx["profile_version"], - "knowledge_collection": str(request.get("_kb_collection") or ""), - "knowledge_collection_version": str(request.get("_kb_version") or ""), - "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), - "cache_contract_version": CACHE_CONTRACT_VERSION, - } - canonical = json.dumps(parts, sort_keys=True, ensure_ascii=False).encode("utf-8") - return hashlib.sha256(canonical).hexdigest() - - -# ── Response归一化 · codec_v1(仅 chat_completions dict 形态)────── - -def _extract_cacheable_response(response: Any) -> Dict[str, Any]: - """Normalize provider response → cacheable dict(codec_v1)· 只 chat_completions dict.""" - def _get(obj, k, default=None): - return obj.get(k, default) if isinstance(obj, dict) else getattr(obj, k, default) - origin_usage_obj = _get(response, "usage") or {} - if not isinstance(origin_usage_obj, dict): - origin_usage_obj = { - "prompt_tokens": getattr(origin_usage_obj, "prompt_tokens", 0) or 0, - "completion_tokens": getattr(origin_usage_obj, "completion_tokens", 0) or 0, - } - choices = _get(response, "choices") - return { - "model": _get(response, "model"), - "choices": choices, - "usage": {"prompt_tokens": 0, "completion_tokens": 0}, # 顶层 usage = billable_usage · 全 0 - "codec_version": CODEC_VERSION, - "cache_meta": { - "hit": True, - "cache_contract_version": CACHE_CONTRACT_VERSION, - "origin_usage": origin_usage_obj, - "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, - "saved_usage": origin_usage_obj, - "cache_lookup_cost": None, # in-memory · 通常 0 - }, - } - - -# ── Input token 估算(Codex §V:不得静默 0) ────────────────────── - -def _estimate_input_tokens_from_messages(request: Dict[str, Any]) -> int: - """从 messages content 字符数估算 input tokens · **M0 approximation**。 - - heuristic:总字符数 ÷ 4(rough 英文 char-per-token)· 中文/emoji 会低估 · - 但对 M0 预算护栏"防明显超支"目标足够 · 生产 M1+ 需接 tiktoken/tokenizer 精算 - (out-of-band 契约字段 · 见 docs/fork/budget-hook.md §5b)。 - - 保底:若 messages 全空/无 content · 返回 0 · 由调用方结合 max_tokens 判断。 - """ - if not isinstance(request, dict): - return 0 - msgs = request.get("messages") or [] - if not isinstance(msgs, list): - return 0 - total_chars = 0 - for m in msgs: - if not isinstance(m, dict): - continue - content = m.get("content") - if isinstance(content, str): - total_chars += len(content) - elif isinstance(content, list): - for part in content: - if isinstance(part, dict): - t = part.get("text") - if isinstance(t, str): - total_chars += len(t) - return total_chars // 4 - - -def _estimate_needed_tokens(request: Any, ctx: Dict[str, Any]) -> int: - """总预留 = input 估算 + max_tokens · **不静默 0**(Codex §V)。 - - 优先信 ctx.approx_input_tokens(若 int/float > 0);否则从 messages 估算 · - 加上 request.max_tokens(或 ctx.max_tokens)· 得 tokens_needed。 - """ - explicit = ctx.get("approx_input_tokens") - if isinstance(explicit, bool): - explicit = None - if isinstance(explicit, (int, float)) and explicit > 0: - approx_input = int(explicit) - else: - approx_input = _estimate_input_tokens_from_messages(request if isinstance(request, dict) else {}) - max_tokens_raw = None - if isinstance(request, dict): - max_tokens_raw = request.get("max_tokens") - if max_tokens_raw is None: - max_tokens_raw = ctx.get("max_tokens") - try: - max_tokens = int(max_tokens_raw or 0) - except (TypeError, ValueError): - max_tokens = 0 - if max_tokens < 0: - max_tokens = 0 - return approx_input + max_tokens - - -# ── Structured error responses(Codex §V:非空、安全、可展示) ───── - -def _fail_closed_response(model: str, *, reason_code: str, **extras: Any) -> Dict[str, Any]: - """结构化 fail-closed response · finish_reason=content_filter 抑制 retry/fallback。 - - Codex §V:BudgetExceeded / 预算护栏未就绪 时返回必须**非空** · 安全可展示 · - 不触发 retry。上层 orchestrator 若追加 retry 决策 · 需读 `cache_meta.reason_code` - 做过滤(M1+ 契约:orchestrator 遇 content_filter + cache_meta.reason_code 明确 · 不 retry)。 - """ - return { - "model": model or "", - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": _BUDGET_EXCEEDED_MESSAGE if reason_code == "budget_exceeded" else _BUDGET_FAIL_CLOSED_MESSAGE, - }, - "finish_reason": "content_filter", - }], - "usage": {"prompt_tokens": 0, "completion_tokens": 0}, - "cache_meta": { - "hit": False, - "budget_exceeded": reason_code == "budget_exceeded", - "budget_fail_closed": reason_code != "budget_exceeded", - "reason_code": reason_code, - **extras, - }, - } - - -# ── Orchestrator middleware(cache + budget 显式组合 · 解耦) ───── - + plugins_cfg = cfg.get("plugins") + if plugins_cfg is None: + return {} + if not isinstance(plugins_cfg, dict): + raise ConfigShapeError("plugins config must be a mapping") + plugin_cfg = plugins_cfg.get("spike_p3_m0_cache") + if plugin_cfg is None: + return {} + if not isinstance(plugin_cfg, dict): + raise ConfigShapeError("spike_p3_m0_cache config must be a mapping") + return plugin_cfg + + +def _is_enabled(plugin_cfg: Optional[Dict[str, Any]] = None) -> bool: + """`plugins.spike_p3_m0_cache.enabled` **严格 bool** · 只接受 literal `True`。 + + Codex §III 硬约束:拒 `bool("false") == True`(str 转 bool 陷阱)· 拒 int/list/dict。 + True → 启用;其他(False / None / "true" / 1 / [] / {} / ...)→ 视 as 禁用。 + """ + if plugin_cfg is None: + plugin_cfg = _get_plugin_config() + return plugin_cfg.get("enabled") is True + + +def _get_tenant_context( + plugin_cfg: Optional[Dict[str, Any]] = None, +) -> Optional[Dict[str, str]]: + """严格 fail-closed tenant context · 从 config `plugins.spike_p3_m0_cache.tenant` 读。 + + 所有 4 字段(tenant_id/principal_id/permission_scope_version/profile_version)必需· + 非空 · 类型 str · 字符白名单 · 任一违反 → 返回 None(fail-closed · log context_invalid). + + **禁**默认填 "system"/"0" 等安全值(Codex §III 明令)。 + + T2 首选:由 provisioning 写入该 tenant 独立 HERMES_HOME 的 config.yaml · fork + 进程仅信 config · 不信 prompt/tool_args。 + """ + if plugin_cfg is None: + plugin_cfg = _get_plugin_config() + tenant_block = plugin_cfg.get("tenant") or {} + if not isinstance(tenant_block, dict): + logger.info("spike-cache: tenant context invalid · not a mapping") + return None + required = ("tenant_id", "principal_id", "permission_scope_version", "profile_version") + resolved: Dict[str, str] = {} + for field in required: + v = tenant_block.get(field) + if v is None: + logger.info("spike-cache: tenant context invalid · missing field %s", field) + return None + if not isinstance(v, str): + logger.info("spike-cache: tenant context invalid · field %s type=%s(need str)", field, type(v).__name__) + return None + if not v.strip(): + logger.info("spike-cache: tenant context invalid · field %s empty/whitespace", field) + return None + if len(v) > _ID_MAX_LEN: + logger.info("spike-cache: tenant context invalid · field %s len=%d > %d", field, len(v), _ID_MAX_LEN) + return None + if not _ID_ALLOWED_RE.match(v): + logger.info("spike-cache: tenant context invalid · field %s non-whitelist chars", field) + return None + resolved[field] = v + return resolved + + +# ── Cacheability guards ───────────────────────────────────────────── + +def _api_mode_supported(api_mode: str) -> bool: + """M0 只支持 `chat_completions`(见 §II)· 其他 mode fail-CLOSED · 不 next_call。""" + return api_mode == "chat_completions" + + +def is_cacheable_request(request: Any, *, api_mode: str = "chat_completions") -> bool: + """判 request 是否可 lookup/insert cache(**与 budget 决策解耦**)。 + + Codex §II:streaming / tools 可以不缓存 · 但**不能不计预算**。 + """ + if not _api_mode_supported(api_mode): + return False + if not isinstance(request, dict): + return False + if request.get("stream"): + return False + tools = request.get("tools") or [] + if tools: + return False + return True + + +def is_cacheable_response(response: Any) -> bool: + """仅 finish=stop + 无 tool_calls · 支持 SDK 对象和 dict 形态。""" + try: + if isinstance(response, dict): + choices = response.get("choices") + else: + choices = getattr(response, "choices", None) + if not choices: + return False + first = choices[0] + msg = first.get("message") if isinstance(first, dict) else getattr(first, "message", None) + if not msg: + return False + tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + if tool_calls: + return False + finish = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None) + return finish == "stop" + except Exception: # noqa: BLE001 + return False + + +# ── Cache key builder(§2 schema v1)────────────────────────────── + +def build_cache_key( + request: Dict[str, Any], + tenant_ctx: Dict[str, str], + *, + model: str, + provider: str, +) -> str: + """SHA-256 of deterministic key components(12 字段 schema v1).""" + messages = request.get("messages") or [] + messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") + tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") + # system prompt hash · 从 messages 首 system 消息内容 + sys_msg_content = "" + for m in messages: + if isinstance(m, dict) and m.get("role") == "system": + sys_msg_content = json.dumps(m.get("content"), sort_keys=True, ensure_ascii=False) + break + parts = { + "tenant_id": tenant_ctx["tenant_id"], + "principal_id": tenant_ctx["principal_id"], + "permission_scope_version": tenant_ctx["permission_scope_version"], + "provider": provider or "", + "model": model or "", + "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], + "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], + "profile_version": tenant_ctx["profile_version"], + "knowledge_collection": str(request.get("_kb_collection") or ""), + "knowledge_collection_version": str(request.get("_kb_version") or ""), + "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), + "cache_contract_version": CACHE_CONTRACT_VERSION, + } + canonical = json.dumps(parts, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +# ── Response归一化 · codec_v1(仅 chat_completions dict 形态)────── + +def _extract_cacheable_response(response: Any) -> Dict[str, Any]: + """Normalize provider response → cacheable dict(codec_v1)· 只 chat_completions dict.""" + def _get(obj, k, default=None): + return obj.get(k, default) if isinstance(obj, dict) else getattr(obj, k, default) + origin_usage_obj = _get(response, "usage") or {} + if not isinstance(origin_usage_obj, dict): + origin_usage_obj = { + "prompt_tokens": getattr(origin_usage_obj, "prompt_tokens", 0) or 0, + "completion_tokens": getattr(origin_usage_obj, "completion_tokens", 0) or 0, + } + choices = _get(response, "choices") + return { + "model": _get(response, "model"), + "choices": choices, + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, # 顶层 usage = billable_usage · 全 0 + "codec_version": CODEC_VERSION, + "cache_meta": { + "hit": True, + "cache_contract_version": CACHE_CONTRACT_VERSION, + "origin_usage": origin_usage_obj, + "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "saved_usage": origin_usage_obj, + "cache_lookup_cost": None, # in-memory · 通常 0 + }, + } + + +# ── Input token 估算(Codex §V:不得静默 0) ────────────────────── + +def _estimate_input_tokens_from_messages(request: Dict[str, Any]) -> int: + """从 messages content 字符数估算 input tokens · **M0 approximation**。 + + heuristic:总字符数 ÷ 4(rough 英文 char-per-token)· 中文/emoji 会低估 · + 但对 M0 预算护栏"防明显超支"目标足够 · 生产 M1+ 需接 tiktoken/tokenizer 精算 + (out-of-band 契约字段 · 见 docs/fork/budget-hook.md §5b)。 + + 保底:若 messages 全空/无 content · 返回 0 · 由调用方结合 max_tokens 判断。 + """ + if not isinstance(request, dict): + return 0 + msgs = request.get("messages") or [] + if not isinstance(msgs, list): + return 0 + total_chars = 0 + for m in msgs: + if not isinstance(m, dict): + continue + content = m.get("content") + if isinstance(content, str): + total_chars += len(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict): + t = part.get("text") + if isinstance(t, str): + total_chars += len(t) + return total_chars // 4 + + +def _estimate_needed_tokens(request: Any, ctx: Dict[str, Any]) -> int: + """总预留 = input 估算 + max_tokens · **不静默 0**(Codex §V)。 + + 优先信 ctx.approx_input_tokens(若 int/float > 0);否则从 messages 估算 · + 加上 request.max_tokens(或 ctx.max_tokens)· 得 tokens_needed。 + """ + explicit = ctx.get("approx_input_tokens") + if isinstance(explicit, bool): + explicit = None + if isinstance(explicit, (int, float)) and explicit > 0: + approx_input = int(explicit) + else: + approx_input = _estimate_input_tokens_from_messages(request if isinstance(request, dict) else {}) + max_tokens_raw = None + if isinstance(request, dict): + max_tokens_raw = request.get("max_tokens") + if max_tokens_raw is None: + max_tokens_raw = ctx.get("max_tokens") + try: + max_tokens = int(max_tokens_raw or 0) + except (TypeError, ValueError): + max_tokens = 0 + if max_tokens < 0: + max_tokens = 0 + return approx_input + max_tokens + + +# ── Structured error responses(Codex §V:非空、安全、可展示) ───── + +def _fail_closed_response(model: str, *, reason_code: str, **extras: Any) -> Dict[str, Any]: + """结构化 fail-closed response · finish_reason=content_filter 抑制 retry/fallback。 + + Codex §V:BudgetExceeded / 预算护栏未就绪 时返回必须**非空** · 安全可展示 · + 不触发 retry。上层 orchestrator 若追加 retry 决策 · 需读 `cache_meta.reason_code` + 做过滤(M1+ 契约:orchestrator 遇 content_filter + cache_meta.reason_code 明确 · 不 retry)。 + """ + return { + "model": model or "", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": _BUDGET_EXCEEDED_MESSAGE if reason_code == "budget_exceeded" else _BUDGET_FAIL_CLOSED_MESSAGE, + }, + "finish_reason": "content_filter", + }], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "cache_meta": { + "hit": False, + "budget_exceeded": reason_code == "budget_exceeded", + "budget_fail_closed": reason_code != "budget_exceeded", + "reason_code": reason_code, + **extras, + }, + } + + +# ── Orchestrator middleware(cache + budget 显式组合 · 解耦) ───── + def cache_and_budget_middleware( - request: Any, - next_call: Callable[[Any], Any], - **ctx: Any, -) -> Any: - """`llm_execution` middleware · 严格 Codex 第三轮 §II 控制流(解耦 cache 与 budget)。 - - 见模块 docstring §2 完整控制流。 - """ - api_request_id = str(ctx.get("api_request_id") or "") - api_mode = str(ctx.get("api_mode") or "chat_completions") - provider = str(ctx.get("provider") or "") - model = str(ctx.get("model") or "") + request: Any, + next_call: Callable[[Any], Any], + **ctx: Any, +) -> Any: + """`llm_execution` middleware · 严格 Codex 第三轮 §II 控制流(解耦 cache 与 budget)。 + + 见模块 docstring §2 完整控制流。 + """ + api_request_id = str(ctx.get("api_request_id") or "") + api_mode = str(ctx.get("api_mode") or "chat_completions") + provider = str(ctx.get("provider") or "") + model = str(ctx.get("model") or "") + + # 1. Resolve config exactly once. A registered middleware whose config + # cannot be read/validated must not be mistaken for a disabled plugin: + # the outer execution chain is intentionally fail-open for ordinary + # plugins, so leaking an exception here would call the provider directly. + try: + plugin_cfg = _get_plugin_config() + except ConfigLoadError: + logger.error("spike-cache: config load failed · fail-CLOSED") + return _fail_closed_response(model, reason_code="config_load_failed") + except ConfigShapeError: + logger.error("spike-cache: config shape invalid · fail-CLOSED") + return _fail_closed_response(model, reason_code="config_shape_invalid") + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: unexpected config resolution failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="config_internal_error") - # 1. Plugin disabled → next_call 直通(Codex §II step 1) - if not _is_enabled(): + # 2. Only an explicitly disabled plugin may pass through. Missing/false + # ``enabled`` is a deliberate off switch; read/shape failures above are + # never converted into this branch. + if not _is_enabled(plugin_cfg): return next_call(request) - # 2. Budget module 硬依赖门(Codex §III:cache 单独启用不得因 budget 缺失 fail-open) + # 3. Budget module 硬依赖门(Codex §III:cache 单独启用不得因 budget 缺失 fail-open) try: budget_mod = _budget_module() except ImportError as exc: - logger.error( - "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED: %s", - exc, + logger.error( + "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED: %s", + exc, ) return _fail_closed_response(model, reason_code="budget_module_missing") - - # 3. api_mode 支持范围收紧(Codex §II:非 chat_completions fail-CLOSED · 不静默 pass-through) - if not _api_mode_supported(api_mode): - logger.warning( - "spike-cache: unsupported api_mode=%s · M0 only supports chat_completions · fail-CLOSED", - api_mode, + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: budget module resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, ) - return _fail_closed_response(model, reason_code="unsupported_api_mode", api_mode=api_mode) - - # 4. request 类型门(non-dict 无法 estimate token · fail-CLOSED) - if not isinstance(request, dict): - logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) - return _fail_closed_response(model, reason_code="non_dict_request") - - # 5. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) - tenant_ctx = _get_tenant_context() + return _fail_closed_response(model, reason_code="budget_module_error") + + # 4. api_mode 支持范围收紧(Codex §II:非 chat_completions fail-CLOSED · 不静默 pass-through) + if not _api_mode_supported(api_mode): + logger.warning( + "spike-cache: unsupported api_mode=%s · M0 only supports chat_completions · fail-CLOSED", + api_mode, + ) + return _fail_closed_response(model, reason_code="unsupported_api_mode", api_mode=api_mode) + + # 5. request 类型门(non-dict 无法 estimate token · fail-CLOSED) + if not isinstance(request, dict): + logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) + return _fail_closed_response(model, reason_code="non_dict_request") + + # 6. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) + try: + tenant_ctx = _get_tenant_context(plugin_cfg) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: tenant context resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="tenant_context_error") if tenant_ctx is None: return _fail_closed_response(model, reason_code="tenant_context_invalid") tenant_id = tenant_ctx["tenant_id"] - # 6. Budget config 校验 · BudgetConfigError 硬 fail-CLOSED(Codex §III:不转 None 静默继续) - raw_budget = (_get_plugin_config().get("tenant") or {}).get("daily_budget_tokens") + # 7. Budget config 校验 · BudgetConfigError 硬 fail-CLOSED(Codex §III:不转 None 静默继续) + tenant_cfg = plugin_cfg.get("tenant") + # A valid tenant context guarantees this is a mapping. + raw_budget = tenant_cfg.get("daily_budget_tokens") try: budget_value = budget_mod.validate_daily_budget(raw_budget) except budget_mod.BudgetConfigError as exc: logger.error("spike-cache: daily_budget_tokens config invalid · fail-CLOSED: %s", exc) return _fail_closed_response(model, reason_code="budget_config_invalid") + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: budget config validation failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="budget_config_error") + + # 8. api_request_id 门(reserve 需唯一 id · 空则无法 settle) + if not api_request_id: + logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") + return _fail_closed_response(model, reason_code="missing_api_request_id") + + # 9. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) + try: + cacheable_req = is_cacheable_request(request, api_mode=api_mode) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: request classification failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="request_classification_error") - # 7. api_request_id 门(reserve 需唯一 id · 空则无法 settle) - if not api_request_id: - logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") - return _fail_closed_response(model, reason_code="missing_api_request_id") - - # 8. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) - cacheable_req = is_cacheable_request(request, api_mode=api_mode) - - # 9. 若可缓存 → lookup · HIT 直接返回(不 reserve) + # 10. 若可缓存 → lookup · HIT 直接返回(不 reserve). Any internal + # pre-provider failure is converted locally to fail-CLOSED so it cannot + # escape into the outer middleware chain's fail-open policy. cache_key: Optional[str] = None if cacheable_req: - cache_key = build_cache_key(request, tenant_ctx, model=model, provider=provider) - with _lock: - hit = _cache.get(cache_key) - if hit is not None: - _cache.move_to_end(cache_key) + try: + cache_key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + with _lock: + hit = _cache.get(cache_key) + if hit is not None: + _cache.move_to_end(cache_key) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: cache lookup/key failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="cache_lookup_error") if hit is not None: - # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate - return dict(hit) - - # 10. MISS 或非 cacheable · 都必须 reserve(Codex §II step 6) - tokens_needed = _estimate_needed_tokens(request, ctx) + # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate + return dict(hit) + + # 11. MISS 或非 cacheable · 都必须 reserve(Codex §II step 6) try: - budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) - except budget_mod.BudgetExceeded: - logger.warning( - "spike-cache: budget exceeded · tenant=%s api_request_id=%s tokens_needed=%s budget=%s", - tenant_id, api_request_id, tokens_needed, budget_value, + tokens_needed = _estimate_needed_tokens(request, ctx) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: token estimation failed · type=%s · fail-CLOSED", + type(exc).__name__, ) - return _fail_closed_response( - model, - reason_code="budget_exceeded", - tenant_id=tenant_id, - tokens_needed=tokens_needed, + return _fail_closed_response(model, reason_code="token_estimation_error") + try: + budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) + except budget_mod.BudgetExceeded: + logger.warning( + "spike-cache: budget exceeded · tenant=%s api_request_id=%s tokens_needed=%s budget=%s", + tenant_id, api_request_id, tokens_needed, budget_value, + ) + return _fail_closed_response( + model, + reason_code="budget_exceeded", + tenant_id=tenant_id, + tokens_needed=tokens_needed, budget=budget_value, ) + except Exception as exc: # noqa: BLE001 + # Defensive cleanup in case a future reserve implementation mutates + # state before raising. Never expose exception text to the user. + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass + logger.error( + "spike-cache: budget reserve internal failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="budget_reserve_error") - # 11. next_call · success settle actual / exception release + # 12. next_call · success settle actual / exception release. This narrow + # try block deliberately preserves downstream exception identity. try: response = next_call(request) - except Exception: - budget_mod.release(tenant_id, api_request_id) - raise - - # 12. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) + except Exception: + budget_mod.release(tenant_id, api_request_id) + raise + + # 13. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) actual = _extract_actual_usage(response) try: budget_mod.settle(tenant_id, api_request_id, actual) - except budget_mod.BudgetSettleError as exc: - # 理论不该发生(reserve 刚成功)· 记 audit · 不阻断 response + except Exception as exc: # noqa: BLE001 + # Provider cost has already been incurred. Do not throw into the + # outer fail-open chain after next_call; clean pending state best + # effort, record only the exception type, and preserve the response. + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass logger.error( - "spike-cache: settle raised BudgetSettleError post-reserve · api_request_id=%s: %s", - api_request_id, exc, + "spike-cache: budget settle internal failure · api_request_id=%s type=%s", + api_request_id, type(exc).__name__, ) - # 13. 只有 req + resp 皆 cacheable 才 insert + # 14. 只有 req + resp 皆 cacheable 才 insert. Cache bookkeeping is + # best-effort after a paid provider response and must never replace it. if cacheable_req and cache_key is not None and is_cacheable_response(response): - cached = _extract_cacheable_response(response) - with _lock: - _cache[cache_key] = cached - _cache.move_to_end(cache_key) - while len(_cache) > CACHE_MAX_SIZE: - _cache.popitem(last=False) - - return response - - -def _extract_actual_usage(response: Any) -> int: - """从 response.usage 抽 prompt+completion tokens.""" - try: - if isinstance(response, dict): - usage = response.get("usage") or {} - else: - usage = getattr(response, "usage", None) or {} - if isinstance(usage, dict): - return int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) - return int(getattr(usage, "prompt_tokens", 0) or 0) + int(getattr(usage, "completion_tokens", 0) or 0) - except Exception: # noqa: BLE001 - return 0 - - -def clear_cache() -> None: - """测试用 · 清空 LRU + reset budget bucket。""" - with _lock: - _cache.clear() - - -def cache_size() -> int: - with _lock: - return len(_cache) - - -def register(ctx: Any) -> None: - """PluginContext register · 挂 llm_execution middleware(orchestrator).""" - ctx.register_middleware("llm_execution", cache_and_budget_middleware) + try: + cached = _extract_cacheable_response(response) + with _lock: + _cache[cache_key] = cached + _cache.move_to_end(cache_key) + while len(_cache) > CACHE_MAX_SIZE: + _cache.popitem(last=False) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: cache insert failed after provider response · type=%s", + type(exc).__name__, + ) + + return response + + +def _extract_actual_usage(response: Any) -> int: + """从 response.usage 抽 prompt+completion tokens.""" + try: + if isinstance(response, dict): + usage = response.get("usage") or {} + else: + usage = getattr(response, "usage", None) or {} + if isinstance(usage, dict): + return int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) + return int(getattr(usage, "prompt_tokens", 0) or 0) + int(getattr(usage, "completion_tokens", 0) or 0) + except Exception: # noqa: BLE001 + return 0 + + +def clear_cache() -> None: + """测试用 · 清空 LRU + reset budget bucket。""" + with _lock: + _cache.clear() + + +def cache_size() -> int: + with _lock: + return len(_cache) + + +def register(ctx: Any) -> None: + """PluginContext register · 挂 llm_execution middleware(orchestrator).""" + ctx.register_middleware("llm_execution", cache_and_budget_middleware) + \ No newline at end of file From 3f44c8368007b68606d8d2bbbb5694fdf52dc455 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 04:45:50 +0800 Subject: [PATCH 06/19] test(spike): cover fail-closed enforcement edges --- .../tests/test_cache_middleware.py | 1244 +++++++++-------- 1 file changed, 699 insertions(+), 545 deletions(-) diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index 6f2c39bc3c38c..9042f18323855 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -1,604 +1,758 @@ -"""P3-M0 Spike · cache + budget orchestrator middleware 测试(**真实生产走位**). - -**Codex 第三轮 §I / §II / §III / §V / §VIII 契约**: -- 通过临时 `config.yaml` 启用插件 · 走真实 `PluginManager.discover_and_load` -- `has_middleware("llm_execution") == True` -- 通过 `run_llm_execution_middleware` 触发 · **不**直接调 `cache_and_budget_middleware` -- **不用** `SPIKE_*` env(fork AGENTS.md red-line 3)· **不污染**全局 manager -- **cache eligibility ≠ budget enforcement 严格解耦**: - * streaming / tools 非 cacheable · **但仍 reserve/settle** - * 非 chat_completions api_mode → fail-CLOSED(next_call == 0)· M0 缩小支持范围 - * budget 模块缺失 / tenant context 缺失 / budget config 非法 → fail-CLOSED · next_call == 0 -- cache miss + cacheable req → reserve/settle 各 1 次 -- cache hit → 不 reserve · 顶层 usage=0 -- provider exception → release 且原样传播 -- 严格 bool `enabled`(bool("false")==True 陷阱阻断) -""" -from __future__ import annotations - -import pytest - -from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( - get_discovered_module, - install_fresh_manager, - make_run_llm_execution, - write_config, -) - - -CACHE_KEY = "spike-p3-m0-cache" -BUDGET_KEY = "spike-p3-m0-budget" - - -# ── shared valid tenant block ───────────────────────────────────────── - -_VALID_TENANT = { - "tenant_id": "t_alpha", - "principal_id": "p_alpha", - "permission_scope_version": "1", - "profile_version": "1", -} - - -def _enable_cache( - tmp_path, - monkeypatch, - *, - daily_budget=None, - tenant=_VALID_TENANT, - enabled=True, - include_budget=True, -): - tenant_cfg = dict(tenant) - if daily_budget is not None: - tenant_cfg["daily_budget_tokens"] = daily_budget - enabled_list = [CACHE_KEY] - if include_budget: - enabled_list.append(BUDGET_KEY) - write_config( - tmp_path, - enabled_list, - spike_p3_m0_cache={ - "enabled": enabled, - "tenant": tenant_cfg, - }, - ) - manager = install_fresh_manager(monkeypatch, tmp_path) - cache_mod = get_discovered_module(manager, CACHE_KEY) - cache_mod.clear_cache() - budget_mod = None - if include_budget: - budget_mod = get_discovered_module(manager, BUDGET_KEY) - budget_mod.reset_state() - return manager, cache_mod, budget_mod - - -def _is_fail_closed(resp): - """Structured fail-closed response 特征:cache_meta 含 budget_exceeded 或 budget_fail_closed。""" - if not isinstance(resp, dict): - return False - meta = resp.get("cache_meta") or {} - return bool(meta.get("budget_exceeded") or meta.get("budget_fail_closed")) - - -# ── Registration invariants(§I 强要求) ───────────────────────────── - - -def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): - from hermes_cli.plugins import has_middleware - - manager, _, _ = _enable_cache(tmp_path, monkeypatch) - assert has_middleware("llm_execution") is True - assert len(manager._middleware.get("llm_execution", [])) == 1 - - -def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): - from hermes_cli.plugins import has_middleware - - write_config(tmp_path, []) - install_fresh_manager(monkeypatch, tmp_path) - assert has_middleware("llm_execution") is False - - -def test_enabled_bool_strict_true_only(tmp_path, monkeypatch): - """Codex §III:`enabled` 严格 bool True · 拒 str/int/其他(bool('false')==True 陷阱)。""" - # enabled = "false"(str)· 应视 as 禁用 · 直通 next_call - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=100, enabled="false", - ) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=500, - ) - resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) - # 未启用 → 直通 next · 未 reserve - assert counters["next"] == 1 - assert not _is_fail_closed(resp) - - -@pytest.mark.parametrize("enabled_value", [1, "true", "yes", [1], {"x": 1}]) -def test_enabled_non_bool_treated_as_disabled(tmp_path, monkeypatch, enabled_value): - """任何非 literal-True 值(int / str / list / dict)· 视 as 禁用 · 直通 next。""" - _, _, _ = _enable_cache( - tmp_path, monkeypatch, daily_budget=100, enabled=enabled_value, - ) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - ) - resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) - assert counters["next"] == 1 - assert not _is_fail_closed(resp) - - -# ── Cache miss(cacheable request) → reserve+settle 各 1 次 ───── - - -def test_miss_reserves_and_settles_once(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=10_000 - ) - - call_count = {"next": 0} - - def next_call(_req): - call_count["next"] += 1 - return { - "model": "gpt-4o", - "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 100, "completion_tokens": 50}, - } - - run = make_run_llm_execution( - api_request_id="req_miss_1", - provider="openai", - model="gpt-4o", - api_mode="chat_completions", - approx_input_tokens=200, - ) - resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) - - assert call_count["next"] == 1 - assert resp["choices"][0]["message"]["content"] == "hi" - # settle 累加 actual usage = 150 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 150 - assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 - +"""P3-M0 Spike · cache + budget orchestrator middleware 测试(**真实生产走位**). + +**Codex 第三轮 §I / §II / §III / §V / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 走真实 `PluginManager.discover_and_load` +- `has_middleware("llm_execution") == True` +- 通过 `run_llm_execution_middleware` 触发 · **不**直接调 `cache_and_budget_middleware` +- **不用** `SPIKE_*` env(fork AGENTS.md red-line 3)· **不污染**全局 manager +- **cache eligibility ≠ budget enforcement 严格解耦**: + * streaming / tools 非 cacheable · **但仍 reserve/settle** + * 非 chat_completions api_mode → fail-CLOSED(next_call == 0)· M0 缩小支持范围 + * budget 模块缺失 / tenant context 缺失 / budget config 非法 → fail-CLOSED · next_call == 0 +- cache miss + cacheable req → reserve/settle 各 1 次 +- cache hit → 不 reserve · 顶层 usage=0 +- provider exception → release 且原样传播 +- 严格 bool `enabled`(bool("false")==True 陷阱阻断) +""" +from __future__ import annotations + +import pytest + +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + make_run_llm_execution, + write_config, +) + + +CACHE_KEY = "spike-p3-m0-cache" +BUDGET_KEY = "spike-p3-m0-budget" + + +# ── shared valid tenant block ───────────────────────────────────────── + +_VALID_TENANT = { + "tenant_id": "t_alpha", + "principal_id": "p_alpha", + "permission_scope_version": "1", + "profile_version": "1", +} + + +def _enable_cache( + tmp_path, + monkeypatch, + *, + daily_budget=None, + tenant=_VALID_TENANT, + enabled=True, + include_budget=True, +): + tenant_cfg = dict(tenant) + if daily_budget is not None: + tenant_cfg["daily_budget_tokens"] = daily_budget + enabled_list = [CACHE_KEY] + if include_budget: + enabled_list.append(BUDGET_KEY) + write_config( + tmp_path, + enabled_list, + spike_p3_m0_cache={ + "enabled": enabled, + "tenant": tenant_cfg, + }, + ) + manager = install_fresh_manager(monkeypatch, tmp_path) + cache_mod = get_discovered_module(manager, CACHE_KEY) + cache_mod.clear_cache() + budget_mod = None + if include_budget: + budget_mod = get_discovered_module(manager, BUDGET_KEY) + budget_mod.reset_state() + return manager, cache_mod, budget_mod + + +def _is_fail_closed(resp): + """Structured fail-closed response 特征:cache_meta 含 budget_exceeded 或 budget_fail_closed。""" + if not isinstance(resp, dict): + return False + meta = resp.get("cache_meta") or {} + return bool(meta.get("budget_exceeded") or meta.get("budget_fail_closed")) + + +# ── Registration invariants(§I 强要求) ───────────────────────────── + + +def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): + from hermes_cli.plugins import has_middleware + + manager, _, _ = _enable_cache(tmp_path, monkeypatch) + assert has_middleware("llm_execution") is True + assert len(manager._middleware.get("llm_execution", [])) == 1 + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_middleware + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_middleware("llm_execution") is False + + +def test_enabled_bool_strict_true_only(tmp_path, monkeypatch): + """Codex §III:`enabled` 严格 bool True · 拒 str/int/其他(bool('false')==True 陷阱)。""" + # enabled = "false"(str)· 应视 as 禁用 · 直通 next_call + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled="false", + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + # 未启用 → 直通 next · 未 reserve + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +@pytest.mark.parametrize("enabled_value", [1, "true", "yes", [1], {"x": 1}]) +def test_enabled_non_bool_treated_as_disabled(tmp_path, monkeypatch, enabled_value): + """任何非 literal-True 值(int / str / list / dict)· 视 as 禁用 · 直通 next。""" + _, _, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled=enabled_value, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +# ── Cache miss(cacheable request) → reserve+settle 各 1 次 ───── + + +def test_miss_reserves_and_settles_once(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + call_count = {"next": 0} + + def next_call(_req): + call_count["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 50}, + } + + run = make_run_llm_execution( + api_request_id="req_miss_1", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + approx_input_tokens=200, + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + + assert call_count["next"] == 1 + assert resp["choices"][0]["message"]["content"] == "hi" + # settle 累加 actual usage = 150 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 150 + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Cache hit → budget reserve 0 次 · 顶层 usage=0(§I 强要求) ───── + + +def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 200, "completion_tokens": 80}, + } + + run = make_run_llm_execution( + api_request_id="req_a", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + approx_input_tokens=300, + ) + req = {"messages": [{"role": "user", "content": "same"}]} + + # 首次 miss · settle 280 + run(req, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 + + # 二次 hit · next_call 0 次 · used 不变(reserve 0 次) + run2 = make_run_llm_execution( + api_request_id="req_b", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + ) + resp = run2(req, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp["cache_meta"]["hit"] is True + assert resp["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp["cache_meta"]["origin_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["saved_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["cache_contract_version"] == cache_mod.CACHE_CONTRACT_VERSION + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Budget 超支 → next_call 0 次(§I / §V 强要求 · 非空结构化响应) ─ + + +def test_budget_exceeded_blocks_next_call_returns_safe_response(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100 + ) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"model": "m", "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="req_over", provider="openai", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, + next_call, + ) + # 超支明确 short-circuit · next 未调 + assert counters["next"] == 0 + # 非空结构化 error response · 抑制 retry · Codex §V + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["reason_code"] == "budget_exceeded" + assert resp["cache_meta"]["tenant_id"] == _VALID_TENANT["tenant_id"] + # 非空 assistant message(可展示给用户) + assert resp["choices"][0]["message"]["role"] == "assistant" + assert resp["choices"][0]["message"]["content"] # 非空字符串 + # finish_reason 抑制 retry(content_filter) + assert resp["choices"][0]["finish_reason"] == "content_filter" + # 顶层 usage=0 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + # 无 pending reservation(reserve 抛异常 · 未记账) + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + + +def test_budget_exceeded_on_streaming_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:streaming 请求 + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_stream", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "stream": True, "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 # streaming 也被 reserve 门阻断 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_budget_exceeded_on_tools_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:tools request + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_tools", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}], "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_non_cacheable_request_still_reserves_and_settles(tmp_path, monkeypatch): + """Codex §II:streaming/tools 非 cacheable · **但仍 reserve/settle**(解耦)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "streamed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 60, "completion_tokens": 40}} + + # streaming request(不缓存 · 但必须 reserve) + run = make_run_llm_execution( + api_request_id="r_s1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 100 # settle 60+40 + # 不 cache · 二次同请求仍 miss(需 fresh api_request_id) + run2 = make_run_llm_execution( + api_request_id="r_s2", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + run2({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 2 # 再调 next + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 # 再 settle 100 + + +# ── Provider exception → reservation 释放 + 传播 ──────────────────── + + +def test_provider_exception_releases_reservation(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + class ProviderBoom(RuntimeError): + pass + + def next_call(_req): + raise ProviderBoom("upstream 500") + + run = make_run_llm_execution( + api_request_id="req_boom", provider="openai", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + with pytest.raises(ProviderBoom, match="upstream 500"): + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) + + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None + + +# ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ + + +@pytest.mark.parametrize( + "bad_tenant", + [ + {"tenant_id": "", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, + {"tenant_id": "t/../evil", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "a" * 200, "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + ], + ids=["empty-tenant", "empty-principal", "missing-profile-version", "non-whitelist-chars", "over-max-len"], +) +def test_invalid_tenant_fail_closed_blocks_next_call(tmp_path, monkeypatch, bad_tenant): + """Codex §III:enforcement 开启时 · 缺失/非法 tenant · **next_call == 0** · 结构化 fail-closed。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, tenant=bad_tenant + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + assert counters["next"] == 0 # fail-CLOSED · 严格 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" + assert resp["choices"][0]["message"]["content"] # 非空可展示 error + assert resp["choices"][0]["finish_reason"] == "content_filter" + + +def test_no_tenant_config_fail_closed(tmp_path, monkeypatch): + """config.yaml 无 tenant 段 · fail-CLOSED(不走 next_call)。""" + write_config( + tmp_path, + [CACHE_KEY, BUDGET_KEY], + spike_p3_m0_cache={"enabled": True}, # 无 tenant 段 + ) + install_fresh_manager(monkeypatch, tmp_path) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" + + +def test_non_chat_completions_mode_fail_closed(tmp_path, monkeypatch): + """Codex §II:非 chat_completions api_mode 明确 fail-CLOSED(不静默 pass-through)。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="responses", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "unsupported_api_mode" + assert resp["cache_meta"]["api_mode"] == "responses" + + +def test_non_dict_request_fail_closed(tmp_path, monkeypatch): + """非 dict request 无法 estimate token · fail-CLOSED。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return None + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run(["not", "a", "dict"], next_call) # list · 非 dict + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "non_dict_request" + + +@pytest.mark.parametrize("bad_budget", [True, False, -1, "1000", 1.5]) +def test_invalid_budget_config_fail_closed(tmp_path, monkeypatch, bad_budget): + """Codex §III:BudgetConfigError 硬 fail-CLOSED · **不**转 None 静默继续。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=bad_budget + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "budget_config_invalid" + + +def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): + """空 api_request_id · reserve 无法追踪 · fail-CLOSED。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" + + +# ── Cache alone(budget disabled)· 硬依赖门 fail-CLOSED ───────────── + + +def test_cache_alone_without_budget_fail_closed(tmp_path, monkeypatch): + """Codex §III:cache 单独启用时 budget 缺失 · **必须 fail-CLOSED** · 不 pass-through。""" + # 只启用 cache · 不启用 budget · 但 config tenant 完整 + manager, cache_mod, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=1000, include_budget=False, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + # budget module 未加载 · 硬 fail-CLOSED + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "budget_module_missing" + + +# ── Cache key schema stability(§2 v1 schema) ────────────────────── + + +def test_cache_key_stable_and_deterministic(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": [{"role": "user", "content": "hi"}]} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + assert k1 == k2 and len(k1) == 64 + + +def test_cache_key_changes_on_provider(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="m", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") + assert k1 != k2 + + +def test_cache_key_changes_on_tenant(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + a = dict(_VALID_TENANT, tenant_id="tA") + b = dict(_VALID_TENANT, tenant_id="tB") + ka = cache_mod.build_cache_key(req, a, model="m", provider="p") + kb = cache_mod.build_cache_key(req, b, model="m", provider="p") + assert ka != kb + + +# ── Tool-call response 不 insert · 下次仍 miss ──────────────────── + + +def test_tool_calls_response_not_inserted(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + + run1 = make_run_llm_execution(api_request_id="r1", provider="p", model="m", api_mode="chat_completions") + run2 = make_run_llm_execution(api_request_id="r2", provider="p", model="m", api_mode="chat_completions") + req = {"messages": [{"role": "user", "content": "call tool"}]} + run1(req, next_call) + run2(req, next_call) + assert counters["next"] == 2 + + +# ── Input token estimation (Codex §V:不静默 0) ──────────────────── + + +def test_input_tokens_estimated_from_messages_when_no_ctx(tmp_path, monkeypatch): + """无 ctx.approx_input_tokens · 从 messages char count / 4 估算 · 不静默 0。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2}} + + # 未传 approx_input_tokens · content 长约 4000 字符 → 估算 ~1000 tokens 触发超支 + long_content = "x" * 4000 + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) + # 估算 = 4000/4 + 100 = 1100 · budget=1000 · 超支 + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["tokens_needed"] >= 1000 -# ── Cache hit → budget reserve 0 次 · 顶层 usage=0(§I 强要求) ───── +# ── Registered middleware must not inherit the core fail-open policy ── -def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=10_000 - ) +def _run_with_counter(): counters = {"next": 0} def next_call(_req): counters["next"] += 1 return { - "model": "gpt-4o", - "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 200, "completion_tokens": 80}, + "choices": [{"message": {"content": "provider output"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, } run = make_run_llm_execution( - api_request_id="req_a", - provider="openai", - model="gpt-4o", + api_request_id="r_fail_closed", + provider="p", + model="m", api_mode="chat_completions", - approx_input_tokens=300, + approx_input_tokens=10, ) - req = {"messages": [{"role": "user", "content": "same"}]} - - # 首次 miss · settle 280 - run(req, next_call) - assert counters["next"] == 1 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 - - # 二次 hit · next_call 0 次 · used 不变(reserve 0 次) - run2 = make_run_llm_execution( - api_request_id="req_b", - provider="openai", - model="gpt-4o", - api_mode="chat_completions", - ) - resp = run2(req, next_call) - assert counters["next"] == 1 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 - assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} - assert resp["cache_meta"]["hit"] is True - assert resp["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} - assert resp["cache_meta"]["origin_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} - assert resp["cache_meta"]["saved_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} - assert resp["cache_meta"]["cache_contract_version"] == cache_mod.CACHE_CONTRACT_VERSION - assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 - + return counters, next_call, run -# ── Budget 超支 → next_call 0 次(§I / §V 强要求 · 非空结构化响应) ─ +def _assert_internal_failure_is_safe(resp, counters, reason_code): + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == reason_code + assert resp["choices"][0]["message"]["content"] + rendered = repr(resp) + assert "private user input" not in rendered + assert "TOP_SECRET_INTERNAL_EXCEPTION" not in rendered -def test_budget_exceeded_blocks_next_call_returns_safe_response(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=100 - ) - counters = {"next": 0} +def test_registered_plugin_config_load_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + from hermes_cli import config as config_mod - def next_call(_req): - counters["next"] += 1 - return {"model": "m", "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + def boom(): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - run = make_run_llm_execution( - api_request_id="req_over", provider="openai", model="m", api_mode="chat_completions", - approx_input_tokens=500, - ) + monkeypatch.setattr(config_mod, "load_config", boom) + counters, next_call, run = _run_with_counter() resp = run( - {"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, + {"messages": [{"role": "user", "content": "private user input"}]}, next_call, ) - # 超支明确 short-circuit · next 未调 - assert counters["next"] == 0 - # 非空结构化 error response · 抑制 retry · Codex §V - assert resp["cache_meta"]["budget_exceeded"] is True - assert resp["cache_meta"]["reason_code"] == "budget_exceeded" - assert resp["cache_meta"]["tenant_id"] == _VALID_TENANT["tenant_id"] - # 非空 assistant message(可展示给用户) - assert resp["choices"][0]["message"]["role"] == "assistant" - assert resp["choices"][0]["message"]["content"] # 非空字符串 - # finish_reason 抑制 retry(content_filter) - assert resp["choices"][0]["finish_reason"] == "content_filter" - # 顶层 usage=0 - assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} - # 无 pending reservation(reserve 抛异常 · 未记账) - assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 - - -def test_budget_exceeded_on_streaming_also_blocks_next_call(tmp_path, monkeypatch): - """Codex §V:streaming 请求 + 预算超限 · next_call 也必须 0 次(解耦证明)。""" - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=50 - ) - counters = {"next": 0} + _assert_internal_failure_is_safe(resp, counters, "config_load_failed") - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - run = make_run_llm_execution( - api_request_id="r_stream", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=500, - ) +@pytest.mark.parametrize( + "bad_config", + [ + {"plugins": []}, + {"plugins": {"spike_p3_m0_cache": []}}, + ], + ids=["plugins-not-mapping", "plugin-config-not-mapping"], +) +def test_registered_plugin_malformed_config_shape_fail_closed( + tmp_path, monkeypatch, bad_config, +): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + monkeypatch.setattr(cache_mod, "_load_config", lambda: bad_config) + counters, next_call, run = _run_with_counter() resp = run( - {"messages": [{"role": "user", "content": "x"}], "stream": True, "max_tokens": 100}, + {"messages": [{"role": "user", "content": "private user input"}]}, next_call, ) - assert counters["next"] == 0 # streaming 也被 reserve 门阻断 - assert resp["cache_meta"]["budget_exceeded"] is True + _assert_internal_failure_is_safe(resp, counters, "config_shape_invalid") -def test_budget_exceeded_on_tools_also_blocks_next_call(tmp_path, monkeypatch): - """Codex §V:tools request + 预算超限 · next_call 也必须 0 次(解耦证明)。""" - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=50 - ) - counters = {"next": 0} +def test_tenant_context_internal_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + def boom(_plugin_cfg): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - run = make_run_llm_execution( - api_request_id="r_tools", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=500, - ) + monkeypatch.setattr(cache_mod, "_get_tenant_context", boom) + counters, next_call, run = _run_with_counter() resp = run( - {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}], "max_tokens": 100}, + {"messages": [{"role": "user", "content": "private user input"}]}, next_call, ) - assert counters["next"] == 0 - assert resp["cache_meta"]["budget_exceeded"] is True + _assert_internal_failure_is_safe(resp, counters, "tenant_context_error") -def test_non_cacheable_request_still_reserves_and_settles(tmp_path, monkeypatch): - """Codex §II:streaming/tools 非 cacheable · **但仍 reserve/settle**(解耦)。""" - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=10_000 - ) - counters = {"next": 0} +def test_cache_key_internal_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "streamed"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 60, "completion_tokens": 40}} + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - # streaming request(不缓存 · 但必须 reserve) - run = make_run_llm_execution( - api_request_id="r_s1", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=100, - ) - resp = run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) - assert counters["next"] == 1 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 100 # settle 60+40 - # 不 cache · 二次同请求仍 miss(需 fresh api_request_id) - run2 = make_run_llm_execution( - api_request_id="r_s2", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=100, + monkeypatch.setattr(cache_mod, "build_cache_key", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, ) - run2({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) - assert counters["next"] == 2 # 再调 next - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 # 再 settle 100 + _assert_internal_failure_is_safe(resp, counters, "cache_lookup_error") -# ── Provider exception → reservation 释放 + 传播 ──────────────────── - - -def test_provider_exception_releases_reservation(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=10_000 - ) +def test_budget_reserve_internal_error_fail_closed(tmp_path, monkeypatch): + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) - class ProviderBoom(RuntimeError): - pass + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - def next_call(_req): - raise ProviderBoom("upstream 500") - - run = make_run_llm_execution( - api_request_id="req_boom", provider="openai", model="m", api_mode="chat_completions", - approx_input_tokens=100, + monkeypatch.setattr(budget_mod, "reserve", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, ) - with pytest.raises(ProviderBoom, match="upstream 500"): - run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) - - assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 - assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 - assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None - - -# ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ + _assert_internal_failure_is_safe(resp, counters, "budget_reserve_error") @pytest.mark.parametrize( - "bad_tenant", + ("target", "reason_code"), [ - {"tenant_id": "", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, - {"tenant_id": "t", "principal_id": "", "permission_scope_version": "1", "profile_version": "1"}, - {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, - {"tenant_id": "t/../evil", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, - {"tenant_id": "a" * 200, "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + ("is_cacheable_request", "request_classification_error"), + ("_estimate_needed_tokens", "token_estimation_error"), ], - ids=["empty-tenant", "empty-principal", "missing-profile-version", "non-whitelist-chars", "over-max-len"], ) -def test_invalid_tenant_fail_closed_blocks_next_call(tmp_path, monkeypatch, bad_tenant): - """Codex §III:enforcement 开启时 · 缺失/非法 tenant · **next_call == 0** · 结构化 fail-closed。""" - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=100, tenant=bad_tenant - ) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - approx_input_tokens=500, - ) - resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) - assert counters["next"] == 0 # fail-CLOSED · 严格 - assert resp["cache_meta"]["budget_fail_closed"] is True - assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" - assert resp["choices"][0]["message"]["content"] # 非空可展示 error - assert resp["choices"][0]["finish_reason"] == "content_filter" - - -def test_no_tenant_config_fail_closed(tmp_path, monkeypatch): - """config.yaml 无 tenant 段 · fail-CLOSED(不走 next_call)。""" - write_config( - tmp_path, - [CACHE_KEY, BUDGET_KEY], - spike_p3_m0_cache={"enabled": True}, # 无 tenant 段 - ) - install_fresh_manager(monkeypatch, tmp_path) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - ) - resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" - - -def test_non_chat_completions_mode_fail_closed(tmp_path, monkeypatch): - """Codex §II:非 chat_completions api_mode 明确 fail-CLOSED(不静默 pass-through)。""" - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="responses", - ) - resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "unsupported_api_mode" - assert resp["cache_meta"]["api_mode"] == "responses" - - -def test_non_dict_request_fail_closed(tmp_path, monkeypatch): - """非 dict request 无法 estimate token · fail-CLOSED。""" - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return None - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - ) - resp = run(["not", "a", "dict"], next_call) # list · 非 dict - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "non_dict_request" - - -@pytest.mark.parametrize("bad_budget", [True, False, -1, "1000", 1.5]) -def test_invalid_budget_config_fail_closed(tmp_path, monkeypatch, bad_budget): - """Codex §III:BudgetConfigError 硬 fail-CLOSED · **不**转 None 静默继续。""" - manager, cache_mod, budget_mod = _enable_cache( - tmp_path, monkeypatch, daily_budget=bad_budget - ) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", - ) - resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "budget_config_invalid" - - -def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): - """空 api_request_id · reserve 无法追踪 · fail-CLOSED。""" - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} - - run = make_run_llm_execution( - api_request_id="", provider="p", model="m", api_mode="chat_completions", - ) - resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" - - -# ── Cache alone(budget disabled)· 硬依赖门 fail-CLOSED ───────────── - - -def test_cache_alone_without_budget_fail_closed(tmp_path, monkeypatch): - """Codex §III:cache 单独启用时 budget 缺失 · **必须 fail-CLOSED** · 不 pass-through。""" - # 只启用 cache · 不启用 budget · 但 config tenant 完整 - manager, cache_mod, _ = _enable_cache( - tmp_path, monkeypatch, daily_budget=1000, include_budget=False, - ) - counters = {"next": 0} +def test_other_pre_provider_internal_errors_fail_closed( + tmp_path, monkeypatch, target, reason_code, +): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + monkeypatch.setattr(cache_mod, target, boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, ) - resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) - # budget module 未加载 · 硬 fail-CLOSED - assert counters["next"] == 0 - assert resp["cache_meta"]["budget_fail_closed"] is True - assert resp["cache_meta"]["reason_code"] == "budget_module_missing" - - -# ── Cache key schema stability(§2 v1 schema) ────────────────────── - - -def test_cache_key_stable_and_deterministic(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - req = {"messages": [{"role": "user", "content": "hi"}]} - ctx = dict(_VALID_TENANT) - k1 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") - k2 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") - assert k1 == k2 and len(k1) == 64 - + _assert_internal_failure_is_safe(resp, counters, reason_code) -def test_cache_key_changes_on_provider(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - req = {"messages": []} - ctx = dict(_VALID_TENANT) - k1 = cache_mod.build_cache_key(req, ctx, model="m", provider="openai") - k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") - assert k1 != k2 +def test_budget_validation_internal_error_fail_closed(tmp_path, monkeypatch): + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) -def test_cache_key_changes_on_tenant(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - req = {"messages": []} - a = dict(_VALID_TENANT, tenant_id="tA") - b = dict(_VALID_TENANT, tenant_id="tB") - ka = cache_mod.build_cache_key(req, a, model="m", provider="p") - kb = cache_mod.build_cache_key(req, b, model="m", provider="p") - assert ka != kb + def boom(_value): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") - -# ── Tool-call response 不 insert · 下次仍 miss ──────────────────── - - -def test_tool_calls_response_not_inserted(tmp_path, monkeypatch): - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) - - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return { - "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5}, - } - - run1 = make_run_llm_execution(api_request_id="r1", provider="p", model="m", api_mode="chat_completions") - run2 = make_run_llm_execution(api_request_id="r2", provider="p", model="m", api_mode="chat_completions") - req = {"messages": [{"role": "user", "content": "call tool"}]} - run1(req, next_call) - run2(req, next_call) - assert counters["next"] == 2 - - -# ── Input token estimation (Codex §V:不静默 0) ──────────────────── - - -def test_input_tokens_estimated_from_messages_when_no_ctx(tmp_path, monkeypatch): - """无 ctx.approx_input_tokens · 从 messages char count / 4 估算 · 不静默 0。""" - manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) - counters = {"next": 0} - - def next_call(_req): - counters["next"] += 1 - return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 5, "completion_tokens": 2}} - - # 未传 approx_input_tokens · content 长约 4000 字符 → 估算 ~1000 tokens 触发超支 - long_content = "x" * 4000 - run = make_run_llm_execution( - api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + monkeypatch.setattr(budget_mod, "validate_daily_budget", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, ) - resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) - # 估算 = 4000/4 + 100 = 1100 · budget=1000 · 超支 - assert counters["next"] == 0 - assert resp["cache_meta"]["budget_exceeded"] is True - assert resp["cache_meta"]["tokens_needed"] >= 1000 + _assert_internal_failure_is_safe(resp, counters, "budget_config_error") + \ No newline at end of file From abca00c095cc163af91db3d97c0fa6109f160212 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 04:45:56 +0800 Subject: [PATCH 07/19] test(spike): cover fail-closed enforcement edges --- plugins/spike_p3_m0_budget/__init__.py | 416 +++++++++++++------------ 1 file changed, 209 insertions(+), 207 deletions(-) diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py index c83ae2d7f8ef7..6cfb8efca8479 100644 --- a/plugins/spike_p3_m0_budget/__init__.py +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -1,207 +1,209 @@ -"""P3-M0 Spike · tenant budget service module(no hooks · used by cache orchestrator). - -**Codex §I / §V 契约**: -- **不**通过 `pre_api_request` 抛 BudgetExceeded(生产调用点会吞异常) -- 由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内部按序调用: - cache lookup → miss → **budget reserve** → next_call → success settle actual / - exception release reservation -- `pre_api_request/post_api_request` 只做观察 audit(见 __init__.py 底部可选注册) - -**4 态 budget**(与 Hermes_AI P2-0.1 一致): -- 缺失(config 无 `tenant.daily_budget_tokens` 键)· 不启用租户 budget -- `null`(显式 None)· 不启用(provisioning 明确无限期不 enforce · 与 0 区分) -- `0`(显式 int 0)· 显式无限 · reserve 恒 True · used 仍计 -- `>0`(int)· 具体上限 -- **拒**:bool / 负数 / str / float · 配置错误 · 视 as 缺失 · fail-open + audit - -**UTC daily bucket**:key = (tenant_id, utc_date_str) · 天粒度 · pending reservations 与 -used 一起判断超支。 - -**并发**:threading.Lock · reserve/settle/release 全序 · api_request_id 唯一. -""" -from __future__ import annotations - -import datetime -import logging -import threading -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - -BUDGET_CONTRACT_VERSION = "v1" - - -class BudgetExceeded(RuntimeError): - """租户预算超支 · **不由 hook 抛** · 由 llm_execution middleware 内部捕获转 audit。""" - - -class BudgetConfigError(ValueError): - """budget 配置字段类型/值非法 · 由 orchestrator 硬 fail-CLOSED(**不**再静默转 None)。""" - - -class BudgetSettleError(RuntimeError): - """settle/release 在无 reservation 上被调用 · **不改** used(严格幂等)。 - - Codex 第三轮 §IV:settle 必须仅结算仍存在的 reservation;unknown/double settle - 不得增加 used。抛此异常供 orchestrator 记 audit 但不阻断。 - """ - - -# ── Module state · 单进程 · 非 persistent ───────────────────────────── - -_lock = threading.RLock() # reentrant · reserve/settle 内部可能连续调用 -# 天粒度桶 · key = (tenant_id, "YYYY-MM-DD" UTC) · value = { "used": int, "reservations": {req_id: tokens} } -_bucket: Dict[str, Dict[str, Any]] = {} - - -def _today_utc() -> str: - """UTC 日期字符串 · YYYY-MM-DD 格式 · 天粒度 bucket key。""" - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") - - -def _bucket_key(tenant_id: str, utc_date: Optional[str] = None) -> str: - return f"{tenant_id}::{utc_date or _today_utc()}" - - -def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: - key = _bucket_key(tenant_id) - b = _bucket.get(key) - if b is None: - b = {"used": 0, "reservations": {}} - _bucket[key] = b - return b - - -# ── 4 态 budget validation ─────────────────────────────────────────── - -def validate_daily_budget(value: Any) -> Optional[int]: - """校验 config `tenant.daily_budget_tokens` 字段 · 返回归一化 int | None。 - - None → None(未启用) - 0 → 0(显式无限) - 正整数 → 正整数 - bool / 负数 / str / float → raise BudgetConfigError · 上层视 as None(fail-open + audit) - """ - if value is None: - return None - # bool 是 int 子类 · 必须先拒 - if isinstance(value, bool): - raise BudgetConfigError(f"daily_budget_tokens type=bool 非法 · got {value!r}") - if not isinstance(value, int): - raise BudgetConfigError(f"daily_budget_tokens 需 int · got {type(value).__name__} {value!r}") - if value < 0: - raise BudgetConfigError(f"daily_budget_tokens 需 >=0 · got {value}") - return value - - -# ── Reserve / Settle / Release · 由 orchestrator middleware 调用 ───── - -def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: Optional[int]) -> None: - """尝试预留 · 超预算 raise BudgetExceeded · budget=None 直接 no-op(未启用)。 - - `budget=0` 是显式无限 · reserve 恒成功 · used 仍会累加供审计观察。 - """ - if not api_request_id: - raise BudgetExceeded("reserve: empty api_request_id · 拒绝 anonymous 预留") - if tokens_needed < 0: - tokens_needed = 0 - if budget is None: - return # 未启用 - with _lock: - b = _get_or_create_bucket(tenant_id) - # api_request_id 唯一 · 重复 reserve 视 as 错误 - if api_request_id in b["reservations"]: - raise BudgetExceeded( - f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" - ) - pending_total = sum(b["reservations"].values()) - # budget=0 无限 · used + pending + new 不做上限比较 - if budget > 0: - projected = b["used"] + pending_total + tokens_needed - if projected > budget: - raise BudgetExceeded( - f"tenant {tenant_id} daily budget exhausted " - f"(used={b['used']}, pending={pending_total}, need={tokens_needed}, budget={budget})" - ) - b["reservations"][api_request_id] = tokens_needed - - -def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: - """结算实际用量 · **仅结算仍存在的 reservation** · 累加 used。 - - Codex 第三轮 §IV(**严格幂等**): - - 若 reservation 存在 · 移除并累加 used - - 若 reservation 不存在(未 reserve / double settle / release 后 settle): - **不改** used · raise BudgetSettleError · 由 orchestrator 记 audit - """ - if actual_tokens < 0: - actual_tokens = 0 - with _lock: - b = _get_or_create_bucket(tenant_id) - if api_request_id not in b["reservations"]: - raise BudgetSettleError( - f"settle: no reservation for api_request_id={api_request_id!r} · " - "double settle or missing reserve" - ) - b["reservations"].pop(api_request_id) - b["used"] = b["used"] + actual_tokens - - -def release(tenant_id: str, api_request_id: str) -> bool: - """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。 - - Codex 第三轮 §IV(**幂等**): - - 若 reservation 存在 · 移除 · 返回 True - - 若 reservation 不存在 · **no-op** · 返回 False(便于 orchestrator 检测 double release) - - 任何情况下 · used 不变 - """ - with _lock: - b = _get_or_create_bucket(tenant_id) - if api_request_id in b["reservations"]: - b["reservations"].pop(api_request_id) - return True - return False - - -# ── 观察 helpers · 测试用 ───────────────────────────────────────────── - -def get_used(tenant_id: str) -> int: - with _lock: - return _get_or_create_bucket(tenant_id)["used"] - - -def get_pending_total(tenant_id: str) -> int: - with _lock: - return sum(_get_or_create_bucket(tenant_id)["reservations"].values()) - - -def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: - with _lock: - return _get_or_create_bucket(tenant_id)["reservations"].get(api_request_id) - - -def reset_state() -> None: - """测试用 · 清空全部 bucket。""" - with _lock: - _bucket.clear() - - -# ── register: 只挂 observer hooks · 无 enforcement · 无 middleware ──── - -def _on_pre_api_request_observer(**_kwargs: Any) -> None: - """观察者:reserve 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" - return - - -def _on_post_api_request_observer(**_kwargs: Any) -> None: - """观察者:settle 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" - return - - -def register(ctx: Any) -> None: - """PluginContext register · 只挂观察 hooks · 不含 enforcement。 - - 真实 enforcement 在 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内。 - """ - ctx.register_hook("pre_api_request", _on_pre_api_request_observer) - ctx.register_hook("post_api_request", _on_post_api_request_observer) +"""P3-M0 Spike · tenant budget service module(no hooks · used by cache orchestrator). + +**Codex §I / §V 契约**: +- **不**通过 `pre_api_request` 抛 BudgetExceeded(生产调用点会吞异常) +- 由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内部按序调用: + cache lookup → miss → **budget reserve** → next_call → success settle actual / + exception release reservation +- `pre_api_request/post_api_request` 只做观察 audit(见 __init__.py 底部可选注册) + +**4 态 budget**(与 Hermes_AI P2-0.1 一致): +- 缺失(config 无 `tenant.daily_budget_tokens` 键)· 不启用租户 budget +- `null`(显式 None)· 不启用(provisioning 明确无限期不 enforce · 与 0 区分) +- `0`(显式 int 0)· 显式无限 · reserve 恒 True · used 仍计 +- `>0`(int)· 具体上限 +- **拒**:bool / 负数 / str / float · 抛 `BudgetConfigError` · enforcement 上层必须 fail-CLOSED + +**UTC daily bucket**:key = (tenant_id, utc_date_str) · 天粒度 · pending reservations 与 +used 一起判断超支。 + +**并发**:threading.Lock · reserve/settle/release 全序 · api_request_id 唯一. +""" +from __future__ import annotations + +import datetime +import logging +import threading +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +BUDGET_CONTRACT_VERSION = "v1" + + +class BudgetExceeded(RuntimeError): + """租户预算超支 · **不由 hook 抛** · 由 llm_execution middleware 内部捕获转 audit。""" + + +class BudgetConfigError(ValueError): + """budget 配置字段类型/值非法 · 由 orchestrator 硬 fail-CLOSED(**不**再静默转 None)。""" + + +class BudgetSettleError(RuntimeError): + """settle/release 在无 reservation 上被调用 · **不改** used(严格幂等)。 + + Codex 第三轮 §IV:settle 必须仅结算仍存在的 reservation;unknown/double settle + 不得增加 used。抛此异常供 orchestrator 记 audit 但不阻断。 + """ + + +# ── Module state · 单进程 · 非 persistent ───────────────────────────── + +_lock = threading.RLock() # reentrant · reserve/settle 内部可能连续调用 +# 天粒度桶 · key = (tenant_id, "YYYY-MM-DD" UTC) · value = { "used": int, "reservations": {req_id: tokens} } +_bucket: Dict[str, Dict[str, Any]] = {} + + +def _today_utc() -> str: + """UTC 日期字符串 · YYYY-MM-DD 格式 · 天粒度 bucket key。""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + + +def _bucket_key(tenant_id: str, utc_date: Optional[str] = None) -> str: + return f"{tenant_id}::{utc_date or _today_utc()}" + + +def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: + key = _bucket_key(tenant_id) + b = _bucket.get(key) + if b is None: + b = {"used": 0, "reservations": {}} + _bucket[key] = b + return b + + +# ── 4 态 budget validation ─────────────────────────────────────────── + +def validate_daily_budget(value: Any) -> Optional[int]: + """校验 config `tenant.daily_budget_tokens` 字段 · 返回归一化 int | None。 + + None → None(未启用) + 0 → 0(显式无限) + 正整数 → 正整数 + bool / 负数 / str / float → raise BudgetConfigError · enforcement 上层必须 + 返回结构化 fail-CLOSED response，禁止转换为 None 后继续 provider 调用。 + """ + if value is None: + return None + # bool 是 int 子类 · 必须先拒 + if isinstance(value, bool): + raise BudgetConfigError(f"daily_budget_tokens type=bool 非法 · got {value!r}") + if not isinstance(value, int): + raise BudgetConfigError(f"daily_budget_tokens 需 int · got {type(value).__name__} {value!r}") + if value < 0: + raise BudgetConfigError(f"daily_budget_tokens 需 >=0 · got {value}") + return value + + +# ── Reserve / Settle / Release · 由 orchestrator middleware 调用 ───── + +def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: Optional[int]) -> None: + """尝试预留 · 超预算 raise BudgetExceeded · budget=None 直接 no-op(未启用)。 + + `budget=0` 是显式无限 · reserve 恒成功 · used 仍会累加供审计观察。 + """ + if not api_request_id: + raise BudgetExceeded("reserve: empty api_request_id · 拒绝 anonymous 预留") + if tokens_needed < 0: + tokens_needed = 0 + if budget is None: + return # 未启用 + with _lock: + b = _get_or_create_bucket(tenant_id) + # api_request_id 唯一 · 重复 reserve 视 as 错误 + if api_request_id in b["reservations"]: + raise BudgetExceeded( + f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" + ) + pending_total = sum(b["reservations"].values()) + # budget=0 无限 · used + pending + new 不做上限比较 + if budget > 0: + projected = b["used"] + pending_total + tokens_needed + if projected > budget: + raise BudgetExceeded( + f"tenant {tenant_id} daily budget exhausted " + f"(used={b['used']}, pending={pending_total}, need={tokens_needed}, budget={budget})" + ) + b["reservations"][api_request_id] = tokens_needed + + +def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: + """结算实际用量 · **仅结算仍存在的 reservation** · 累加 used。 + + Codex 第三轮 §IV(**严格幂等**): + - 若 reservation 存在 · 移除并累加 used + - 若 reservation 不存在(未 reserve / double settle / release 后 settle): + **不改** used · raise BudgetSettleError · 由 orchestrator 记 audit + """ + if actual_tokens < 0: + actual_tokens = 0 + with _lock: + b = _get_or_create_bucket(tenant_id) + if api_request_id not in b["reservations"]: + raise BudgetSettleError( + f"settle: no reservation for api_request_id={api_request_id!r} · " + "double settle or missing reserve" + ) + b["reservations"].pop(api_request_id) + b["used"] = b["used"] + actual_tokens + + +def release(tenant_id: str, api_request_id: str) -> bool: + """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。 + + Codex 第三轮 §IV(**幂等**): + - 若 reservation 存在 · 移除 · 返回 True + - 若 reservation 不存在 · **no-op** · 返回 False(便于 orchestrator 检测 double release) + - 任何情况下 · used 不变 + """ + with _lock: + b = _get_or_create_bucket(tenant_id) + if api_request_id in b["reservations"]: + b["reservations"].pop(api_request_id) + return True + return False + + +# ── 观察 helpers · 测试用 ───────────────────────────────────────────── + +def get_used(tenant_id: str) -> int: + with _lock: + return _get_or_create_bucket(tenant_id)["used"] + + +def get_pending_total(tenant_id: str) -> int: + with _lock: + return sum(_get_or_create_bucket(tenant_id)["reservations"].values()) + + +def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: + with _lock: + return _get_or_create_bucket(tenant_id)["reservations"].get(api_request_id) + + +def reset_state() -> None: + """测试用 · 清空全部 bucket。""" + with _lock: + _bucket.clear() + + +# ── register: 只挂 observer hooks · 无 enforcement · 无 middleware ──── + +def _on_pre_api_request_observer(**_kwargs: Any) -> None: + """观察者:reserve 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return + + +def _on_post_api_request_observer(**_kwargs: Any) -> None: + """观察者:settle 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return + + +def register(ctx: Any) -> None: + """PluginContext register · 只挂观察 hooks · 不含 enforcement。 + + 真实 enforcement 在 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内。 + """ + ctx.register_hook("pre_api_request", _on_pre_api_request_observer) + ctx.register_hook("post_api_request", _on_post_api_request_observer) + \ No newline at end of file From 59a1a1d386af6fde0be9447b637167d7181b83a3 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 05:07:05 +0800 Subject: [PATCH 08/19] fix(spike): harden budget accounting and cache isolation --- plugins/spike_p3_m0_cache/__init__.py | 191 +++++++++++++++++--------- 1 file changed, 124 insertions(+), 67 deletions(-) diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index a2f218a1a7b12..b1be74dacc805 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -50,9 +50,10 @@ - 否则从 request.messages content 字符数 / 4(粗略英文 heuristic · M0 approximation · 见文档) - 加上 request.max_tokens 上限 · 得 tokens_needed """ -from __future__ import annotations - -import hashlib +from __future__ import annotations + +import copy +import hashlib import importlib import json import logging @@ -308,7 +309,7 @@ def _get(obj, k, default=None): "prompt_tokens": getattr(origin_usage_obj, "prompt_tokens", 0) or 0, "completion_tokens": getattr(origin_usage_obj, "completion_tokens", 0) or 0, } - choices = _get(response, "choices") + choices = copy.deepcopy(_get(response, "choices")) return { "model": _get(response, "model"), "choices": choices, @@ -317,9 +318,9 @@ def _get(obj, k, default=None): "cache_meta": { "hit": True, "cache_contract_version": CACHE_CONTRACT_VERSION, - "origin_usage": origin_usage_obj, + "origin_usage": copy.deepcopy(origin_usage_obj), "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, - "saved_usage": origin_usage_obj, + "saved_usage": copy.deepcopy(origin_usage_obj), "cache_lookup_cost": None, # in-memory · 通常 0 }, } @@ -370,18 +371,23 @@ def _estimate_needed_tokens(request: Any, ctx: Dict[str, Any]) -> int: approx_input = int(explicit) else: approx_input = _estimate_input_tokens_from_messages(request if isinstance(request, dict) else {}) - max_tokens_raw = None + max_tokens_raw = None if isinstance(request, dict): max_tokens_raw = request.get("max_tokens") if max_tokens_raw is None: max_tokens_raw = ctx.get("max_tokens") - try: - max_tokens = int(max_tokens_raw or 0) - except (TypeError, ValueError): - max_tokens = 0 - if max_tokens < 0: - max_tokens = 0 - return approx_input + max_tokens + if isinstance(max_tokens_raw, bool): + raise ValueError("max_tokens must not be bool") + try: + max_tokens = int(max_tokens_raw or 0) + except (TypeError, ValueError) as exc: + raise ValueError("max_tokens must be an integer") from exc + if max_tokens < 0: + raise ValueError("max_tokens must be >= 0") + estimated = approx_input + max_tokens + if estimated <= 0: + raise ValueError("token estimate unavailable") + return estimated # ── Structured error responses(Codex §V:非空、安全、可展示) ───── @@ -452,16 +458,23 @@ def cache_and_budget_middleware( # 2. Only an explicitly disabled plugin may pass through. Missing/false # ``enabled`` is a deliberate off switch; read/shape failures above are # never converted into this branch. - if not _is_enabled(plugin_cfg): + try: + enabled = _is_enabled(plugin_cfg) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: enabled-state resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="config_enabled_error") + if not enabled: return next_call(request) # 3. Budget module 硬依赖门(Codex §III:cache 单独启用不得因 budget 缺失 fail-open) try: budget_mod = _budget_module() - except ImportError as exc: - logger.error( - "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED: %s", - exc, + except ImportError: + logger.error( + "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED", ) return _fail_closed_response(model, reason_code="budget_module_missing") except Exception as exc: # noqa: BLE001 @@ -472,7 +485,15 @@ def cache_and_budget_middleware( return _fail_closed_response(model, reason_code="budget_module_error") # 4. api_mode 支持范围收紧(Codex §II:非 chat_completions fail-CLOSED · 不静默 pass-through) - if not _api_mode_supported(api_mode): + try: + api_mode_supported = _api_mode_supported(api_mode) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: api-mode classification failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="api_mode_error") + if not api_mode_supported: logger.warning( "spike-cache: unsupported api_mode=%s · M0 only supports chat_completions · fail-CLOSED", api_mode, @@ -487,6 +508,7 @@ def cache_and_budget_middleware( # 6. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) try: tenant_ctx = _get_tenant_context(plugin_cfg) + tenant_id = tenant_ctx["tenant_id"] if tenant_ctx is not None else None except Exception as exc: # noqa: BLE001 logger.error( "spike-cache: tenant context resolution failed · type=%s · fail-CLOSED", @@ -495,7 +517,7 @@ def cache_and_budget_middleware( return _fail_closed_response(model, reason_code="tenant_context_error") if tenant_ctx is None: return _fail_closed_response(model, reason_code="tenant_context_invalid") - tenant_id = tenant_ctx["tenant_id"] + assert tenant_id is not None # 7. Budget config 校验 · BudgetConfigError 硬 fail-CLOSED(Codex §III:不转 None 静默继续) tenant_cfg = plugin_cfg.get("tenant") @@ -503,8 +525,8 @@ def cache_and_budget_middleware( raw_budget = tenant_cfg.get("daily_budget_tokens") try: budget_value = budget_mod.validate_daily_budget(raw_budget) - except budget_mod.BudgetConfigError as exc: - logger.error("spike-cache: daily_budget_tokens config invalid · fail-CLOSED: %s", exc) + except budget_mod.BudgetConfigError: + logger.error("spike-cache: daily_budget_tokens config invalid · fail-CLOSED") return _fail_closed_response(model, reason_code="budget_config_invalid") except Exception as exc: # noqa: BLE001 logger.error( @@ -547,7 +569,7 @@ def cache_and_budget_middleware( return _fail_closed_response(model, reason_code="cache_lookup_error") if hit is not None: # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate - return dict(hit) + return copy.deepcopy(hit) # 11. MISS 或非 cacheable · 都必须 reserve(Codex §II step 6) try: @@ -558,27 +580,27 @@ def cache_and_budget_middleware( type(exc).__name__, ) return _fail_closed_response(model, reason_code="token_estimation_error") + reservation_active = budget_value is not None try: - budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) + if reservation_active: + budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) except budget_mod.BudgetExceeded: logger.warning( "spike-cache: budget exceeded · tenant=%s api_request_id=%s tokens_needed=%s budget=%s", tenant_id, api_request_id, tokens_needed, budget_value, ) - return _fail_closed_response( - model, - reason_code="budget_exceeded", - tenant_id=tenant_id, - tokens_needed=tokens_needed, - budget=budget_value, + return _fail_closed_response( + model, + reason_code="budget_exceeded", ) except Exception as exc: # noqa: BLE001 # Defensive cleanup in case a future reserve implementation mutates # state before raising. Never expose exception text to the user. - try: - budget_mod.release(tenant_id, api_request_id) - except Exception: # noqa: BLE001 - pass + if reservation_active: + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass logger.error( "spike-cache: budget reserve internal failure · type=%s · fail-CLOSED", type(exc).__name__, @@ -589,30 +611,52 @@ def cache_and_budget_middleware( # try block deliberately preserves downstream exception identity. try: response = next_call(request) - except Exception: - budget_mod.release(tenant_id, api_request_id) - raise - + except Exception: + if reservation_active: + try: + budget_mod.release(tenant_id, api_request_id) + except Exception as cleanup_exc: # noqa: BLE001 + logger.error( + "spike-cache: reservation release failed after provider error · type=%s", + type(cleanup_exc).__name__, + ) + raise + # 13. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) - actual = _extract_actual_usage(response) - try: - budget_mod.settle(tenant_id, api_request_id, actual) - except Exception as exc: # noqa: BLE001 - # Provider cost has already been incurred. Do not throw into the - # outer fail-open chain after next_call; clean pending state best - # effort, record only the exception type, and preserve the response. + if reservation_active: + actual = _extract_actual_usage(response) + if actual is None: + actual = tokens_needed try: - budget_mod.release(tenant_id, api_request_id) - except Exception: # noqa: BLE001 - pass - logger.error( - "spike-cache: budget settle internal failure · api_request_id=%s type=%s", - api_request_id, type(exc).__name__, - ) + budget_mod.settle(tenant_id, api_request_id, actual) + except Exception as exc: # noqa: BLE001 + # Provider cost has already been incurred. Do not throw into the + # outer fail-open chain after next_call; clean pending state best + # effort, record only the exception type, and preserve the response. + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass + logger.error( + "spike-cache: budget settle internal failure · type=%s", + type(exc).__name__, + ) # 14. 只有 req + resp 皆 cacheable 才 insert. Cache bookkeeping is # best-effort after a paid provider response and must never replace it. - if cacheable_req and cache_key is not None and is_cacheable_response(response): + try: + cacheable_resp = ( + cacheable_req + and cache_key is not None + and is_cacheable_response(response) + ) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: response classification failed after provider response · type=%s", + type(exc).__name__, + ) + cacheable_resp = False + if cacheable_resp: try: cached = _extract_cacheable_response(response) with _lock: @@ -629,18 +673,31 @@ def cache_and_budget_middleware( return response -def _extract_actual_usage(response: Any) -> int: - """从 response.usage 抽 prompt+completion tokens.""" - try: - if isinstance(response, dict): - usage = response.get("usage") or {} - else: - usage = getattr(response, "usage", None) or {} - if isinstance(usage, dict): - return int(usage.get("prompt_tokens", 0) or 0) + int(usage.get("completion_tokens", 0) or 0) - return int(getattr(usage, "prompt_tokens", 0) or 0) + int(getattr(usage, "completion_tokens", 0) or 0) - except Exception: # noqa: BLE001 - return 0 +def _extract_actual_usage(response: Any) -> Optional[int]: + """从 response.usage 抽 token 数；缺失/非法返回 None 触发 reservation fallback。""" + try: + if isinstance(response, dict): + usage = response.get("usage") + else: + usage = getattr(response, "usage", None) + if usage is None: + return None + if isinstance(usage, dict): + if "prompt_tokens" not in usage and "completion_tokens" not in usage: + return None + prompt = usage.get("prompt_tokens", 0) + completion = usage.get("completion_tokens", 0) + else: + if not hasattr(usage, "prompt_tokens") and not hasattr(usage, "completion_tokens"): + return None + prompt = getattr(usage, "prompt_tokens", 0) + completion = getattr(usage, "completion_tokens", 0) + if isinstance(prompt, bool) or isinstance(completion, bool): + return None + total = int(prompt or 0) + int(completion or 0) + return total if total >= 0 else None + except Exception: # noqa: BLE001 + return None def clear_cache() -> None: @@ -657,4 +714,4 @@ def cache_size() -> int: def register(ctx: Any) -> None: """PluginContext register · 挂 llm_execution middleware(orchestrator).""" ctx.register_middleware("llm_execution", cache_and_budget_middleware) - \ No newline at end of file + From 969e8bfce8bde3e4dcb635d8cde2c0306c4b7a7d Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 05:07:14 +0800 Subject: [PATCH 09/19] test(spike): cover accounting fallbacks and response isolation --- .../tests/test_cache_middleware.py | 184 ++++++++++++++++-- 1 file changed, 173 insertions(+), 11 deletions(-) diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index 9042f18323855..16e0c9f83ae04 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -179,7 +179,7 @@ def next_call(_req): # ── Cache hit → budget reserve 0 次 · 顶层 usage=0(§I 强要求) ───── -def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): +def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): manager, cache_mod, budget_mod = _enable_cache( tmp_path, monkeypatch, daily_budget=10_000 ) @@ -252,9 +252,12 @@ def next_call(_req): # 超支明确 short-circuit · next 未调 assert counters["next"] == 0 # 非空结构化 error response · 抑制 retry · Codex §V - assert resp["cache_meta"]["budget_exceeded"] is True - assert resp["cache_meta"]["reason_code"] == "budget_exceeded" - assert resp["cache_meta"]["tenant_id"] == _VALID_TENANT["tenant_id"] + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["reason_code"] == "budget_exceeded" + # 用户可见响应不得暴露租户标识、预算值或内部估算。 + assert "tenant_id" not in resp["cache_meta"] + assert "budget" not in resp["cache_meta"] + assert "tokens_needed" not in resp["cache_meta"] # 非空 assistant message(可展示给用户) assert resp["choices"][0]["message"]["role"] == "assistant" assert resp["choices"][0]["message"]["content"] # 非空字符串 @@ -346,7 +349,7 @@ def next_call(_req): # ── Provider exception → reservation 释放 + 传播 ──────────────────── -def test_provider_exception_releases_reservation(tmp_path, monkeypatch): +def test_provider_exception_releases_reservation(tmp_path, monkeypatch): manager, cache_mod, budget_mod = _enable_cache( tmp_path, monkeypatch, daily_budget=10_000 ) @@ -366,7 +369,121 @@ def next_call(_req): assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 - assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None + assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None + + +def test_provider_exception_identity_survives_release_failure(tmp_path, monkeypatch): + """清理失败不得遮蔽 provider 原异常。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + class ProviderBoom(RuntimeError): + pass + + def next_call(_req): + raise ProviderBoom("provider-original") + + def release_boom(*_args, **_kwargs): + raise RuntimeError("cleanup-failed") + + monkeypatch.setattr(budget_mod, "release", release_boom) + run = make_run_llm_execution( + api_request_id="req_release_boom", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=100, + ) + with pytest.raises(ProviderBoom, match="provider-original"): + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) + + +def test_null_budget_skips_reservation_and_settle(tmp_path, monkeypatch): + """null 是合法无限态：允许 provider，且不制造 unknown-settle 错误。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=None) + calls = {"reserve": 0, "settle": 0, "next": 0} + + monkeypatch.setattr( + budget_mod, + "reserve", + lambda *_args, **_kwargs: calls.__setitem__("reserve", calls["reserve"] + 1), + ) + monkeypatch.setattr( + budget_mod, + "settle", + lambda *_args, **_kwargs: calls.__setitem__("settle", calls["settle"] + 1), + ) + + def next_call(_req): + calls["next"] += 1 + return { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2}, + } + + run = make_run_llm_execution( + api_request_id="req_null_budget", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=10, + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert resp["choices"][0]["message"]["content"] == "ok" + assert calls == {"reserve": 0, "settle": 0, "next": 1} + + +def test_missing_usage_settles_reserved_estimate(tmp_path, monkeypatch): + """provider 未回 usage 时按预留值结算，禁止按 0 洗预算。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + def next_call(_req): + return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}]} + + run = make_run_llm_execution( + api_request_id="req_missing_usage", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=123, + max_tokens=77, + ) + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 77}, next_call) + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +def test_cache_hit_returns_deep_copy(tmp_path, monkeypatch): + """调用方修改一次 hit 结果不得污染后续缓存。""" + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + calls = {"next": 0} + + def next_call(_req): + calls["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "stable"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}, + } + + req = {"messages": [{"role": "user", "content": "same"}]} + make_run_llm_execution( + api_request_id="copy_miss", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit = make_run_llm_execution( + api_request_id="copy_hit_1", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit["choices"][0]["message"]["content"] = "poisoned" + first_hit["cache_meta"]["origin_usage"]["prompt_tokens"] = 999 + + second_hit = make_run_llm_execution( + api_request_id="copy_hit_2", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + assert calls["next"] == 1 + assert second_hit["choices"][0]["message"]["content"] == "stable" + assert second_hit["cache_meta"]["origin_usage"]["prompt_tokens"] == 5 # ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ @@ -597,11 +714,32 @@ def next_call(_req): run = make_run_llm_execution( api_request_id="r1", provider="p", model="m", api_mode="chat_completions", ) - resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) + resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) # 估算 = 4000/4 + 100 = 1100 · budget=1000 · 超支 - assert counters["next"] == 0 - assert resp["cache_meta"]["budget_exceeded"] is True - assert resp["cache_meta"]["tokens_needed"] >= 1000 + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + assert "tokens_needed" not in resp["cache_meta"] + + +@pytest.mark.parametrize("bad_max_tokens", [True, -1, "not-an-int"]) +def test_invalid_or_unknown_token_estimate_fails_closed( + tmp_path, monkeypatch, bad_max_tokens, +): + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + + run = make_run_llm_execution( + api_request_id="bad_estimate", + provider="p", + model="m", + api_mode="chat_completions", + ) + resp = run({"messages": [], "max_tokens": bad_max_tokens}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "token_estimation_error" # ── Registered middleware must not inherit the core fail-open policy ── @@ -721,6 +859,8 @@ def boom(*_args, **_kwargs): @pytest.mark.parametrize( ("target", "reason_code"), [ + ("_is_enabled", "config_enabled_error"), + ("_api_mode_supported", "api_mode_error"), ("is_cacheable_request", "request_classification_error"), ("_estimate_needed_tokens", "token_estimation_error"), ], @@ -755,4 +895,26 @@ def boom(_value): next_call, ) _assert_internal_failure_is_safe(resp, counters, "budget_config_error") - \ No newline at end of file + + +def test_response_classification_error_preserves_provider_response(tmp_path, monkeypatch): + """provider 已成功后，缓存分类错误只能降级为不缓存。""" + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(_response): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(cache_mod, "is_cacheable_response", boom) + expected = { + "choices": [{"message": {"content": "provider-success"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 3}, + } + run = make_run_llm_execution( + api_request_id="response_classification", + provider="p", + model="m", + api_mode="chat_completions", + approx_input_tokens=10, + ) + assert run({"messages": [{"role": "user", "content": "x"}]}, lambda _req: expected) is expected + From 84397c5493d76ca3f7c0ba2c0d14efa621a8aa57 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 16:23:57 +0800 Subject: [PATCH 10/19] docs(spike): clarify budget accounting and cache isolation --- plugins/spike_p3_m0_cache/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index b1be74dacc805..88f3b08f1a43c 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -21,10 +21,10 @@ ├─ cacheable_request → cache lookup │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) │ - ├─ (MISS 或 非 cacheable_request)→ budget.reserve + ├─ (MISS 或非 cacheable_request)→ budget 非 null 时 reserve │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) │ └─ 成功 → next_call - │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) + │ ├─ 成功 → usage 缺失时按 reservation 结算 + cache insert(若 req+resp 皆 cacheable) │ └─ 异常 → budget.release + re-raise 3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / @@ -35,7 +35,7 @@ bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 -5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` +5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` · cache insert/hit 均深复制防调用方污染 6. 并发:threading.RLock 保护 LRU + insert 序 From 1d81cf761e0b3031e7018f6847d373412ddad98a Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 16:50:45 +0800 Subject: [PATCH 11/19] chore(contributors): map PR #1 commit email --- contributors/emails/bri12afdsarker96@gmail.com | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 contributors/emails/bri12afdsarker96@gmail.com diff --git a/contributors/emails/bri12afdsarker96@gmail.com b/contributors/emails/bri12afdsarker96@gmail.com new file mode 100644 index 0000000000000..d0bdbc8685d5e --- /dev/null +++ b/contributors/emails/bri12afdsarker96@gmail.com @@ -0,0 +1,2 @@ +bri12afdsarker96-lgtm +# PR #1 P3-M0 spike contributor mapping From 6ca5e361ae170227a07d3b9cb2d5016940c9a8dc Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:03:44 +0800 Subject: [PATCH 12/19] fix(spike): independent review hardening (1/7) --- plugins/spike_p3_m0_budget/__init__.py | 115 +++++++++++++++---------- 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py index 6cfb8efca8479..e778755bb742d 100644 --- a/plugins/spike_p3_m0_budget/__init__.py +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -51,7 +51,11 @@ class BudgetSettleError(RuntimeError): _lock = threading.RLock() # reentrant · reserve/settle 内部可能连续调用 # 天粒度桶 · key = (tenant_id, "YYYY-MM-DD" UTC) · value = { "used": int, "reservations": {req_id: tokens} } -_bucket: Dict[str, Dict[str, Any]] = {} +_bucket: Dict[str, Dict[str, Any]] = {} +# Active reservations must remember the UTC bucket in which they were created. +# A provider call can cross midnight; settle/release must then touch the original +# bucket instead of leaking the reservation in yesterday's ledger. +_reservation_bucket: Dict[tuple[str, str], str] = {} def _today_utc() -> str: @@ -63,13 +67,16 @@ def _bucket_key(tenant_id: str, utc_date: Optional[str] = None) -> str: return f"{tenant_id}::{utc_date or _today_utc()}" -def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: - key = _bucket_key(tenant_id) - b = _bucket.get(key) - if b is None: - b = {"used": 0, "reservations": {}} - _bucket[key] = b - return b +def _get_or_create_bucket_by_key(key: str) -> Dict[str, Any]: + b = _bucket.get(key) + if b is None: + b = {"used": 0, "reservations": {}} + _bucket[key] = b + return b + + +def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: + return _get_or_create_bucket_by_key(_bucket_key(tenant_id)) # ── 4 态 budget validation ─────────────────────────────────────────── @@ -106,16 +113,18 @@ def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: raise BudgetExceeded("reserve: empty api_request_id · 拒绝 anonymous 预留") if tokens_needed < 0: tokens_needed = 0 - if budget is None: - return # 未启用 - with _lock: - b = _get_or_create_bucket(tenant_id) - # api_request_id 唯一 · 重复 reserve 视 as 错误 - if api_request_id in b["reservations"]: - raise BudgetExceeded( - f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" - ) - pending_total = sum(b["reservations"].values()) + if budget is None: + return # 未启用 + with _lock: + reservation_key = (tenant_id, api_request_id) + # api_request_id 在该 tenant 的所有活跃 UTC bucket 中唯一。 + if reservation_key in _reservation_bucket: + raise BudgetExceeded( + f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" + ) + bucket_key = _bucket_key(tenant_id) + b = _get_or_create_bucket_by_key(bucket_key) + pending_total = sum(b["reservations"].values()) # budget=0 无限 · used + pending + new 不做上限比较 if budget > 0: projected = b["used"] + pending_total + tokens_needed @@ -123,8 +132,9 @@ def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: raise BudgetExceeded( f"tenant {tenant_id} daily budget exhausted " f"(used={b['used']}, pending={pending_total}, need={tokens_needed}, budget={budget})" - ) - b["reservations"][api_request_id] = tokens_needed + ) + b["reservations"][api_request_id] = tokens_needed + _reservation_bucket[reservation_key] = bucket_key def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: @@ -135,17 +145,26 @@ def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: - 若 reservation 不存在(未 reserve / double settle / release 后 settle): **不改** used · raise BudgetSettleError · 由 orchestrator 记 audit """ - if actual_tokens < 0: - actual_tokens = 0 - with _lock: - b = _get_or_create_bucket(tenant_id) - if api_request_id not in b["reservations"]: - raise BudgetSettleError( - f"settle: no reservation for api_request_id={api_request_id!r} · " - "double settle or missing reserve" - ) - b["reservations"].pop(api_request_id) - b["used"] = b["used"] + actual_tokens + if actual_tokens < 0: + actual_tokens = 0 + with _lock: + reservation_key = (tenant_id, api_request_id) + bucket_key = _reservation_bucket.get(reservation_key) + if bucket_key is None: + raise BudgetSettleError( + f"settle: no reservation for api_request_id={api_request_id!r} · " + "double settle or missing reserve" + ) + b = _get_or_create_bucket_by_key(bucket_key) + if api_request_id not in b["reservations"]: + # The index and bucket must move atomically under the same lock. + _reservation_bucket.pop(reservation_key, None) + raise BudgetSettleError( + f"settle: reservation index corrupt for api_request_id={api_request_id!r}" + ) + b["reservations"].pop(api_request_id) + _reservation_bucket.pop(reservation_key, None) + b["used"] = b["used"] + actual_tokens def release(tenant_id: str, api_request_id: str) -> bool: @@ -155,13 +174,14 @@ def release(tenant_id: str, api_request_id: str) -> bool: - 若 reservation 存在 · 移除 · 返回 True - 若 reservation 不存在 · **no-op** · 返回 False(便于 orchestrator 检测 double release) - 任何情况下 · used 不变 - """ - with _lock: - b = _get_or_create_bucket(tenant_id) - if api_request_id in b["reservations"]: - b["reservations"].pop(api_request_id) - return True - return False + """ + with _lock: + reservation_key = (tenant_id, api_request_id) + bucket_key = _reservation_bucket.pop(reservation_key, None) + if bucket_key is None: + return False + b = _get_or_create_bucket_by_key(bucket_key) + return b["reservations"].pop(api_request_id, None) is not None # ── 观察 helpers · 测试用 ───────────────────────────────────────────── @@ -176,15 +196,19 @@ def get_pending_total(tenant_id: str) -> int: return sum(_get_or_create_bucket(tenant_id)["reservations"].values()) -def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: - with _lock: - return _get_or_create_bucket(tenant_id)["reservations"].get(api_request_id) +def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: + with _lock: + bucket_key = _reservation_bucket.get((tenant_id, api_request_id)) + if bucket_key is None: + return None + return _get_or_create_bucket_by_key(bucket_key)["reservations"].get(api_request_id) -def reset_state() -> None: - """测试用 · 清空全部 bucket。""" - with _lock: - _bucket.clear() +def reset_state() -> None: + """测试用 · 清空全部 bucket。""" + with _lock: + _bucket.clear() + _reservation_bucket.clear() # ── register: 只挂 observer hooks · 无 enforcement · 无 middleware ──── @@ -206,4 +230,3 @@ def register(ctx: Any) -> None: """ ctx.register_hook("pre_api_request", _on_pre_api_request_observer) ctx.register_hook("post_api_request", _on_post_api_request_observer) - \ No newline at end of file From 7181b63fed7a520ceacac895ab3a658e34179ea9 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:03:54 +0800 Subject: [PATCH 13/19] fix(spike): independent review hardening (2/7) --- .../tests/test_budget_reserve.py | 660 +++++++++--------- 1 file changed, 346 insertions(+), 314 deletions(-) diff --git a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py index 7182d9789645e..83d5bfeaeb34c 100644 --- a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py +++ b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py @@ -1,324 +1,356 @@ -"""P3-M0 Spike · budget service 测试(observer hook · service API). - -**Codex 第三轮 §IV 契约(修订)**: -- settle 仅结算**仍存在**的 reservation · unknown/double settle → raise BudgetSettleError · **不改** used -- release 幂等 · 返回 True(存在)/ False(不存在)· 不改 used -- 40 线程 barrier 精确超限测试:budget = N * per-req · 超过 N 个必 raise -- UTC 跨日 rollover 真测试(monkeypatch `_today_utc`) -- observer hooks 恒 no-op(无 enforcement) -""" -from __future__ import annotations - -import threading - -import pytest - -from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( - get_discovered_module, - install_fresh_manager, - write_config, -) - - -BUDGET_KEY = "spike-p3-m0-budget" - - -def _enable_budget(tmp_path, monkeypatch): - write_config(tmp_path, [BUDGET_KEY]) - manager = install_fresh_manager(monkeypatch, tmp_path) - mod = get_discovered_module(manager, BUDGET_KEY) - mod.reset_state() - return manager, mod - - -# ── Registration(§VIII 真实 PluginManager 门槛) ────────────────── - - -def test_discovery_registers_observer_hooks(tmp_path, monkeypatch): - from hermes_cli.plugins import has_hook - - manager, mod = _enable_budget(tmp_path, monkeypatch) - assert has_hook("pre_api_request") is True - assert has_hook("post_api_request") is True - assert manager._middleware.get("llm_execution") in (None, []) - - -def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): - from hermes_cli.plugins import has_hook - - write_config(tmp_path, []) - install_fresh_manager(monkeypatch, tmp_path) - assert has_hook("pre_api_request") is False - - -# ── validate_daily_budget · 4 态 + 拒非法(§V) ───────────────────── - - -def test_validate_daily_budget_states(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - assert mod.validate_daily_budget(None) is None - assert mod.validate_daily_budget(0) == 0 - assert mod.validate_daily_budget(1000) == 1000 - - -@pytest.mark.parametrize( - "bad_value", - [True, False, -1, "1000", 1.5, [], {}], - ids=["true", "false", "negative", "string", "float", "list", "dict"], -) -def test_validate_daily_budget_rejects(tmp_path, monkeypatch, bad_value): - _, mod = _enable_budget(tmp_path, monkeypatch) - with pytest.raises(mod.BudgetConfigError): - mod.validate_daily_budget(bad_value) - - -# ── reserve / settle · 基本路径 ───────────────────────────────────── - - -def test_reserve_none_budget_no_op(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_a", 500, budget=None) - assert mod.get_pending_total("t1") == 0 - assert mod.get_reservation("t1", "req_a") is None - - -def test_reserve_then_settle_charges_actual(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_1", 300, budget=10_000) - assert mod.get_pending_total("t1") == 300 - mod.settle("t1", "req_1", 120) - assert mod.get_used("t1") == 120 - assert mod.get_pending_total("t1") == 0 - - -def test_release_removes_reservation_returns_true(tmp_path, monkeypatch): - """release 存在的 reservation · 返回 True · used 不变。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_x", 200, budget=10_000) - assert mod.release("t1", "req_x") is True - assert mod.get_used("t1") == 0 - assert mod.get_pending_total("t1") == 0 - - -def test_release_unknown_returns_false_no_op(tmp_path, monkeypatch): - """Codex §IV:release unknown reservation · 返回 False · no-op(幂等)。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - assert mod.release("t1", "never_reserved") is False - assert mod.get_used("t1") == 0 - assert mod.get_pending_total("t1") == 0 - - -def test_release_after_release_returns_false(tmp_path, monkeypatch): - """double release · 第二次返回 False · 无副作用。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_x", 100, budget=1000) - assert mod.release("t1", "req_x") is True - assert mod.release("t1", "req_x") is False - assert mod.get_used("t1") == 0 - - -# ── reserve 超预算 · raise BudgetExceeded ──────────────────────── - - -def test_reserve_over_budget_raises(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - with pytest.raises(mod.BudgetExceeded, match="daily budget exhausted"): - mod.reserve("t1", "req_over", 200, budget=100) - assert mod.get_pending_total("t1") == 0 - - -def test_used_plus_pending_plus_new_over_budget_raises(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "r_a", 60, budget=200) - mod.settle("t1", "r_a", 60) - mod.reserve("t1", "r_b", 100, budget=200) - with pytest.raises(mod.BudgetExceeded): - mod.reserve("t1", "r_c", 50, budget=200) - - -def test_budget_zero_is_unlimited(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "r_big", 10_000_000, budget=0) - mod.settle("t1", "r_big", 5_000_000) - assert mod.get_used("t1") == 5_000_000 - - -# ── api_request_id 唯一性 ───────────────────────────────────────── - - -def test_duplicate_reserve_raises(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_dup", 50, budget=1000) - with pytest.raises(mod.BudgetExceeded, match="duplicate"): - mod.reserve("t1", "req_dup", 50, budget=1000) - - -def test_empty_api_request_id_raises(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - with pytest.raises(mod.BudgetExceeded, match="anonymous"): - mod.reserve("t1", "", 50, budget=1000) - - -# ── settle 严格幂等(Codex §IV 修订)───────────────────────────── - - -def test_settle_unknown_id_raises_and_does_not_charge(tmp_path, monkeypatch): - """Codex §IV 修订:settle 无 reservation · **raise BudgetSettleError · 不改 used**。 - - 旧契约"unknown settle 累加 used"是**错误**的 · 会导致 unknown id 洗钱。 - """ - _, mod = _enable_budget(tmp_path, monkeypatch) - with pytest.raises(mod.BudgetSettleError, match="no reservation"): - mod.settle("t1", "unknown_req", 25) - assert mod.get_used("t1") == 0 - - -def test_double_settle_raises_after_first_ok(tmp_path, monkeypatch): - """double settle · 第二次 raise · used 不再加。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_1", 50, budget=1000) - mod.settle("t1", "req_1", 50) # 首次 OK - assert mod.get_used("t1") == 50 - with pytest.raises(mod.BudgetSettleError): - mod.settle("t1", "req_1", 999) # 第二次 · reservation 已消 · raise - assert mod.get_used("t1") == 50 # 未增 - - -def test_settle_after_release_raises(tmp_path, monkeypatch): - """release 后 settle · reservation 不存在 · raise · used 不变。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("t1", "req_1", 50, budget=1000) - mod.release("t1", "req_1") - with pytest.raises(mod.BudgetSettleError): - mod.settle("t1", "req_1", 25) - assert mod.get_used("t1") == 0 - - -# ── 多 tenant 隔离 ────────────────────────────────────────────────── - - -def test_multiple_tenants_isolated(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - mod.reserve("tA", "r1", 40, budget=100) - mod.settle("tA", "r1", 40) - mod.reserve("tB", "r2", 90, budget=100) - mod.settle("tB", "r2", 90) - assert mod.get_used("tA") == 40 - assert mod.get_used("tB") == 90 - - -# ── 并发 reserve/settle · 无 race ──────────────────────────────── - - -def test_concurrent_reserve_settle_thread_safe(tmp_path, monkeypatch): - _, mod = _enable_budget(tmp_path, monkeypatch) - N = 40 - barrier = threading.Barrier(N) - - def worker(i): - barrier.wait() - mod.reserve("t_shared", f"req_{i}", 10, budget=10_000) - mod.settle("t_shared", f"req_{i}", 5) - - threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)] - for t in threads: - t.start() - for t in threads: - t.join() - - assert mod.get_used("t_shared") == N * 5 - assert mod.get_pending_total("t_shared") == 0 - - -# ── 真并发超限:Codex §IV 强要求 · 精确 N 成功 / rest 失败 ───────── - - -def test_concurrent_reserve_precise_success_and_exceeded_counts(tmp_path, monkeypatch): - """40 线程 barrier reserve · budget 只允许 N 个成功 · 精确 N 成功 · 40-N 失败。 - - Codex §IV:证明无超扣、无 race race-double-count。 - """ - _, mod = _enable_budget(tmp_path, monkeypatch) - N_THREADS = 40 - PER_REQ = 10 - N_SUCCESS = 15 # 只允许 15 个成功 - BUDGET = N_SUCCESS * PER_REQ # 150 - barrier = threading.Barrier(N_THREADS) - successes = [] - exceeded = [] - success_lock = threading.Lock() - - def worker(i): - barrier.wait() - try: - mod.reserve("t_precise", f"r{i}", PER_REQ, budget=BUDGET) - with success_lock: - successes.append(i) - except mod.BudgetExceeded: - with success_lock: - exceeded.append(i) - - threads = [threading.Thread(target=worker, args=(i,)) for i in range(N_THREADS)] - for t in threads: - t.start() - for t in threads: - t.join() - - assert len(successes) == N_SUCCESS, ( - f"expected exactly {N_SUCCESS} successes · got {len(successes)}" - ) - assert len(exceeded) == N_THREADS - N_SUCCESS - assert mod.get_pending_total("t_precise") == BUDGET # 全部成功者仍 pending - - # settle 全部成功者 · 无超扣 - for i in successes: - mod.settle("t_precise", f"r{i}", PER_REQ) - assert mod.get_used("t_precise") == BUDGET - assert mod.get_pending_total("t_precise") == 0 - - -# ── UTC 跨日 rollover 真测试(Codex §IV) ──────────────────────── - - +"""P3-M0 Spike · budget service 测试(observer hook · service API). + +**Codex 第三轮 §IV 契约(修订)**: +- settle 仅结算**仍存在**的 reservation · unknown/double settle → raise BudgetSettleError · **不改** used +- release 幂等 · 返回 True(存在)/ False(不存在)· 不改 used +- 40 线程 barrier 精确超限测试:budget = N * per-req · 超过 N 个必 raise +- UTC 跨日 rollover 真测试(monkeypatch `_today_utc`) +- observer hooks 恒 no-op(无 enforcement) +""" +from __future__ import annotations + +import threading + +import pytest + +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + write_config, +) + + +BUDGET_KEY = "spike-p3-m0-budget" + + +def _enable_budget(tmp_path, monkeypatch): + write_config(tmp_path, [BUDGET_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, BUDGET_KEY) + mod.reset_state() + return manager, mod + + +# ── Registration(§VIII 真实 PluginManager 门槛) ────────────────── + + +def test_discovery_registers_observer_hooks(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + manager, mod = _enable_budget(tmp_path, monkeypatch) + assert has_hook("pre_api_request") is True + assert has_hook("post_api_request") is True + assert manager._middleware.get("llm_execution") in (None, []) + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_hook("pre_api_request") is False + + +# ── validate_daily_budget · 4 态 + 拒非法(§V) ───────────────────── + + +def test_validate_daily_budget_states(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.validate_daily_budget(None) is None + assert mod.validate_daily_budget(0) == 0 + assert mod.validate_daily_budget(1000) == 1000 + + +@pytest.mark.parametrize( + "bad_value", + [True, False, -1, "1000", 1.5, [], {}], + ids=["true", "false", "negative", "string", "float", "list", "dict"], +) +def test_validate_daily_budget_rejects(tmp_path, monkeypatch, bad_value): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetConfigError): + mod.validate_daily_budget(bad_value) + + +# ── reserve / settle · 基本路径 ───────────────────────────────────── + + +def test_reserve_none_budget_no_op(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_a", 500, budget=None) + assert mod.get_pending_total("t1") == 0 + assert mod.get_reservation("t1", "req_a") is None + + +def test_reserve_then_settle_charges_actual(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 300, budget=10_000) + assert mod.get_pending_total("t1") == 300 + mod.settle("t1", "req_1", 120) + assert mod.get_used("t1") == 120 + assert mod.get_pending_total("t1") == 0 + + +def test_release_removes_reservation_returns_true(tmp_path, monkeypatch): + """release 存在的 reservation · 返回 True · used 不变。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 200, budget=10_000) + assert mod.release("t1", "req_x") is True + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +def test_release_unknown_returns_false_no_op(tmp_path, monkeypatch): + """Codex §IV:release unknown reservation · 返回 False · no-op(幂等)。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.release("t1", "never_reserved") is False + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +def test_release_after_release_returns_false(tmp_path, monkeypatch): + """double release · 第二次返回 False · 无副作用。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 100, budget=1000) + assert mod.release("t1", "req_x") is True + assert mod.release("t1", "req_x") is False + assert mod.get_used("t1") == 0 + + +# ── reserve 超预算 · raise BudgetExceeded ──────────────────────── + + +def test_reserve_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="daily budget exhausted"): + mod.reserve("t1", "req_over", 200, budget=100) + assert mod.get_pending_total("t1") == 0 + + +def test_used_plus_pending_plus_new_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_a", 60, budget=200) + mod.settle("t1", "r_a", 60) + mod.reserve("t1", "r_b", 100, budget=200) + with pytest.raises(mod.BudgetExceeded): + mod.reserve("t1", "r_c", 50, budget=200) + + +def test_budget_zero_is_unlimited(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_big", 10_000_000, budget=0) + mod.settle("t1", "r_big", 5_000_000) + assert mod.get_used("t1") == 5_000_000 + + +# ── api_request_id 唯一性 ───────────────────────────────────────── + + +def test_duplicate_reserve_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_dup", 50, budget=1000) + with pytest.raises(mod.BudgetExceeded, match="duplicate"): + mod.reserve("t1", "req_dup", 50, budget=1000) + + +def test_empty_api_request_id_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="anonymous"): + mod.reserve("t1", "", 50, budget=1000) + + +# ── settle 严格幂等(Codex §IV 修订)───────────────────────────── + + +def test_settle_unknown_id_raises_and_does_not_charge(tmp_path, monkeypatch): + """Codex §IV 修订:settle 无 reservation · **raise BudgetSettleError · 不改 used**。 + + 旧契约"unknown settle 累加 used"是**错误**的 · 会导致 unknown id 洗钱。 + """ + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetSettleError, match="no reservation"): + mod.settle("t1", "unknown_req", 25) + assert mod.get_used("t1") == 0 + + +def test_double_settle_raises_after_first_ok(tmp_path, monkeypatch): + """double settle · 第二次 raise · used 不再加。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.settle("t1", "req_1", 50) # 首次 OK + assert mod.get_used("t1") == 50 + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 999) # 第二次 · reservation 已消 · raise + assert mod.get_used("t1") == 50 # 未增 + + +def test_settle_after_release_raises(tmp_path, monkeypatch): + """release 后 settle · reservation 不存在 · raise · used 不变。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.release("t1", "req_1") + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 25) + assert mod.get_used("t1") == 0 + + +# ── 多 tenant 隔离 ────────────────────────────────────────────────── + + +def test_multiple_tenants_isolated(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("tA", "r1", 40, budget=100) + mod.settle("tA", "r1", 40) + mod.reserve("tB", "r2", 90, budget=100) + mod.settle("tB", "r2", 90) + assert mod.get_used("tA") == 40 + assert mod.get_used("tB") == 90 + + +# ── 并发 reserve/settle · 无 race ──────────────────────────────── + + +def test_concurrent_reserve_settle_thread_safe(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + N = 40 + barrier = threading.Barrier(N) + + def worker(i): + barrier.wait() + mod.reserve("t_shared", f"req_{i}", 10, budget=10_000) + mod.settle("t_shared", f"req_{i}", 5) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert mod.get_used("t_shared") == N * 5 + assert mod.get_pending_total("t_shared") == 0 + + +# ── 真并发超限:Codex §IV 强要求 · 精确 N 成功 / rest 失败 ───────── + + +def test_concurrent_reserve_precise_success_and_exceeded_counts(tmp_path, monkeypatch): + """40 线程 barrier reserve · budget 只允许 N 个成功 · 精确 N 成功 · 40-N 失败。 + + Codex §IV:证明无超扣、无 race race-double-count。 + """ + _, mod = _enable_budget(tmp_path, monkeypatch) + N_THREADS = 40 + PER_REQ = 10 + N_SUCCESS = 15 # 只允许 15 个成功 + BUDGET = N_SUCCESS * PER_REQ # 150 + barrier = threading.Barrier(N_THREADS) + successes = [] + exceeded = [] + success_lock = threading.Lock() + + def worker(i): + barrier.wait() + try: + mod.reserve("t_precise", f"r{i}", PER_REQ, budget=BUDGET) + with success_lock: + successes.append(i) + except mod.BudgetExceeded: + with success_lock: + exceeded.append(i) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(successes) == N_SUCCESS, ( + f"expected exactly {N_SUCCESS} successes · got {len(successes)}" + ) + assert len(exceeded) == N_THREADS - N_SUCCESS + assert mod.get_pending_total("t_precise") == BUDGET # 全部成功者仍 pending + + # settle 全部成功者 · 无超扣 + for i in successes: + mod.settle("t_precise", f"r{i}", PER_REQ) + assert mod.get_used("t_precise") == BUDGET + assert mod.get_pending_total("t_precise") == 0 + + +# ── UTC 跨日 rollover 真测试(Codex §IV) ──────────────────────── + + def test_utc_day_rollover_isolates_bucket(tmp_path, monkeypatch): - """monkeypatch `_today_utc` 模拟跨日 · 昨日 used 不影响今日 budget。""" - _, mod = _enable_budget(tmp_path, monkeypatch) - - day1 = "2026-08-09" - day2 = "2026-08-10" - - # Day 1 · 用满 budget=100 的 80 + """monkeypatch `_today_utc` 模拟跨日 · 昨日 used 不影响今日 budget。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + + day1 = "2026-08-09" + day2 = "2026-08-10" + + # Day 1 · 用满 budget=100 的 80 + monkeypatch.setattr(mod, "_today_utc", lambda: day1) + mod.reserve("t_roll", "r_d1", 80, budget=100) + mod.settle("t_roll", "r_d1", 80) + assert mod.get_used("t_roll") == 80 + + # Day 2 · 昨日 used 不算入今日 · 今日仍可 reserve 满 100 + monkeypatch.setattr(mod, "_today_utc", lambda: day2) + # 新 bucket · used 归 0 + assert mod.get_used("t_roll") == 0 + mod.reserve("t_roll", "r_d2", 100, budget=100) + mod.settle("t_roll", "r_d2", 100) + assert mod.get_used("t_roll") == 100 + + # 回到 Day 1 · 昨日 bucket 仍 80(未被今日污染) monkeypatch.setattr(mod, "_today_utc", lambda: day1) - mod.reserve("t_roll", "r_d1", 80, budget=100) - mod.settle("t_roll", "r_d1", 80) assert mod.get_used("t_roll") == 80 - # Day 2 · 昨日 used 不算入今日 · 今日仍可 reserve 满 100 - monkeypatch.setattr(mod, "_today_utc", lambda: day2) - # 新 bucket · used 归 0 - assert mod.get_used("t_roll") == 0 - mod.reserve("t_roll", "r_d2", 100, budget=100) - mod.settle("t_roll", "r_d2", 100) - assert mod.get_used("t_roll") == 100 - - # 回到 Day 1 · 昨日 bucket 仍 80(未被今日污染) - monkeypatch.setattr(mod, "_today_utc", lambda: day1) - assert mod.get_used("t_roll") == 80 +def test_settle_after_utc_rollover_charges_original_bucket(tmp_path, monkeypatch): + """跨零点 provider 调用必须结算创建 reservation 的原 UTC 日。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + current_day = {"value": "2026-08-09"} + monkeypatch.setattr(mod, "_today_utc", lambda: current_day["value"]) -# ── observer hooks 不 enforce(§I 契约) ───────────────────────── + mod.reserve("t_cross", "req_cross", 80, budget=100) + current_day["value"] = "2026-08-10" + mod.settle("t_cross", "req_cross", 60) + assert mod.get_used("t_cross") == 0 + assert mod.get_reservation("t_cross", "req_cross") is None + current_day["value"] = "2026-08-09" + assert mod.get_used("t_cross") == 60 + assert mod.get_pending_total("t_cross") == 0 -def test_pre_api_request_observer_never_enforces(tmp_path, monkeypatch): - from hermes_cli.plugins import invoke_hook +def test_release_after_utc_rollover_clears_original_bucket(tmp_path, monkeypatch): + """跨零点异常清理不得把昨日 pending reservation 永久遗留。""" _, mod = _enable_budget(tmp_path, monkeypatch) - results = invoke_hook( - "pre_api_request", - api_request_id="r1", - approx_input_tokens=999999, - max_tokens=999999, - ) - assert results == [] - assert mod.get_used("t_any") == 0 - assert mod.get_pending_total("t_any") == 0 + current_day = {"value": "2026-08-09"} + monkeypatch.setattr(mod, "_today_utc", lambda: current_day["value"]) + + mod.reserve("t_cross", "req_release_cross", 80, budget=100) + current_day["value"] = "2026-08-10" + assert mod.release("t_cross", "req_release_cross") is True + assert mod.get_reservation("t_cross", "req_release_cross") is None + current_day["value"] = "2026-08-09" + assert mod.get_pending_total("t_cross") == 0 + assert mod.get_used("t_cross") == 0 + + +# ── observer hooks 不 enforce(§I 契约) ───────────────────────── + + +def test_pre_api_request_observer_never_enforces(tmp_path, monkeypatch): + from hermes_cli.plugins import invoke_hook + + _, mod = _enable_budget(tmp_path, monkeypatch) + results = invoke_hook( + "pre_api_request", + api_request_id="r1", + approx_input_tokens=999999, + max_tokens=999999, + ) + assert results == [] + assert mod.get_used("t_any") == 0 + assert mod.get_pending_total("t_any") == 0 From 89656abdeca52dfbf9eed97fe378bd0243b927b8 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:04:05 +0800 Subject: [PATCH 14/19] fix(spike): independent review hardening (3/7) --- plugins/spike_p3_m0_cache/__init__.py | 145 +++++++++++++++++--------- 1 file changed, 96 insertions(+), 49 deletions(-) diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py index 88f3b08f1a43c..396562b682ab2 100644 --- a/plugins/spike_p3_m0_cache/__init__.py +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -21,10 +21,10 @@ ├─ cacheable_request → cache lookup │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) │ - ├─ (MISS 或非 cacheable_request)→ budget 非 null 时 reserve + ├─ (MISS 或 非 cacheable_request)→ budget.reserve │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) │ └─ 成功 → next_call - │ ├─ 成功 → usage 缺失时按 reservation 结算 + cache insert(若 req+resp 皆 cacheable) + │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) │ └─ 异常 → budget.release + re-raise 3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / @@ -35,7 +35,7 @@ bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 -5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` · cache insert/hit 均深复制防调用方污染 +5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` 6. 并发:threading.RLock 保护 LRU + insert 序 @@ -98,8 +98,8 @@ def _budget_module(): ) -CACHE_CONTRACT_VERSION = "v1" -CODEC_VERSION = "codec_v1" +CACHE_CONTRACT_VERSION = "v2" +CODEC_VERSION = "codec_v1" CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent # 允许 tenant_id/principal_id/permission_scope_version/profile_version 的字符白名单 @@ -262,17 +262,35 @@ def is_cacheable_response(response: Any) -> bool: # ── Cache key builder(§2 schema v1)────────────────────────────── -def build_cache_key( - request: Dict[str, Any], - tenant_ctx: Dict[str, str], - *, - model: str, - provider: str, -) -> str: - """SHA-256 of deterministic key components(12 字段 schema v1).""" - messages = request.get("messages") or [] - messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") - tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") +def build_cache_key( + request: Dict[str, Any], + tenant_ctx: Dict[str, str], + *, + model: str, + provider: str, + api_mode: str = "chat_completions", +) -> str: + """SHA-256 of deterministic semantic request components(schema v2). + + Besides identity, messages, and tool schema, every public request option + that can affect generation is hashed. This intentionally prefers a cache + miss over reusing a response produced with different sampling, output, + reasoning, or provider-specific options. + """ + messages = request.get("messages") or [] + messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") + tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") + generation_options = { + key: value + for key, value in request.items() + if key not in {"messages", "tools", "stream", "_kb_collection", "_kb_version"} + and not key.startswith("_") + } + generation_options_bytes = json.dumps( + generation_options, + sort_keys=True, + ensure_ascii=False, + ).encode("utf-8") # system prompt hash · 从 messages 首 system 消息内容 sys_msg_content = "" for m in messages: @@ -283,11 +301,13 @@ def build_cache_key( "tenant_id": tenant_ctx["tenant_id"], "principal_id": tenant_ctx["principal_id"], "permission_scope_version": tenant_ctx["permission_scope_version"], - "provider": provider or "", - "model": model or "", - "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], - "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], - "profile_version": tenant_ctx["profile_version"], + "provider": provider or "", + "model": model or "", + "api_mode": api_mode, + "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], + "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], + "generation_options_version": hashlib.sha256(generation_options_bytes).hexdigest(), + "profile_version": tenant_ctx["profile_version"], "knowledge_collection": str(request.get("_kb_collection") or ""), "knowledge_collection_version": str(request.get("_kb_version") or ""), "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), @@ -431,10 +451,17 @@ def cache_and_budget_middleware( 见模块 docstring §2 完整控制流。 """ - api_request_id = str(ctx.get("api_request_id") or "") - api_mode = str(ctx.get("api_mode") or "chat_completions") - provider = str(ctx.get("provider") or "") - model = str(ctx.get("model") or "") + raw_api_request_id = ctx.get("api_request_id") + raw_api_mode = ctx.get("api_mode") + raw_provider = ctx.get("provider") + raw_model = ctx.get("model") + context_values = (raw_api_request_id, raw_api_mode, raw_provider, raw_model) + if any(value is not None and not isinstance(value, str) for value in context_values): + return _fail_closed_response("", reason_code="invalid_execution_context") + api_request_id = raw_api_request_id or "" + api_mode = raw_api_mode or "chat_completions" + provider = raw_provider or "" + model = raw_model or "" # 1. Resolve config exactly once. A registered middleware whose config # cannot be read/validated must not be mistaken for a disabled plugin: @@ -501,9 +528,15 @@ def cache_and_budget_middleware( return _fail_closed_response(model, reason_code="unsupported_api_mode", api_mode=api_mode) # 5. request 类型门(non-dict 无法 estimate token · fail-CLOSED) - if not isinstance(request, dict): - logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) - return _fail_closed_response(model, reason_code="non_dict_request") + if not isinstance(request, dict): + logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) + return _fail_closed_response(model, reason_code="non_dict_request") + + # Provider and model are mandatory cache identity fields. Empty values + # would collapse unrelated upstream attempts into the same key. + if not provider or not model: + logger.warning("spike-cache: provider/model context missing · fail-CLOSED") + return _fail_closed_response(model, reason_code="missing_provider_or_model") # 6. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) try: @@ -535,12 +568,7 @@ def cache_and_budget_middleware( ) return _fail_closed_response(model, reason_code="budget_config_error") - # 8. api_request_id 门(reserve 需唯一 id · 空则无法 settle) - if not api_request_id: - logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") - return _fail_closed_response(model, reason_code="missing_api_request_id") - - # 9. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) + # 8. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) try: cacheable_req = is_cacheable_request(request, api_mode=api_mode) except Exception as exc: # noqa: BLE001 @@ -550,13 +578,19 @@ def cache_and_budget_middleware( ) return _fail_closed_response(model, reason_code="request_classification_error") - # 10. 若可缓存 → lookup · HIT 直接返回(不 reserve). Any internal + # 9. 若可缓存 → lookup · HIT 直接返回(不 reserve). Any internal # pre-provider failure is converted locally to fail-CLOSED so it cannot # escape into the outer middleware chain's fail-open policy. cache_key: Optional[str] = None if cacheable_req: try: - cache_key = build_cache_key(request, tenant_ctx, model=model, provider=provider) + cache_key = build_cache_key( + request, + tenant_ctx, + model=model, + provider=provider, + api_mode=api_mode, + ) with _lock: hit = _cache.get(cache_key) if hit is not None: @@ -571,18 +605,25 @@ def cache_and_budget_middleware( # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate return copy.deepcopy(hit) - # 11. MISS 或非 cacheable · 都必须 reserve(Codex §II step 6) - try: - tokens_needed = _estimate_needed_tokens(request, ctx) - except Exception as exc: # noqa: BLE001 - logger.error( - "spike-cache: token estimation failed · type=%s · fail-CLOSED", - type(exc).__name__, - ) - return _fail_closed_response(model, reason_code="token_estimation_error") + # 10. MISS 或非 cacheable · budget=None 明确表示不启用护栏，因此既不 + # 需要 api_request_id，也不做 token estimation/reserve/settle。 reservation_active = budget_value is not None + tokens_needed: Optional[int] = None + if reservation_active: + if not api_request_id: + logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") + return _fail_closed_response(model, reason_code="missing_api_request_id") + try: + tokens_needed = _estimate_needed_tokens(request, ctx) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: token estimation failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="token_estimation_error") try: if reservation_active: + assert tokens_needed is not None budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) except budget_mod.BudgetExceeded: logger.warning( @@ -624,6 +665,7 @@ def cache_and_budget_middleware( # 13. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) if reservation_active: + assert tokens_needed is not None actual = _extract_actual_usage(response) if actual is None: actual = tokens_needed @@ -692,10 +734,16 @@ def _extract_actual_usage(response: Any) -> Optional[int]: return None prompt = getattr(usage, "prompt_tokens", 0) completion = getattr(usage, "completion_tokens", 0) - if isinstance(prompt, bool) or isinstance(completion, bool): + if ( + isinstance(prompt, bool) + or isinstance(completion, bool) + or not isinstance(prompt, int) + or not isinstance(completion, int) + or prompt < 0 + or completion < 0 + ): return None - total = int(prompt or 0) + int(completion or 0) - return total if total >= 0 else None + return prompt + completion except Exception: # noqa: BLE001 return None @@ -714,4 +762,3 @@ def cache_size() -> int: def register(ctx: Any) -> None: """PluginContext register · 挂 llm_execution middleware(orchestrator).""" ctx.register_middleware("llm_execution", cache_and_budget_middleware) - From 4bf88c7901d78e57f7452f06b5c19427aa085990 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:04:17 +0800 Subject: [PATCH 15/19] fix(spike): independent review hardening (4/7) --- .../tests/test_cache_middleware.py | 182 ++++++++++++++---- 1 file changed, 149 insertions(+), 33 deletions(-) diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index 16e0c9f83ae04..9c2b47a68276c 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -432,6 +432,42 @@ def next_call(_req): assert calls == {"reserve": 0, "settle": 0, "next": 1} +def test_null_budget_requires_neither_request_id_nor_token_estimate(tmp_path, monkeypatch): + """null 明确关闭预算护栏；miss 不应被预算专用字段阻断。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=None) + calls = {"reserve": 0, "settle": 0, "next": 0} + monkeypatch.setattr( + budget_mod, + "reserve", + lambda *_args, **_kwargs: calls.__setitem__("reserve", calls["reserve"] + 1), + ) + monkeypatch.setattr( + budget_mod, + "settle", + lambda *_args, **_kwargs: calls.__setitem__("settle", calls["settle"] + 1), + ) + + def next_call(_req): + calls["next"] += 1 + return { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2}, + } + + run = make_run_llm_execution( + api_request_id="", + provider="openai", + model="m", + api_mode="chat_completions", + ) + response = run( + {"messages": [], "max_tokens": "not-needed-with-null-budget"}, + next_call, + ) + assert response["choices"][0]["message"]["content"] == "ok" + assert calls == {"reserve": 0, "settle": 0, "next": 1} + + def test_missing_usage_settles_reserved_estimate(tmp_path, monkeypatch): """provider 未回 usage 时按预留值结算，禁止按 0 洗预算。""" _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) @@ -452,38 +488,53 @@ def next_call(_req): assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 +@pytest.mark.parametrize( + "usage", + [ + {"prompt_tokens": "5", "completion_tokens": 2}, + {"prompt_tokens": 1.5, "completion_tokens": 2}, + {"prompt_tokens": -1, "completion_tokens": 2}, + {"prompt_tokens": True, "completion_tokens": 2}, + ], +) +def test_invalid_usage_types_settle_reserved_estimate(tmp_path, monkeypatch, usage): + """非严格非负整数 usage 不可信，必须按 reservation 保守结算。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + run = make_run_llm_execution( + api_request_id="req_invalid_usage", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=123, + max_tokens=77, + ) + run( + {"messages": [{"role": "user", "content": "x"}], "max_tokens": 77}, + lambda _req: { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": usage, + }, + ) + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 + + def test_cache_hit_returns_deep_copy(tmp_path, monkeypatch): """调用方修改一次 hit 结果不得污染后续缓存。""" _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) calls = {"next": 0} - def next_call(_req): - calls["next"] += 1 - return { - "model": "gpt-4o", - "choices": [{"message": {"content": "stable"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 5, "completion_tokens": 5}, - } - - req = {"messages": [{"role": "user", "content": "same"}]} - make_run_llm_execution( - api_request_id="copy_miss", provider="openai", model="gpt-4o", - api_mode="chat_completions", approx_input_tokens=10, - )(req, next_call) - first_hit = make_run_llm_execution( - api_request_id="copy_hit_1", provider="openai", model="gpt-4o", - api_mode="chat_completions", approx_input_tokens=10, - )(req, next_call) - first_hit["choices"][0]["message"]["content"] = "poisoned" - first_hit["cache_meta"]["origin_usage"]["prompt_tokens"] = 999 - - second_hit = make_run_llm_execution( - api_request_id="copy_hit_2", provider="openai", model="gpt-4o", - api_mode="chat_completions", approx_input_tokens=10, - )(req, next_call) + def next_call(_req…475 tokens truncated…mode="chat_completions", + approx_input_tokens=10, + )(request, next_call) + hit = make_run_llm_execution( + api_request_id="", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + )(request, next_call) assert calls["next"] == 1 - assert second_hit["choices"][0]["message"]["content"] == "stable" - assert second_hit["cache_meta"]["origin_usage"]["prompt_tokens"] == 5 + assert hit["cache_meta"]["hit"] is True # ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ @@ -600,7 +651,7 @@ def next_call(_req): assert resp["cache_meta"]["reason_code"] == "budget_config_invalid" -def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): +def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): """空 api_request_id · reserve 无法追踪 · fail-CLOSED。""" manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) counters = {"next": 0} @@ -613,8 +664,49 @@ def next_call(_req): api_request_id="", provider="p", model="m", api_mode="chat_completions", ) resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) - assert counters["next"] == 0 - assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" + + +@pytest.mark.parametrize( + ("provider", "model"), + [("", "m"), ("openai", "")], +) +def test_missing_provider_or_model_fail_closed( + tmp_path, monkeypatch, provider, model, +): + """缓存身份字段缺失时不得生成可碰撞的空值 key。""" + _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + + response = make_run_llm_execution( + api_request_id="req_identity", + provider=provider, + model=model, + api_mode="chat_completions", + )({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert response["cache_meta"]["reason_code"] == "missing_provider_or_model" + + +def test_non_string_execution_context_fail_closed(tmp_path, monkeypatch): + _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + response = make_run_llm_execution( + api_request_id="req_identity", + provider={"unexpected": "mapping"}, + model="m", + api_mode="chat_completions", + )( + {"messages": [{"role": "user", "content": "x"}]}, + lambda _req: counters.__setitem__("next", counters["next"] + 1), + ) + assert counters["next"] == 0 + assert response["cache_meta"]["reason_code"] == "invalid_execution_context" # ── Cache alone(budget disabled)· 硬依赖门 fail-CLOSED ───────────── @@ -654,13 +746,38 @@ def test_cache_key_stable_and_deterministic(tmp_path, monkeypatch): assert k1 == k2 and len(k1) == 64 -def test_cache_key_changes_on_provider(tmp_path, monkeypatch): +def test_cache_key_changes_on_provider(tmp_path, monkeypatch): manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) req = {"messages": []} ctx = dict(_VALID_TENANT) k1 = cache_mod.build_cache_key(req, ctx, model="m", provider="openai") - k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") - assert k1 != k2 + k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") + assert k1 != k2 + + +@pytest.mark.parametrize( + ("field", "first", "second"), + [ + ("temperature", 0, 0.8), + ("top_p", 0.9, 1.0), + ("max_tokens", 64, 128), + ("response_format", {"type": "text"}, {"type": "json_object"}), + ("seed", 1, 2), + ("extra_body", {"reasoning": {"effort": "low"}}, {"reasoning": {"effort": "high"}}), + ], +) +def test_cache_key_changes_on_generation_options( + tmp_path, monkeypatch, field, first, second, +): + """相同 messages 但生成语义不同，必须产生不同 cache key。""" + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + ctx = dict(_VALID_TENANT) + base = {"messages": [{"role": "user", "content": "same"}]} + req_a = dict(base, **{field: first}) + req_b = dict(base, **{field: second}) + key_a = cache_mod.build_cache_key(req_a, ctx, model="m", provider="openai") + key_b = cache_mod.build_cache_key(req_b, ctx, model="m", provider="openai") + assert key_a != key_b def test_cache_key_changes_on_tenant(tmp_path, monkeypatch): @@ -917,4 +1034,3 @@ def boom(_response): approx_input_tokens=10, ) assert run({"messages": [{"role": "user", "content": "x"}]}, lambda _req: expected) is expected - From 47b2009f9ad8dde5caa10bad1cc56eee5bd05d83 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:04:25 +0800 Subject: [PATCH 16/19] fix(spike): independent review hardening (5/7) --- plugins/spike_p3_m0_strip/__init__.py | 163 +++++++++++++------------- 1 file changed, 82 insertions(+), 81 deletions(-) diff --git a/plugins/spike_p3_m0_strip/__init__.py b/plugins/spike_p3_m0_strip/__init__.py index d2630ffdd8464..880e57ed54af1 100644 --- a/plugins/spike_p3_m0_strip/__init__.py +++ b/plugins/spike_p3_m0_strip/__init__.py @@ -1,83 +1,84 @@ -"""P3-M0 Spike · strip-sources prototype(exit ① 普通最终响应). - -**Codex §VI fail-closed 契约**: -- sanitizer 异常 → 返回**安全占位** · 记录 audit · **不**返回 None(否则原文透出) -- 无改动 · 返回 None(标准 hook "first non-None str wins" 契约) - -**范围**: -- 仅出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` -- streaming/interim/interrupt/history-replay/tool-result 五出口 fixture 冻结 - · **本 spike 不实装**(fixture present · implementation pending) -- `_sanitize_surrogates`(fresh baseline turn_finalizer 已加)独立 · 不冲突 - -**插件启用**:通过 fork `config.yaml` 的 `plugins.enabled` 列表 · **不**用 env。 -""" -from __future__ import annotations - -import logging -from typing import Any, Optional - -from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-closed 测试 - -logger = logging.getLogger(__name__) - -# fail-closed 占位 · 出现 sanitizer 崩时返回 · 提示用户重试 -FAIL_CLOSED_PLACEHOLDER = ( - "[response filtered · sanitizer error · please retry]" -) - - -def on_transform_llm_output( - *, - response_text: str, - session_id: str = "", - model: str = "", - platform: str = "", - **_kwargs: Any, -) -> Optional[str]: - """`transform_llm_output` hook · fail-closed on sanitizer exception. - - Contract(`hermes_cli/plugins.py::VALID_HOOKS`): - - 返回 non-empty str → 覆盖 `final_response`(first-wins 跨 plugins) - - 返回 None / "" → 保留 `final_response` 不变 - - Codex §VI · fail-closed 补丁: - - 无异常 · 无改动 → 返回 None - - 无异常 · 有改动 → 返回 stripped str - - 异常 → 返回 `FAIL_CLOSED_PLACEHOLDER`(**不返回原文**)· 记录 audit - """ - text = response_text or "" - try: - stripped = _sanitize_mod.sanitize_presentation(text, mode="final") - except Exception as exc: # noqa: BLE001 — fail-closed · 拒原文泄漏 +"""P3-M0 Spike · strip-sources prototype(exit ① 普通最终响应). + +**Codex §VI fail-closed 契约**: +- sanitizer 异常 → 返回**安全占位** · 记录 audit · **不**返回 None(否则原文透出) +- 无改动 · 返回 None(标准 hook "first non-None str wins" 契约) + +**范围**: +- 仅出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` +- streaming/interim/interrupt/history-replay/tool-result 五出口 fixture 冻结 + · **本 spike 不实装**(fixture present · implementation pending) +- `_sanitize_surrogates`(fresh baseline turn_finalizer 已加)独立 · 不冲突 + +**插件启用**:通过 fork `config.yaml` 的 `plugins.enabled` 列表 · **不**用 env。 +""" +from __future__ import annotations + +import logging +from typing import Any, Optional + +from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-closed 测试 + +logger = logging.getLogger(__name__) + +# fail-closed 占位 · 出现 sanitizer 崩时返回 · 提示用户重试 +FAIL_CLOSED_PLACEHOLDER = ( + "[response filtered · sanitizer error · please retry]" +) + + +def on_transform_llm_output( + *, + response_text: str, + session_id: str = "", + model: str = "", + platform: str = "", + **_kwargs: Any, +) -> Optional[str]: + """`transform_llm_output` hook · fail-closed on sanitizer exception. + + Contract(`hermes_cli/plugins.py::VALID_HOOKS`): + - 返回 non-empty str → 覆盖 `final_response`(first-wins 跨 plugins) + - 返回 None / "" → 保留 `final_response` 不变 + + Codex §VI · fail-closed 补丁: + - 无异常 · 无改动 → 返回 None + - 无异常 · 有改动 → 返回 stripped str + - 异常 → 返回 `FAIL_CLOSED_PLACEHOLDER`(**不返回原文**)· 记录 audit + """ + text = response_text or "" + try: + stripped = _sanitize_mod.sanitize_presentation(text, mode="final") + except Exception as exc: # noqa: BLE001 — fail-closed · 拒原文泄漏 logger.error( - "spike-p3-m0-strip: sanitizer FAILED · fail-closed (session_id=%s model=%s platform=%s): %s", - session_id, model, platform, exc, + "spike-p3-m0-strip: sanitizer FAILED · fail-closed " + "(session_id=%s model=%s platform=%s error_class=%s)", + session_id, model, platform, type(exc).__name__, ) - # audit event · 便于监控告警 - _emit_audit( - event="strip.sanitizer_failed", - session_id=session_id, - model=model, - platform=platform, - error_class=type(exc).__name__, - ) - return FAIL_CLOSED_PLACEHOLDER - if stripped == text: - return None # 无改动 - return stripped - - -def _emit_audit(**payload: Any) -> None: - """轻量 audit hook · 通过 lifecycle invoke_hook 触发观察者(如 langfuse/nemo_relay)。""" - try: - from hermes_cli.lifecycle import invoke_hook - - invoke_hook("api_request_error", **payload) # 复用现有 API error 通道 - except Exception: # noqa: BLE001 — audit never breaks hook - pass - - -def register(ctx: Any) -> None: - """PluginContext hook · called at plugin discovery.""" - ctx.register_hook("transform_llm_output", on_transform_llm_output) + # audit event · 便于监控告警 + _emit_audit( + event="strip.sanitizer_failed", + session_id=session_id, + model=model, + platform=platform, + error_class=type(exc).__name__, + ) + return FAIL_CLOSED_PLACEHOLDER + if stripped == text: + return None # 无改动 + return stripped + + +def _emit_audit(**payload: Any) -> None: + """轻量 audit hook · 通过 lifecycle invoke_hook 触发观察者(如 langfuse/nemo_relay)。""" + try: + from hermes_cli.lifecycle import invoke_hook + + invoke_hook("api_request_error", **payload) # 复用现有 API error 通道 + except Exception: # noqa: BLE001 — audit never breaks hook + pass + + +def register(ctx: Any) -> None: + """PluginContext hook · called at plugin discovery.""" + ctx.register_hook("transform_llm_output", on_transform_llm_output) From 5559bf51f483691f1216672ce9ac9793461ecb35 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:04:35 +0800 Subject: [PATCH 17/19] fix(spike): independent review hardening (6/7) --- .../tests/test_strip_final.py | 565 +++++++++--------- 1 file changed, 284 insertions(+), 281 deletions(-) diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_final.py b/plugins/spike_p3_m0_strip/tests/test_strip_final.py index b97d323b2b182..97249d35dfe59 100644 --- a/plugins/spike_p3_m0_strip/tests/test_strip_final.py +++ b/plugins/spike_p3_m0_strip/tests/test_strip_final.py @@ -1,286 +1,289 @@ -"""P3-M0 Spike · strip prototype 出口 ① 测试(**真实 PluginManager · invoke_hook**). - -**Codex §VI / §VIII 契约**: -- 通过临时 `config.yaml` 启用插件 · 真实 `PluginManager.discover_and_load` -- 通过 `hermes_cli.plugins.invoke_hook("transform_llm_output", ...)` 触发 -- 覆盖: - * fixture SHA-256 双仓一致(cross-repo lock) - * final-mode 全 fixture case parametrize · sanitize_presentation 直调 - * 有改动 hook 返回 stripped str · 无改动返回 None - * sanitizer 异常 → hook 返回 FAIL_CLOSED_PLACEHOLDER(**fail-closed** · 不返回原文) - * negative case:bare [N] / JSON / footnote ref 等**必须**原样保留 - * streaming byte-split · UTF-8 缓冲 · sanitize 结果与整体一致 -""" -from __future__ import annotations - -import hashlib -from pathlib import Path - -import pytest - -from plugins.spike_p3_m0_fixtures.citation_cases import ( - ALL_CASES, - StripCase, - cases_by_marker, - cases_by_mode, - enumerate_byte_split_chunks, -) -from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( - assert_no_plugin_module_leak, - get_discovered_module, - install_fresh_manager, - snapshot_plugin_modules, - write_config, -) - - -STRIP_KEY = "spike-p3-m0-strip" -_HERE = Path(__file__).parent -_FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" - -# 与 Hermes_AI `tests/test_p3_m0_fixture_gate.py::EXPECTED_SHA256` 完全一致 -# **规范 LF 字节** SHA-256(Git normalized · 见 fork .gitattributes 声明) -EXPECTED_FIXTURE_SHA256 = ( - "47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666" -) - - -def _canonical_lf_bytes(path: Path) -> bytes: - """Read file bytes with CRLF/CR line endings normalized to LF. - - Cross-repo SHA gate contract locks the **canonical LF bytes** (not disk - bytes) so Windows autocrlf checkout does not produce a false SHA mismatch. - Paired with fork `.gitattributes text eol=lf` rule for this file. - """ - raw = path.read_bytes() - return raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - - -# ── Fixture SHA-256 校验(跨仓一致性 门槛 · 规范 LF 字节) ───── - - -def test_fixture_sha256_matches_hermes_ai(): - fixture_path = _FIXTURE_DIR / "citation_cases.py" - actual = hashlib.sha256(_canonical_lf_bytes(fixture_path)).hexdigest() - assert actual == EXPECTED_FIXTURE_SHA256, ( - f"fixture canonical LF SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " - f"actual: {actual}\n跨仓 fixture 不一致 · 需双仓同步更新" - ) - - -def test_fixture_sha256_crlf_input_normalizes(): - """CRLF checkout(Windows autocrlf 模拟)仍匹配 SHA gate · 契约锁 LF 字节。""" - fixture_path = _FIXTURE_DIR / "citation_cases.py" - lf_bytes = fixture_path.read_bytes() - if b"\r\n" in lf_bytes: - pytest.skip("checkout 本就是 CRLF · 直接由 test_fixture_sha256_matches_hermes_ai 覆盖") - crlf_bytes = lf_bytes.replace(b"\n", b"\r\n") - normalized = crlf_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - assert hashlib.sha256(normalized).hexdigest() == EXPECTED_FIXTURE_SHA256, ( - "CRLF → LF 规范化后 SHA 与冻结值不符" - ) - - -# ── Discovery / registration(§VIII) ──────────────────────────── - - -def _enable_strip(tmp_path, monkeypatch): - write_config(tmp_path, [STRIP_KEY]) - manager = install_fresh_manager(monkeypatch, tmp_path) - mod = get_discovered_module(manager, STRIP_KEY) - return manager, mod - - -def test_discover_registers_transform_llm_output_hook(tmp_path, monkeypatch): - from hermes_cli.plugins import has_hook - - manager, mod = _enable_strip(tmp_path, monkeypatch) - assert has_hook("transform_llm_output") is True - - -def test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue(tmp_path): - """Codex 第三轮 §VI:证明 fixture teardown 后 sys.modules 完全恢复(names + object ids)。 - - 比 name-only 快照更强的契约:每个 hermes_plugins.* 键的 module OBJECT 在 teardown - 后必须还是**同一实例**(delitem+monkeypatch.undo 恢复原对象)。若 discover 新建的 - module 对象泄漏 · id() 会不同 · 直接暴露污染。 - """ - import sys - import pytest as _pytest - - before_names = snapshot_plugin_modules() - before_ids = {k: id(sys.modules[k]) for k in before_names} - - mp = _pytest.MonkeyPatch() - finalizers = [] - - class _FakeRequest: - def addfinalizer(self, fn): - finalizers.append(fn) - - try: - write_config(tmp_path, [STRIP_KEY]) - install_fresh_manager(mp, tmp_path, request=_FakeRequest()) - # 验证 discover 确实加载了(要么新键 · 要么替换了 object id) - during_names = snapshot_plugin_modules() - during_ids = {k: id(sys.modules[k]) for k in during_names} - discover_loaded = (during_names - before_names) or { - k for k in before_names & during_names if during_ids.get(k) != before_ids.get(k) - } - assert discover_loaded, "install_fresh_manager 未加载 · 也未替换任何 hermes_plugins.*" - finally: - # 手工 teardown:先 finalizers · 再 monkeypatch undo - for fn in reversed(finalizers): - fn() - mp.undo() - - # 快照完全恢复:names 与 object ids 都必须与 before 一致 - after_names = snapshot_plugin_modules() - assert after_names == before_names, ( - f"name leak · new keys: {sorted(after_names - before_names)} · " - f"missing keys: {sorted(before_names - after_names)}" - ) - for k in before_names: - assert id(sys.modules[k]) == before_ids[k], ( - f"hermes_plugins module {k!r} object identity changed after teardown · " - "discover-inserted new object leaked past finalizer + monkeypatch.undo" - ) - - -def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): - from hermes_cli.plugins import has_hook - - write_config(tmp_path, []) - install_fresh_manager(monkeypatch, tmp_path) - assert has_hook("transform_llm_output") is False - - -# ── final-mode sanitize_presentation 全 fixture parametrize ──── - - -@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) -def test_sanitize_final_mode(case: StripCase): - """全 final mode fixture case · sanitize_presentation(mode='final') 正确。""" - from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation - - got = sanitize_presentation(case.raw, mode="final") - assert got == case.expected, ( - f"case={case.name}\n raw: {case.raw!r}\n " - f"expected: {case.expected!r}\n got: {got!r}\n notes: {case.notes}" - ) - - -# ── Negative case:bare [N] / JSON / footnote 必须原样保留(§VI 硬约束) ── - - -@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) -def test_negative_cases_preserved(case: StripCase): - """negative case:sanitize 不得吞掉 · 原样返回。""" - from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation - - got = sanitize_presentation(case.raw, mode="final") - assert got == case.raw, ( - f"NEGATIVE case={case.name} 被误 strip · raw={case.raw!r} got={got!r}" - ) - - -# ── Hook 契约(通过 invoke_hook · 真实 PluginManager 走位) ──────── - - -@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) -def test_transform_llm_output_hook_via_invoke_hook(tmp_path, monkeypatch, case: StripCase): - from hermes_cli.plugins import invoke_hook - - _enable_strip(tmp_path, monkeypatch) - results = invoke_hook( - "transform_llm_output", - response_text=case.raw, - session_id="s_test", - model="m", - platform="cli", - ) - if case.raw == case.expected: - assert results == [], f"unchanged input · hook 应返回 None(过滤后空 list)· case={case.name}" - else: - assert results == [case.expected], ( - f"case={case.name} · expected [{case.expected!r}] · got {results!r}" - ) - - -def test_hook_empty_input_returns_none(tmp_path, monkeypatch): - from hermes_cli.plugins import invoke_hook - - _enable_strip(tmp_path, monkeypatch) - assert invoke_hook("transform_llm_output", response_text="") == [] - - -# ── fail-CLOSED on sanitizer exception(§VI 硬约束) ───────────── - - -def test_hook_fail_closed_on_sanitizer_exception(tmp_path, monkeypatch): - """sanitizer 崩 · hook 返回 FAIL_CLOSED_PLACEHOLDER · **不**返回原文。 - - Codex §VI:sanitizer 失败必须拒原文透出 · 返回安全占位符 · 记录 audit。 - """ - from hermes_cli.plugins import invoke_hook - - _, strip_mod = _enable_strip(tmp_path, monkeypatch) - from plugins.spike_p3_m0_fixtures.pluginmgr_helper import get_discovered_module - # 从 discover 后的 module 修改 sanitize 依赖(fresh manager 下的 module) - sanitize_dep = strip_mod._sanitize_mod +"""P3-M0 Spike · strip prototype 出口 ① 测试(**真实 PluginManager · invoke_hook**). + +**Codex §VI / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 真实 `PluginManager.discover_and_load` +- 通过 `hermes_cli.plugins.invoke_hook("transform_llm_output", ...)` 触发 +- 覆盖: + * fixture SHA-256 双仓一致(cross-repo lock) + * final-mode 全 fixture case parametrize · sanitize_presentation 直调 + * 有改动 hook 返回 stripped str · 无改动返回 None + * sanitizer 异常 → hook 返回 FAIL_CLOSED_PLACEHOLDER(**fail-closed** · 不返回原文) + * negative case:bare [N] / JSON / footnote ref 等**必须**原样保留 + * streaming byte-split · UTF-8 缓冲 · sanitize 结果与整体一致 +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from plugins.spike_p3_m0_fixtures.citation_cases import ( + ALL_CASES, + StripCase, + cases_by_marker, + cases_by_mode, + enumerate_byte_split_chunks, +) +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + assert_no_plugin_module_leak, + get_discovered_module, + install_fresh_manager, + snapshot_plugin_modules, + write_config, +) + + +STRIP_KEY = "spike-p3-m0-strip" +_HERE = Path(__file__).parent +_FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" + +# 与 Hermes_AI `tests/test_p3_m0_fixture_gate.py::EXPECTED_SHA256` 完全一致 +# **规范 LF 字节** SHA-256(Git normalized · 见 fork .gitattributes 声明) +EXPECTED_FIXTURE_SHA256 = ( + "47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666" +) + + +def _canonical_lf_bytes(path: Path) -> bytes: + """Read file bytes with CRLF/CR line endings normalized to LF. + + Cross-repo SHA gate contract locks the **canonical LF bytes** (not disk + bytes) so Windows autocrlf checkout does not produce a false SHA mismatch. + Paired with fork `.gitattributes text eol=lf` rule for this file. + """ + raw = path.read_bytes() + return raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +# ── Fixture SHA-256 校验(跨仓一致性 门槛 · 规范 LF 字节) ───── + + +def test_fixture_sha256_matches_hermes_ai(): + fixture_path = _FIXTURE_DIR / "citation_cases.py" + actual = hashlib.sha256(_canonical_lf_bytes(fixture_path)).hexdigest() + assert actual == EXPECTED_FIXTURE_SHA256, ( + f"fixture canonical LF SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " + f"actual: {actual}\n跨仓 fixture 不一致 · 需双仓同步更新" + ) + + +def test_fixture_sha256_crlf_input_normalizes(): + """CRLF checkout(Windows autocrlf 模拟)仍匹配 SHA gate · 契约锁 LF 字节。""" + fixture_path = _FIXTURE_DIR / "citation_cases.py" + lf_bytes = fixture_path.read_bytes() + if b"\r\n" in lf_bytes: + pytest.skip("checkout 本就是 CRLF · 直接由 test_fixture_sha256_matches_hermes_ai 覆盖") + crlf_bytes = lf_bytes.replace(b"\n", b"\r\n") + normalized = crlf_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + assert hashlib.sha256(normalized).hexdigest() == EXPECTED_FIXTURE_SHA256, ( + "CRLF → LF 规范化后 SHA 与冻结值不符" + ) + + +# ── Discovery / registration(§VIII) ──────────────────────────── + + +def _enable_strip(tmp_path, monkeypatch): + write_config(tmp_path, [STRIP_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, STRIP_KEY) + return manager, mod + + +def test_discover_registers_transform_llm_output_hook(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + manager, mod = _enable_strip(tmp_path, monkeypatch) + assert has_hook("transform_llm_output") is True + + +def test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue(tmp_path): + """Codex 第三轮 §VI:证明 fixture teardown 后 sys.modules 完全恢复(names + object ids)。 + + 比 name-only 快照更强的契约:每个 hermes_plugins.* 键的 module OBJECT 在 teardown + 后必须还是**同一实例**(delitem+monkeypatch.undo 恢复原对象)。若 discover 新建的 + module 对象泄漏 · id() 会不同 · 直接暴露污染。 + """ + import sys + import pytest as _pytest + + before_names = snapshot_plugin_modules() + before_ids = {k: id(sys.modules[k]) for k in before_names} + + mp = _pytest.MonkeyPatch() + finalizers = [] + + class _FakeRequest: + def addfinalizer(self, fn): + finalizers.append(fn) + + try: + write_config(tmp_path, [STRIP_KEY]) + install_fresh_manager(mp, tmp_path, request=_FakeRequest()) + # 验证 discover 确实加载了(要么新键 · 要么替换了 object id) + during_names = snapshot_plugin_modules() + during_ids = {k: id(sys.modules[k]) for k in during_names} + discover_loaded = (during_names - before_names) or { + k for k in before_names & during_names if during_ids.get(k) != before_ids.get(k) + } + assert discover_loaded, "install_fresh_manager 未加载 · 也未替换任何 hermes_plugins.*" + finally: + # 手工 teardown:先 finalizers · 再 monkeypatch undo + for fn in reversed(finalizers): + fn() + mp.undo() + + # 快照完全恢复:names 与 object ids 都必须与 before 一致 + after_names = snapshot_plugin_modules() + assert after_names == before_names, ( + f"name leak · new keys: {sorted(after_names - before_names)} · " + f"missing keys: {sorted(before_names - after_names)}" + ) + for k in before_names: + assert id(sys.modules[k]) == before_ids[k], ( + f"hermes_plugins module {k!r} object identity changed after teardown · " + "discover-inserted new object leaked past finalizer + monkeypatch.undo" + ) + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_hook("transform_llm_output") is False + + +# ── final-mode sanitize_presentation 全 fixture parametrize ──── + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_sanitize_final_mode(case: StripCase): + """全 final mode fixture case · sanitize_presentation(mode='final') 正确。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + got = sanitize_presentation(case.raw, mode="final") + assert got == case.expected, ( + f"case={case.name}\n raw: {case.raw!r}\n " + f"expected: {case.expected!r}\n got: {got!r}\n notes: {case.notes}" + ) + + +# ── Negative case:bare [N] / JSON / footnote 必须原样保留(§VI 硬约束) ── + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_preserved(case: StripCase): + """negative case:sanitize 不得吞掉 · 原样返回。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + got = sanitize_presentation(case.raw, mode="final") + assert got == case.raw, ( + f"NEGATIVE case={case.name} 被误 strip · raw={case.raw!r} got={got!r}" + ) + + +# ── Hook 契约(通过 invoke_hook · 真实 PluginManager 走位) ──────── + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_transform_llm_output_hook_via_invoke_hook(tmp_path, monkeypatch, case: StripCase): + from hermes_cli.plugins import invoke_hook + + _enable_strip(tmp_path, monkeypatch) + results = invoke_hook( + "transform_llm_output", + response_text=case.raw, + session_id="s_test", + model="m", + platform="cli", + ) + if case.raw == case.expected: + assert results == [], f"unchanged input · hook 应返回 None(过滤后空 list)· case={case.name}" + else: + assert results == [case.expected], ( + f"case={case.name} · expected [{case.expected!r}] · got {results!r}" + ) + + +def test_hook_empty_input_returns_none(tmp_path, monkeypatch): + from hermes_cli.plugins import invoke_hook + + _enable_strip(tmp_path, monkeypatch) + assert invoke_hook("transform_llm_output", response_text="") == [] + + +# ── fail-CLOSED on sanitizer exception(§VI 硬约束) ───────────── + + +def test_hook_fail_closed_on_sanitizer_exception(tmp_path, monkeypatch, caplog): + """sanitizer 崩 · hook 返回 FAIL_CLOSED_PLACEHOLDER · **不**返回原文。 + + Codex §VI:sanitizer 失败必须拒原文透出 · 返回安全占位符 · 记录 audit。 + """ + from hermes_cli.plugins import invoke_hook + + _, strip_mod = _enable_strip(tmp_path, monkeypatch) + from plugins.spike_p3_m0_fixtures.pluginmgr_helper import get_discovered_module + # 从 discover 后的 module 修改 sanitize 依赖(fresh manager 下的 module) + sanitize_dep = strip_mod._sanitize_mod + + internal_secret = "TOP_SECRET_SANITIZER_PAYLOAD" def boom(*_args, **_kwargs): - raise RuntimeError("simulated sanitizer crash") - - monkeypatch.setattr(sanitize_dep, "sanitize_presentation", boom) - - dangerous_text = "SECRET INTERNAL DATA · SHOULD NOT LEAK · [source: internal]" - results = invoke_hook( - "transform_llm_output", - response_text=dangerous_text, - session_id="s_test", - model="m", - platform="cli", - ) - # 关键断言:不返回原文 · 返回占位符 + raise RuntimeError(internal_secret) + + monkeypatch.setattr(sanitize_dep, "sanitize_presentation", boom) + + dangerous_text = "SECRET INTERNAL DATA · SHOULD NOT LEAK · [source: internal]" + results = invoke_hook( + "transform_llm_output", + response_text=dangerous_text, + session_id="s_test", + model="m", + platform="cli", + ) + # 关键断言:不返回原文 · 返回占位符 assert len(results) == 1 assert results[0] == strip_mod.FAIL_CLOSED_PLACEHOLDER assert dangerous_text not in results[0], "fail-closed 泄漏原文 · 违 §VI 硬约束" - - -# ── 其他 mode 契约冻结(skeleton pass-through · M1+ 实装) ────── - - -@pytest.mark.parametrize("mode", ["history", "streaming", "interim", "tool"]) -def test_non_final_modes_skeleton_pass_through(mode: str): - from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation - - text = "Answer with citation [source: kb_2024]. More text." - got = sanitize_presentation(text, mode=mode) - assert got == text, f"mode={mode} 应 pass-through(M0 skeleton · 见 §VI 冻结)" - - -# ── Streaming byte-split · S1 完整缓冲后 sanitize 与整体一致 ─── - - -@pytest.mark.parametrize("case", cases_by_marker("streaming_split"), ids=lambda c: c.name) -def test_streaming_full_buffer_matches_final(case: StripCase): - """S1 策略验证:任意字节切分点分片 → 缓冲拼回 → UTF-8 decode → final sanitize - → 结果必须等于对整体 raw 直接 final sanitize 的结果(与 case.expected 一致)。 - """ - from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation - - expected = sanitize_presentation(case.raw, mode="final") - assert expected == case.expected, ( - f"streaming_split case={case.name} · 直接 final sanitize 与 fixture expected 不一致 · " - f"expected={case.expected!r} got={expected!r}" - ) - # 每个切分点:两段拼接 · decode · sanitize · 与 expected 一致 - for head, tail in enumerate_byte_split_chunks(case.raw): - buf = head + tail - text = buf.decode("utf-8") - got = sanitize_presentation(text, mode="final") - assert got == expected, ( - f"streaming byte-split 破坏 sanitize · case={case.name} " - f"head_len={len(head)} · got={got!r} expected={expected!r}" - ) + assert internal_secret not in caplog.text + + +# ── 其他 mode 契约冻结(skeleton pass-through · M1+ 实装) ────── + + +@pytest.mark.parametrize("mode", ["history", "streaming", "interim", "tool"]) +def test_non_final_modes_skeleton_pass_through(mode: str): + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + text = "Answer with citation [source: kb_2024]. More text." + got = sanitize_presentation(text, mode=mode) + assert got == text, f"mode={mode} 应 pass-through(M0 skeleton · 见 §VI 冻结)" + + +# ── Streaming byte-split · S1 完整缓冲后 sanitize 与整体一致 ─── + + +@pytest.mark.parametrize("case", cases_by_marker("streaming_split"), ids=lambda c: c.name) +def test_streaming_full_buffer_matches_final(case: StripCase): + """S1 策略验证:任意字节切分点分片 → 缓冲拼回 → UTF-8 decode → final sanitize + → 结果必须等于对整体 raw 直接 final sanitize 的结果(与 case.expected 一致)。 + """ + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + expected = sanitize_presentation(case.raw, mode="final") + assert expected == case.expected, ( + f"streaming_split case={case.name} · 直接 final sanitize 与 fixture expected 不一致 · " + f"expected={case.expected!r} got={expected!r}" + ) + # 每个切分点:两段拼接 · decode · sanitize · 与 expected 一致 + for head, tail in enumerate_byte_split_chunks(case.raw): + buf = head + tail + text = buf.decode("utf-8") + got = sanitize_presentation(text, mode="final") + assert got == expected, ( + f"streaming byte-split 破坏 sanitize · case={case.name} " + f"head_len={len(head)} · got={got!r} expected={expected!r}" + ) From 16000d83dc4c91f4ef8fe73f33e363c99712e157 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:04:42 +0800 Subject: [PATCH 18/19] fix(spike): independent review hardening (7/7) --- .../test_strip_differential_vs_hermes_ai.py | 499 ++++++++---------- 1 file changed, 226 insertions(+), 273 deletions(-) diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py index 42c91411f507b..6d9ca68ef12d8 100644 --- a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py +++ b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py @@ -1,273 +1,226 @@ -r"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. - -**Codex 第三轮 §VI 强制**: -- fork sanitizer 与 Hermes_AI `ops/audit.py::strip_sources` 差异**必须**明文锁定 -- 差异**仅**允许:①`\s?` 前导空白吸收 · ②`.rstrip()` 保留首行 indentation -- 覆盖包含关系:baseline strip 的 · fork 必 strip -- 负例一致:baseline 保留的 · fork 必保留 -- baseline 源字符串 verbatim byte-check(飘移即失败) -""" -from __future__ import annotations - -import pytest - -from plugins.spike_p3_m0_fixtures import hermes_ai_baseline -from plugins.spike_p3_m0_fixtures.citation_cases import ALL_CASES, cases_by_marker -from plugins.spike_p3_m0_strip.sanitize import ( - _INLINE_SOURCE_RE, - _LINE_SOURCE_RE, - sanitize_presentation, -) - - -# ── 1. baseline 源字符串 verbatim byte-check ──────────────────── - - -def test_baseline_regex_verbatim(): - """Vendored baseline 与 Hermes_AI 源必须 byte-identical(source of truth 门)。 - - 这两个模式串直接从 `hermes_devices/ops/audit.py::_INLINE_RE._LINE_RE.pattern` - 抄写 · 若上游修改 · 本测试**必须**先失败提醒同步 vendor。 - """ - expected_inline = ( - r"[\[【（(]\s*(?:source|来源)\s*[:：]" - r"(?:[^\[\]【】（）()\n]|[（(][^（）()\n]*[）)]|[\[【][^\[\]【】\n]*[\]】])*" - r"[\]】）)]" - ) - expected_line = r"^[ \t]*(?:source|来源)\s*[:：].*$" - assert hermes_ai_baseline.BASELINE_INLINE_PATTERN == expected_inline, ( - "vendored baseline INLINE pattern drifted · resync from Hermes_AI " - "hermes_devices/ops/audit.py::_INLINE_RE.pattern" - ) - assert hermes_ai_baseline.BASELINE_LINE_PATTERN == expected_line, ( - "vendored baseline LINE pattern drifted · resync from Hermes_AI " - "hermes_devices/ops/audit.py::_LINE_RE.pattern" - ) - - -# ── 2. Fork inline pattern = baseline + `[ \t]?` 前导(**只水平空白**) ─ - - -def test_fork_inline_pattern_is_baseline_plus_leading_horizontal_space(): - """fork `_INLINE_SOURCE_RE` = `[ \\t]?` + baseline INLINE pattern(严格锁定)。 - - Codex §7 反例证明:前导消耗必须是 `[ \\t]?`(空格 / tab)· **不**能是 `\\s?` - (`\\s` 包含 `\\n` · 会把行末换行吞掉,拼接下一行 · 见 - `test_leading_space_does_not_eat_newline_before_marker`)。 - """ - assert _INLINE_SOURCE_RE.pattern == r"[ \t]?" + hermes_ai_baseline.BASELINE_INLINE_PATTERN - - -def test_fork_line_pattern_equals_baseline(): - """fork `_LINE_SOURCE_RE` = baseline LINE pattern(**完全一致**)。""" - assert _LINE_SOURCE_RE.pattern == hermes_ai_baseline.BASELINE_LINE_PATTERN - - -# ── 3. 覆盖包含关系:baseline strip 的 · fork 必 strip ───────── - - -_SOURCE_MARKER_LEFT_BRACKETS = ("[source", "[来源", "[SOURCE", "[Source", - "【source", "【来源", "（source", "(source", - "（来源", "(来源") - - -def _has_source_marker(text: str) -> bool: - """Detect any remaining `source:` / `来源:` inline marker(differential coverage).""" - lowered = text.lower() - for prefix in ("[source", "[来源", "【source", "【来源", "（source", "(source", - "（来源", "(来源"): - if prefix.lower() in lowered: - return True - return False - - -@pytest.mark.parametrize( - "case", - [c for c in ALL_CASES if c.mode == "final" and c.marker_type != "negative"], - ids=lambda c: c.name, -) -def test_fork_strips_everything_baseline_strips(case): - """Positive case:baseline 移除的每个 marker · fork 也必移除(**覆盖包含关系**)。 - - 契约锁定的两点断言(§VI): - A. 两侧输出**均无** source marker 残留(覆盖完整性) - B. 两侧输出去除**所有空白**后 byte-identical(差异仅在空白 · UX 增强范围内) - """ - baseline_out = hermes_ai_baseline.strip_sources(case.raw) - fork_out = sanitize_presentation(case.raw, mode="final") - - # A. 覆盖完整性(仅对 baseline 实际 strip 的 case 生效) - # 未闭合标记(§VI:不吞下一行)· baseline 与 fork 均保留 · 跳过此断言 - if baseline_out != case.raw: - assert not _has_source_marker(baseline_out), ( - f"case={case.name} · baseline 输出仍含 source marker · fixture 缺失或 baseline drift · " - f"baseline={baseline_out!r}" - ) - assert not _has_source_marker(fork_out), ( - f"case={case.name} · fork 输出仍含 source marker · 覆盖不完整 · fork={fork_out!r}" - ) - - # B. 差异仅在空白(核心保证:内容 byte-identical after \s+ 剥离) - import re as _re - baseline_nowhite = _re.sub(r"\s+", "", baseline_out) - fork_nowhite = _re.sub(r"\s+", "", fork_out) - assert baseline_nowhite == fork_nowhite, ( - f"case={case.name} · fork 与 baseline 在去除空白后仍不 byte-identical\n" - f" raw: {case.raw!r}\n" - f" baseline: {baseline_out!r}\n" - f" fork: {fork_out!r}\n" - f" baseline_nowhite: {baseline_nowhite!r}\n" - f" fork_nowhite: {fork_nowhite!r}\n" - "差异必须限于 §VI 声明的两点(前导空白、首尾空白)· " - "否则更新 sanitize.py 或 fixture · 双仓 PR body 同步" - ) - - -# ── 4. Negative case 一致性:baseline 不 strip · fork 也不 strip ─ - - -@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) -def test_negative_cases_identical(case): - """Negative case:baseline 与 fork 都必须原样保留(strict equality)。""" - baseline_out = hermes_ai_baseline.strip_sources(case.raw) - fork_out = sanitize_presentation(case.raw, mode="final") - assert baseline_out == case.raw, ( - f"baseline unexpectedly stripped negative case {case.name!r} · " - f"raw={case.raw!r} baseline={baseline_out!r}" - ) - assert fork_out == case.raw, ( - f"fork unexpectedly stripped negative case {case.name!r} · " - f"raw={case.raw!r} fork={fork_out!r}" - ) - - -# ── 5. UX 差异边界样例(明文示例断言) ───────────────────────── - - -def test_ux_diff_inline_leading_space(): - """UX 差异 ①:`\\s?` 前导空白吸收 · 单空格样例。""" - raw = "Answer is X [source: doc]." - assert hermes_ai_baseline.strip_sources(raw) == "Answer is X ." - assert sanitize_presentation(raw, mode="final") == "Answer is X." - - -def test_ux_diff_preserves_leading_indent(): - """UX 差异 ②:`.rstrip()` 保留首行 indentation。""" - raw = " Answer with [source: x]" - assert hermes_ai_baseline.strip_sources(raw) == "Answer with" # baseline .strip() - assert sanitize_presentation(raw, mode="final") == " Answer with" # fork .rstrip() - - -# ── 6. Codex §7 反例证明:前导差异不吞换行、不扩大删除范围 ───── - - -def test_leading_space_does_not_eat_newline_before_marker(): - """前一行的 `\\n` **不得**被前导消耗吞掉 · 否则会拼接下一行(§7 反例)。 - - 这是当前实施 `[ \\t]?` 而非 `\\s?` 的核心理由。曾用的 `\\s?` 匹配 `\\n` · - `"Line A\\n[source: kb]Next"` → `"Line ANext"`(BUG)· 现改为 `[ \\t]?` · - 结果应保持换行结构:`"Line A\\nNext"`。 - """ - raw = "Line A\n[source: kb]Next word" - fork_out = sanitize_presentation(raw, mode="final") - # baseline 不吃前导 · 直接删 marker · 结果保持换行 · 后接 Next word - baseline_out = hermes_ai_baseline.strip_sources(raw) - # 关键断言:换行未被吞 · 无行拼接 - assert "\n" in fork_out, ( - f"fork 前导消耗吞掉了 `\\n` · 拼接了下一行 · got={fork_out!r} · " - "前导 regex 必须是 `[ \\t]?` 不是 `\\s?`" - ) - assert fork_out == "Line A\nNext word", ( - f"fork 处理跨行 marker 错 · raw={raw!r} got={fork_out!r} expected='Line A\\nNext word'" - ) - # 同时 · fork 与 baseline 保持一致(此 case 前导不消耗 · 空白差异不体现) - assert fork_out == baseline_out - - -def test_leading_space_does_not_eat_double_newline_before_marker(): - """双换行边界同理 · 前导消耗**不吃** `\\n` · 段落分隔保留。""" - raw = "Paragraph A ends here.\n\n[source: kb]Paragraph B starts." - fork_out = sanitize_presentation(raw, mode="final") - # 双 \n 应保留(不吞任何 \n · 段落分隔完整) - assert "\n\n" in fork_out, ( - f"fork 吞掉了段落分隔 · got={fork_out!r}" - ) - assert fork_out == "Paragraph A ends here.\n\nParagraph B starts." - - -def test_leading_space_only_absorbs_single_space_or_tab(): - """前导 `[ \\t]?` 只吸收 1 个空格 / tab · 不吸收其他 whitespace。""" - # 空格 · 吸收 - assert sanitize_presentation("X [source: k]Y", mode="final") == "XY" - # tab · 吸收 - assert sanitize_presentation("X\t[source: k]Y", mode="final") == "XY" - # 换行 · 不吸收(§7 硬约束) - assert sanitize_presentation("X\n[source: k]Y", mode="final") == "X\nY" - # 回车 · 不吸收(避免 Windows 场景意外拼接) - assert sanitize_presentation("X\r[source: k]Y", mode="final") == "X\rY" - - -def test_whitespace_diff_does_not_widen_source_deletion_range(): - """空白差异**不**扩大 source 删除范围:非 source 括号一律保留。 - - Codex §7:确保 `\\s?` → `[ \\t]?` 与 `.rstrip()` 的两处 UX 增强· - 不会让 fork 意外删掉 baseline 保留的内容。 - """ - # 非 source 括号(`[normal]`)紧跟 source marker · 只 source 被删 - raw = "Result: [normal] [source: kb] end." - fork_out = sanitize_presentation(raw, mode="final") - baseline_out = hermes_ai_baseline.strip_sources(raw) - assert "[normal]" in fork_out, f"fork 误删了 [normal] · got={fork_out!r}" - assert "[normal]" in baseline_out - # source marker 已删 - assert "[source" not in fork_out.lower() - # 两侧内容(去空白)一致 - import re as _re - assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) - - -def test_trailing_whitespace_rstrip_scope_matches_baseline_content(): - """`.rstrip()` 只掐尾空白 · 不改变文本主体 · 与 baseline `.strip()` 内容一致(仅首空白 diff)。""" - raw = "Answer here [source: k]\n\n " - fork_out = sanitize_presentation(raw, mode="final") - baseline_out = hermes_ai_baseline.strip_sources(raw) - # 两侧 trailing 皆掐 · 主体一致 - assert fork_out == "Answer here" - assert baseline_out == "Answer here" - - -def test_leading_whitespace_only_diff_between_baseline_and_fork(): - """首行 indentation 场景 · fork 保留 · baseline 掐 · 主体 byte-identical。""" - raw = " Answer [source: k]" - fork_out = sanitize_presentation(raw, mode="final") - baseline_out = hermes_ai_baseline.strip_sources(raw) - # fork 保留首 3 空格 · baseline 全 strip - assert fork_out == " Answer" - assert baseline_out == "Answer" - # 去空白后主体 byte-identical - import re as _re - assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) == "Answer" - - -def test_middle_line_source_marker_preserves_surrounding_newlines(): - """中间行的 inline source marker · 两侧换行完整保留 · 无行拼接。""" - raw = "Line 1\nLine 2 [source: k]\nLine 3" - fork_out = sanitize_presentation(raw, mode="final") - assert fork_out == "Line 1\nLine 2\nLine 3" - - -def test_marker_at_start_of_line_preserves_prev_line_terminator(): - """行首 marker 场景 · 前一行的 `\\n` 保留 · 只删 marker + 前导水平空白(若有)。""" - raw_no_lead = "Line 1\n[source: k]Line 2 content" - raw_with_lead = "Line 1\n \t[source: k]Line 2 content" - - out_no_lead = sanitize_presentation(raw_no_lead, mode="final") - out_with_lead = sanitize_presentation(raw_with_lead, mode="final") - - # 无 leading horizontal ws · 结果 "Line 1\nLine 2 content" - assert out_no_lead == "Line 1\nLine 2 content" - # 有 leading horizontal ws · marker 前导吸收 1 个 · 剩下 " \t"? - # 实际:`[ \t]?[source: k]` 匹配 `\t[source: k]` · 前面的 " " 保留 · 结果 "Line 1\n Line 2 content" - # 关键点:换行未被吞 · 有 \n - assert "\n" in out_with_lead - assert out_with_lead == "Line 1\n Line 2 content" +r"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. + +**Codex 第三轮 §VI 强制**: +- fork sanitizer 与 Hermes_AI `ops/audit.py::strip_sources` 差异**必须**明文锁定 +- 差异**仅**允许:①`\s?` 前导空白吸收 · ②`.rstrip()` 保留首行 indentation +- 覆盖包含关系:baseline strip 的 · fork 必 strip +- 负例一致:baseline 保留的 · fork 必保留 +- 用输入/输出行为证明覆盖包含关系，不锁死正则实现文本 +""" +from __future__ import annotations + +import pytest + +from plugins.spike_p3_m0_fixtures import hermes_ai_baseline +from plugins.spike_p3_m0_fixtures.citation_cases import ALL_CASES, cases_by_marker +from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + +# ── 1. 覆盖包含关系:baseline strip 的 · fork 必 strip ───────── + + +_SOURCE_MARKER_LEFT_BRACKETS = ("[source", "[来源", "[SOURCE", "[Source", + "【source", "【来源", "（source", "(source", + "（来源", "(来源") + + +def _has_source_marker(text: str) -> bool: + """Detect any remaining `source:` / `来源:` inline marker(differential coverage).""" + lowered = text.lower() + for prefix in ("[source", "[来源", "【source", "【来源", "（source", "(source", + "（来源", "(来源"): + if prefix.lower() in lowered: + return True + return False + + +@pytest.mark.parametrize( + "case", + [c for c in ALL_CASES if c.mode == "final" and c.marker_type != "negative"], + ids=lambda c: c.name, +) +def test_fork_strips_everything_baseline_strips(case): + """Positive case:baseline 移除的每个 marker · fork 也必移除(**覆盖包含关系**)。 + + 契约锁定的两点断言(§VI): + A. 两侧输出**均无** source marker 残留(覆盖完整性) + B. 两侧输出去除**所有空白**后 byte-identical(差异仅在空白 · UX 增强范围内) + """ + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + + # A. 覆盖完整性(仅对 baseline 实际 strip 的 case 生效) + # 未闭合标记(§VI:不吞下一行)· baseline 与 fork 均保留 · 跳过此断言 + if baseline_out != case.raw: + assert not _has_source_marker(baseline_out), ( + f"case={case.name} · baseline 输出仍含 source marker · fixture 缺失或 baseline drift · " + f"baseline={baseline_out!r}" + ) + assert not _has_source_marker(fork_out), ( + f"case={case.name} · fork 输出仍含 source marker · 覆盖不完整 · fork={fork_out!r}" + ) + + # B. 差异仅在空白(核心保证:内容 byte-identical after \s+ 剥离) + import re as _re + baseline_nowhite = _re.sub(r"\s+", "", baseline_out) + fork_nowhite = _re.sub(r"\s+", "", fork_out) + assert baseline_nowhite == fork_nowhite, ( + f"case={case.name} · fork 与 baseline 在去除空白后仍不 byte-identical\n" + f" raw: {case.raw!r}\n" + f" baseline: {baseline_out!r}\n" + f" fork: {fork_out!r}\n" + f" baseline_nowhite: {baseline_nowhite!r}\n" + f" fork_nowhite: {fork_nowhite!r}\n" + "差异必须限于 §VI 声明的两点(前导空白、首尾空白)· " + "否则更新 sanitize.py 或 fixture · 双仓 PR body 同步" + ) + + +# ── 4. Negative case 一致性:baseline 不 strip · fork 也不 strip ─ + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_identical(case): + """Negative case:baseline 与 fork 都必须原样保留(strict equality)。""" + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + assert baseline_out == case.raw, ( + f"baseline unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} baseline={baseline_out!r}" + ) + assert fork_out == case.raw, ( + f"fork unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} fork={fork_out!r}" + ) + + +# ── 5. UX 差异边界样例(明文示例断言) ───────────────────────── + + +def test_ux_diff_inline_leading_space(): + """UX 差异 ①:`\\s?` 前导空白吸收 · 单空格样例。""" + raw = "Answer is X [source: doc]." + assert hermes_ai_baseline.strip_sources(raw) == "Answer is X ." + assert sanitize_presentation(raw, mode="final") == "Answer is X." + + +def test_ux_diff_preserves_leading_indent(): + """UX 差异 ②:`.rstrip()` 保留首行 indentation。""" + raw = " Answer with [source: x]" + assert hermes_ai_baseline.strip_sources(raw) == "Answer with" # baseline .strip() + assert sanitize_presentation(raw, mode="final") == " Answer with" # fork .rstrip() + + +# ── 6. Codex §7 反例证明:前导差异不吞换行、不扩大删除范围 ───── + + +def test_leading_space_does_not_eat_newline_before_marker(): + """前一行的 `\\n` **不得**被前导消耗吞掉 · 否则会拼接下一行(§7 反例)。 + + 这是当前实施 `[ \\t]?` 而非 `\\s?` 的核心理由。曾用的 `\\s?` 匹配 `\\n` · + `"Line A\\n[source: kb]Next"` → `"Line ANext"`(BUG)· 现改为 `[ \\t]?` · + 结果应保持换行结构:`"Line A\\nNext"`。 + """ + raw = "Line A\n[source: kb]Next word" + fork_out = sanitize_presentation(raw, mode="final") + # baseline 不吃前导 · 直接删 marker · 结果保持换行 · 后接 Next word + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 关键断言:换行未被吞 · 无行拼接 + assert "\n" in fork_out, ( + f"fork 前导消耗吞掉了 `\\n` · 拼接了下一行 · got={fork_out!r} · " + "前导 regex 必须是 `[ \\t]?` 不是 `\\s?`" + ) + assert fork_out == "Line A\nNext word", ( + f"fork 处理跨行 marker 错 · raw={raw!r} got={fork_out!r} expected='Line A\\nNext word'" + ) + # 同时 · fork 与 baseline 保持一致(此 case 前导不消耗 · 空白差异不体现) + assert fork_out == baseline_out + + +def test_leading_space_does_not_eat_double_newline_before_marker(): + """双换行边界同理 · 前导消耗**不吃** `\\n` · 段落分隔保留。""" + raw = "Paragraph A ends here.\n\n[source: kb]Paragraph B starts." + fork_out = sanitize_presentation(raw, mode="final") + # 双 \n 应保留(不吞任何 \n · 段落分隔完整) + assert "\n\n" in fork_out, ( + f"fork 吞掉了段落分隔 · got={fork_out!r}" + ) + assert fork_out == "Paragraph A ends here.\n\nParagraph B starts." + + +def test_leading_space_only_absorbs_single_space_or_tab(): + """前导 `[ \\t]?` 只吸收 1 个空格 / tab · 不吸收其他 whitespace。""" + # 空格 · 吸收 + assert sanitize_presentation("X [source: k]Y", mode="final") == "XY" + # tab · 吸收 + assert sanitize_presentation("X\t[source: k]Y", mode="final") == "XY" + # 换行 · 不吸收(§7 硬约束) + assert sanitize_presentation("X\n[source: k]Y", mode="final") == "X\nY" + # 回车 · 不吸收(避免 Windows 场景意外拼接) + assert sanitize_presentation("X\r[source: k]Y", mode="final") == "X\rY" + + +def test_whitespace_diff_does_not_widen_source_deletion_range(): + """空白差异**不**扩大 source 删除范围:非 source 括号一律保留。 + + Codex §7:确保 `\\s?` → `[ \\t]?` 与 `.rstrip()` 的两处 UX 增强· + 不会让 fork 意外删掉 baseline 保留的内容。 + """ + # 非 source 括号(`[normal]`)紧跟 source marker · 只 source 被删 + raw = "Result: [normal] [source: kb] end." + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + assert "[normal]" in fork_out, f"fork 误删了 [normal] · got={fork_out!r}" + assert "[normal]" in baseline_out + # source marker 已删 + assert "[source" not in fork_out.lower() + # 两侧内容(去空白)一致 + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) + + +def test_trailing_whitespace_rstrip_scope_matches_baseline_content(): + """`.rstrip()` 只掐尾空白 · 不改变文本主体 · 与 baseline `.strip()` 内容一致(仅首空白 diff)。""" + raw = "Answer here [source: k]\n\n " + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 两侧 trailing 皆掐 · 主体一致 + assert fork_out == "Answer here" + assert baseline_out == "Answer here" + + +def test_leading_whitespace_only_diff_between_baseline_and_fork(): + """首行 indentation 场景 · fork 保留 · baseline 掐 · 主体 byte-identical。""" + raw = " Answer [source: k]" + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # fork 保留首 3 空格 · baseline 全 strip + assert fork_out == " Answer" + assert baseline_out == "Answer" + # 去空白后主体 byte-identical + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) == "Answer" + + +def test_middle_line_source_marker_preserves_surrounding_newlines(): + """中间行的 inline source marker · 两侧换行完整保留 · 无行拼接。""" + raw = "Line 1\nLine 2 [source: k]\nLine 3" + fork_out = sanitize_presentation(raw, mode="final") + assert fork_out == "Line 1\nLine 2\nLine 3" + + +def test_marker_at_start_of_line_preserves_prev_line_terminator(): + """行首 marker 场景 · 前一行的 `\\n` 保留 · 只删 marker + 前导水平空白(若有)。""" + raw_no_lead = "Line 1\n[source: k]Line 2 content" + raw_with_lead = "Line 1\n \t[source: k]Line 2 content" + + out_no_lead = sanitize_presentation(raw_no_lead, mode="final") + out_with_lead = sanitize_presentation(raw_with_lead, mode="final") + + # 无 leading horizontal ws · 结果 "Line 1\nLine 2 content" + assert out_no_lead == "Line 1\nLine 2 content" + # 有 leading horizontal ws · marker 前导吸收 1 个 · 剩下 " \t"? + # 实际:`[ \t]?[source: k]` 匹配 `\t[source: k]` · 前面的 " " 保留 · 结果 "Line 1\n Line 2 content" + # 关键点:换行未被吞 · 有 \n + assert "\n" in out_with_lead + assert out_with_lead == "Line 1\n Line 2 content" From 0170c9bc70c9309e670495e1466eae3a1e4b7029 Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 10 Aug 2026 21:22:46 +0800 Subject: [PATCH 19/19] fix(p3-m0): restore complete cache middleware test source --- .../tests/test_cache_middleware.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py index 9c2b47a68276c..baf792cddcf4c 100644 --- a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -524,7 +524,54 @@ def test_cache_hit_returns_deep_copy(tmp_path, monkeypatch): _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) calls = {"next": 0} - def next_call(_req…475 tokens truncated…mode="chat_completions", + def next_call(_req): + calls["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "stable"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}, + } + + req = {"messages": [{"role": "user", "content": "same"}]} + make_run_llm_execution( + api_request_id="copy_miss", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit = make_run_llm_execution( + api_request_id="copy_hit_1", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit["choices"][0]["message"]["content"] = "poisoned" + first_hit["cache_meta"]["origin_usage"]["prompt_tokens"] = 999 + + second_hit = make_run_llm_execution( + api_request_id="copy_hit_2", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + assert calls["next"] == 1 + assert second_hit["choices"][0]["message"]["content"] == "stable" + assert second_hit["cache_meta"]["origin_usage"]["prompt_tokens"] == 5 + + +def test_cache_hit_does_not_require_budget_request_id(tmp_path, monkeypatch): + """命中不调用 provider/预算，因此不应要求 api_request_id。""" + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + calls = {"next": 0} + request = {"messages": [{"role": "user", "content": "same"}]} + + def next_call(_req): + calls["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}, + } + + make_run_llm_execution( + api_request_id="cache_seed", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, )(request, next_call) hit = make_run_llm_execution(