diff --git a/.gitattributes b/.gitattributes index 553e3cd21b3cf..6cedf3280c476 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,7 @@ web/package-lock.json linguist-generated=true Dockerfile text eol=lf *.dockerfile text eol=lf docker/entrypoint.sh text eol=lf + +# P3-M0 shared strip fixture — SHA-256 is a cross-repo lock; must stay LF-normalized +# on every checkout so the double-repo fixture SHA gate matches on Windows too. +plugins/spike_p3_m0_fixtures/citation_cases.py text eol=lf diff --git a/contributors/emails/bri12afdsarker96@gmail.com b/contributors/emails/bri12afdsarker96@gmail.com new file mode 100644 index 0000000000000..d0bdbc8685d5e --- /dev/null +++ b/contributors/emails/bri12afdsarker96@gmail.com @@ -0,0 +1,2 @@ +bri12afdsarker96-lgtm +# PR #1 P3-M0 spike contributor mapping diff --git a/plugins/spike_p3_m0_budget/__init__.py b/plugins/spike_p3_m0_budget/__init__.py new file mode 100644 index 0000000000000..e778755bb742d --- /dev/null +++ b/plugins/spike_p3_m0_budget/__init__.py @@ -0,0 +1,232 @@ +"""P3-M0 Spike · tenant budget service module(no hooks · used by cache orchestrator). + +**Codex §I / §V 契约**: +- **不**通过 `pre_api_request` 抛 BudgetExceeded(生产调用点会吞异常) +- 由 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内部按序调用: + cache lookup → miss → **budget reserve** → next_call → success settle actual / + exception release reservation +- `pre_api_request/post_api_request` 只做观察 audit(见 __init__.py 底部可选注册) + +**4 态 budget**(与 Hermes_AI P2-0.1 一致): +- 缺失(config 无 `tenant.daily_budget_tokens` 键)· 不启用租户 budget +- `null`(显式 None)· 不启用(provisioning 明确无限期不 enforce · 与 0 区分) +- `0`(显式 int 0)· 显式无限 · reserve 恒 True · used 仍计 +- `>0`(int)· 具体上限 +- **拒**:bool / 负数 / str / float · 抛 `BudgetConfigError` · enforcement 上层必须 fail-CLOSED + +**UTC daily bucket**:key = (tenant_id, utc_date_str) · 天粒度 · pending reservations 与 +used 一起判断超支。 + +**并发**:threading.Lock · reserve/settle/release 全序 · api_request_id 唯一. +""" +from __future__ import annotations + +import datetime +import logging +import threading +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +BUDGET_CONTRACT_VERSION = "v1" + + +class BudgetExceeded(RuntimeError): + """租户预算超支 · **不由 hook 抛** · 由 llm_execution middleware 内部捕获转 audit。""" + + +class BudgetConfigError(ValueError): + """budget 配置字段类型/值非法 · 由 orchestrator 硬 fail-CLOSED(**不**再静默转 None)。""" + + +class BudgetSettleError(RuntimeError): + """settle/release 在无 reservation 上被调用 · **不改** used(严格幂等)。 + + Codex 第三轮 §IV:settle 必须仅结算仍存在的 reservation;unknown/double settle + 不得增加 used。抛此异常供 orchestrator 记 audit 但不阻断。 + """ + + +# ── Module state · 单进程 · 非 persistent ───────────────────────────── + +_lock = threading.RLock() # reentrant · reserve/settle 内部可能连续调用 +# 天粒度桶 · key = (tenant_id, "YYYY-MM-DD" UTC) · value = { "used": int, "reservations": {req_id: tokens} } +_bucket: Dict[str, Dict[str, Any]] = {} +# Active reservations must remember the UTC bucket in which they were created. +# A provider call can cross midnight; settle/release must then touch the original +# bucket instead of leaking the reservation in yesterday's ledger. +_reservation_bucket: Dict[tuple[str, str], str] = {} + + +def _today_utc() -> str: + """UTC 日期字符串 · YYYY-MM-DD 格式 · 天粒度 bucket key。""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + + +def _bucket_key(tenant_id: str, utc_date: Optional[str] = None) -> str: + return f"{tenant_id}::{utc_date or _today_utc()}" + + +def _get_or_create_bucket_by_key(key: str) -> Dict[str, Any]: + b = _bucket.get(key) + if b is None: + b = {"used": 0, "reservations": {}} + _bucket[key] = b + return b + + +def _get_or_create_bucket(tenant_id: str) -> Dict[str, Any]: + return _get_or_create_bucket_by_key(_bucket_key(tenant_id)) + + +# ── 4 态 budget validation ─────────────────────────────────────────── + +def validate_daily_budget(value: Any) -> Optional[int]: + """校验 config `tenant.daily_budget_tokens` 字段 · 返回归一化 int | None。 + + None → None(未启用) + 0 → 0(显式无限) + 正整数 → 正整数 + bool / 负数 / str / float → raise BudgetConfigError · enforcement 上层必须 + 返回结构化 fail-CLOSED response,禁止转换为 None 后继续 provider 调用。 + """ + if value is None: + return None + # bool 是 int 子类 · 必须先拒 + if isinstance(value, bool): + raise BudgetConfigError(f"daily_budget_tokens type=bool 非法 · got {value!r}") + if not isinstance(value, int): + raise BudgetConfigError(f"daily_budget_tokens 需 int · got {type(value).__name__} {value!r}") + if value < 0: + raise BudgetConfigError(f"daily_budget_tokens 需 >=0 · got {value}") + return value + + +# ── Reserve / Settle / Release · 由 orchestrator middleware 调用 ───── + +def reserve(tenant_id: str, api_request_id: str, tokens_needed: int, *, budget: Optional[int]) -> None: + """尝试预留 · 超预算 raise BudgetExceeded · budget=None 直接 no-op(未启用)。 + + `budget=0` 是显式无限 · reserve 恒成功 · used 仍会累加供审计观察。 + """ + if not api_request_id: + raise BudgetExceeded("reserve: empty api_request_id · 拒绝 anonymous 预留") + if tokens_needed < 0: + tokens_needed = 0 + if budget is None: + return # 未启用 + with _lock: + reservation_key = (tenant_id, api_request_id) + # api_request_id 在该 tenant 的所有活跃 UTC bucket 中唯一。 + if reservation_key in _reservation_bucket: + raise BudgetExceeded( + f"reserve: api_request_id={api_request_id!r} already reserved · duplicate" + ) + bucket_key = _bucket_key(tenant_id) + b = _get_or_create_bucket_by_key(bucket_key) + pending_total = sum(b["reservations"].values()) + # budget=0 无限 · used + pending + new 不做上限比较 + if budget > 0: + projected = b["used"] + pending_total + tokens_needed + if projected > budget: + raise BudgetExceeded( + f"tenant {tenant_id} daily budget exhausted " + f"(used={b['used']}, pending={pending_total}, need={tokens_needed}, budget={budget})" + ) + b["reservations"][api_request_id] = tokens_needed + _reservation_bucket[reservation_key] = bucket_key + + +def settle(tenant_id: str, api_request_id: str, actual_tokens: int) -> None: + """结算实际用量 · **仅结算仍存在的 reservation** · 累加 used。 + + Codex 第三轮 §IV(**严格幂等**): + - 若 reservation 存在 · 移除并累加 used + - 若 reservation 不存在(未 reserve / double settle / release 后 settle): + **不改** used · raise BudgetSettleError · 由 orchestrator 记 audit + """ + if actual_tokens < 0: + actual_tokens = 0 + with _lock: + reservation_key = (tenant_id, api_request_id) + bucket_key = _reservation_bucket.get(reservation_key) + if bucket_key is None: + raise BudgetSettleError( + f"settle: no reservation for api_request_id={api_request_id!r} · " + "double settle or missing reserve" + ) + b = _get_or_create_bucket_by_key(bucket_key) + if api_request_id not in b["reservations"]: + # The index and bucket must move atomically under the same lock. + _reservation_bucket.pop(reservation_key, None) + raise BudgetSettleError( + f"settle: reservation index corrupt for api_request_id={api_request_id!r}" + ) + b["reservations"].pop(api_request_id) + _reservation_bucket.pop(reservation_key, None) + b["used"] = b["used"] + actual_tokens + + +def release(tenant_id: str, api_request_id: str) -> bool: + """异常 / cancel / timeout 路径 · 释放 reservation · 不计 used。 + + Codex 第三轮 §IV(**幂等**): + - 若 reservation 存在 · 移除 · 返回 True + - 若 reservation 不存在 · **no-op** · 返回 False(便于 orchestrator 检测 double release) + - 任何情况下 · used 不变 + """ + with _lock: + reservation_key = (tenant_id, api_request_id) + bucket_key = _reservation_bucket.pop(reservation_key, None) + if bucket_key is None: + return False + b = _get_or_create_bucket_by_key(bucket_key) + return b["reservations"].pop(api_request_id, None) is not None + + +# ── 观察 helpers · 测试用 ───────────────────────────────────────────── + +def get_used(tenant_id: str) -> int: + with _lock: + return _get_or_create_bucket(tenant_id)["used"] + + +def get_pending_total(tenant_id: str) -> int: + with _lock: + return sum(_get_or_create_bucket(tenant_id)["reservations"].values()) + + +def get_reservation(tenant_id: str, api_request_id: str) -> Optional[int]: + with _lock: + bucket_key = _reservation_bucket.get((tenant_id, api_request_id)) + if bucket_key is None: + return None + return _get_or_create_bucket_by_key(bucket_key)["reservations"].get(api_request_id) + + +def reset_state() -> None: + """测试用 · 清空全部 bucket。""" + with _lock: + _bucket.clear() + _reservation_bucket.clear() + + +# ── register: 只挂 observer hooks · 无 enforcement · 无 middleware ──── + +def _on_pre_api_request_observer(**_kwargs: Any) -> None: + """观察者:reserve 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return + + +def _on_post_api_request_observer(**_kwargs: Any) -> None: + """观察者:settle 已由 orchestrator middleware 完成 · 此 hook 无 enforcement。""" + return + + +def register(ctx: Any) -> None: + """PluginContext register · 只挂观察 hooks · 不含 enforcement。 + + 真实 enforcement 在 `spike_p3_m0_cache` 注册的 `llm_execution` middleware 内。 + """ + ctx.register_hook("pre_api_request", _on_pre_api_request_observer) + ctx.register_hook("post_api_request", _on_post_api_request_observer) diff --git a/plugins/spike_p3_m0_budget/plugin.yaml b/plugins/spike_p3_m0_budget/plugin.yaml new file mode 100644 index 0000000000000..dfa81c27c21da --- /dev/null +++ b/plugins/spike_p3_m0_budget/plugin.yaml @@ -0,0 +1,7 @@ +name: spike-p3-m0-budget +version: "0.0.1" +description: "P3-M0 Spike · tenant budget service (UTC daily bucket, 4-state validation, thread-safe reserve/settle/release) called from spike-p3-m0-cache orchestrator middleware. Registers observer-only pre_api_request / post_api_request hooks (NO enforcement — production call sites swallow hook exceptions). MUST NOT use HERMES_DAILY_TOKEN_BUDGET to impersonate per-tenant budget. Off by default (opt-in via plugins.enabled)." +author: "Hermes_AI × NousResearch/hermes-agent Spike" +provides_hooks: + - pre_api_request + - post_api_request diff --git a/plugins/spike_p3_m0_budget/tests/__init__.py b/plugins/spike_p3_m0_budget/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py new file mode 100644 index 0000000000000..83d5bfeaeb34c --- /dev/null +++ b/plugins/spike_p3_m0_budget/tests/test_budget_reserve.py @@ -0,0 +1,356 @@ +"""P3-M0 Spike · budget service 测试(observer hook · service API). + +**Codex 第三轮 §IV 契约(修订)**: +- settle 仅结算**仍存在**的 reservation · unknown/double settle → raise BudgetSettleError · **不改** used +- release 幂等 · 返回 True(存在)/ False(不存在)· 不改 used +- 40 线程 barrier 精确超限测试:budget = N * per-req · 超过 N 个必 raise +- UTC 跨日 rollover 真测试(monkeypatch `_today_utc`) +- observer hooks 恒 no-op(无 enforcement) +""" +from __future__ import annotations + +import threading + +import pytest + +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + write_config, +) + + +BUDGET_KEY = "spike-p3-m0-budget" + + +def _enable_budget(tmp_path, monkeypatch): + write_config(tmp_path, [BUDGET_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, BUDGET_KEY) + mod.reset_state() + return manager, mod + + +# ── Registration(§VIII 真实 PluginManager 门槛) ────────────────── + + +def test_discovery_registers_observer_hooks(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + manager, mod = _enable_budget(tmp_path, monkeypatch) + assert has_hook("pre_api_request") is True + assert has_hook("post_api_request") is True + assert manager._middleware.get("llm_execution") in (None, []) + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_hook("pre_api_request") is False + + +# ── validate_daily_budget · 4 态 + 拒非法(§V) ───────────────────── + + +def test_validate_daily_budget_states(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.validate_daily_budget(None) is None + assert mod.validate_daily_budget(0) == 0 + assert mod.validate_daily_budget(1000) == 1000 + + +@pytest.mark.parametrize( + "bad_value", + [True, False, -1, "1000", 1.5, [], {}], + ids=["true", "false", "negative", "string", "float", "list", "dict"], +) +def test_validate_daily_budget_rejects(tmp_path, monkeypatch, bad_value): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetConfigError): + mod.validate_daily_budget(bad_value) + + +# ── reserve / settle · 基本路径 ───────────────────────────────────── + + +def test_reserve_none_budget_no_op(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_a", 500, budget=None) + assert mod.get_pending_total("t1") == 0 + assert mod.get_reservation("t1", "req_a") is None + + +def test_reserve_then_settle_charges_actual(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 300, budget=10_000) + assert mod.get_pending_total("t1") == 300 + mod.settle("t1", "req_1", 120) + assert mod.get_used("t1") == 120 + assert mod.get_pending_total("t1") == 0 + + +def test_release_removes_reservation_returns_true(tmp_path, monkeypatch): + """release 存在的 reservation · 返回 True · used 不变。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 200, budget=10_000) + assert mod.release("t1", "req_x") is True + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +def test_release_unknown_returns_false_no_op(tmp_path, monkeypatch): + """Codex §IV:release unknown reservation · 返回 False · no-op(幂等)。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + assert mod.release("t1", "never_reserved") is False + assert mod.get_used("t1") == 0 + assert mod.get_pending_total("t1") == 0 + + +def test_release_after_release_returns_false(tmp_path, monkeypatch): + """double release · 第二次返回 False · 无副作用。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_x", 100, budget=1000) + assert mod.release("t1", "req_x") is True + assert mod.release("t1", "req_x") is False + assert mod.get_used("t1") == 0 + + +# ── reserve 超预算 · raise BudgetExceeded ──────────────────────── + + +def test_reserve_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="daily budget exhausted"): + mod.reserve("t1", "req_over", 200, budget=100) + assert mod.get_pending_total("t1") == 0 + + +def test_used_plus_pending_plus_new_over_budget_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_a", 60, budget=200) + mod.settle("t1", "r_a", 60) + mod.reserve("t1", "r_b", 100, budget=200) + with pytest.raises(mod.BudgetExceeded): + mod.reserve("t1", "r_c", 50, budget=200) + + +def test_budget_zero_is_unlimited(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "r_big", 10_000_000, budget=0) + mod.settle("t1", "r_big", 5_000_000) + assert mod.get_used("t1") == 5_000_000 + + +# ── api_request_id 唯一性 ───────────────────────────────────────── + + +def test_duplicate_reserve_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_dup", 50, budget=1000) + with pytest.raises(mod.BudgetExceeded, match="duplicate"): + mod.reserve("t1", "req_dup", 50, budget=1000) + + +def test_empty_api_request_id_raises(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetExceeded, match="anonymous"): + mod.reserve("t1", "", 50, budget=1000) + + +# ── settle 严格幂等(Codex §IV 修订)───────────────────────────── + + +def test_settle_unknown_id_raises_and_does_not_charge(tmp_path, monkeypatch): + """Codex §IV 修订:settle 无 reservation · **raise BudgetSettleError · 不改 used**。 + + 旧契约"unknown settle 累加 used"是**错误**的 · 会导致 unknown id 洗钱。 + """ + _, mod = _enable_budget(tmp_path, monkeypatch) + with pytest.raises(mod.BudgetSettleError, match="no reservation"): + mod.settle("t1", "unknown_req", 25) + assert mod.get_used("t1") == 0 + + +def test_double_settle_raises_after_first_ok(tmp_path, monkeypatch): + """double settle · 第二次 raise · used 不再加。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.settle("t1", "req_1", 50) # 首次 OK + assert mod.get_used("t1") == 50 + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 999) # 第二次 · reservation 已消 · raise + assert mod.get_used("t1") == 50 # 未增 + + +def test_settle_after_release_raises(tmp_path, monkeypatch): + """release 后 settle · reservation 不存在 · raise · used 不变。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("t1", "req_1", 50, budget=1000) + mod.release("t1", "req_1") + with pytest.raises(mod.BudgetSettleError): + mod.settle("t1", "req_1", 25) + assert mod.get_used("t1") == 0 + + +# ── 多 tenant 隔离 ────────────────────────────────────────────────── + + +def test_multiple_tenants_isolated(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + mod.reserve("tA", "r1", 40, budget=100) + mod.settle("tA", "r1", 40) + mod.reserve("tB", "r2", 90, budget=100) + mod.settle("tB", "r2", 90) + assert mod.get_used("tA") == 40 + assert mod.get_used("tB") == 90 + + +# ── 并发 reserve/settle · 无 race ──────────────────────────────── + + +def test_concurrent_reserve_settle_thread_safe(tmp_path, monkeypatch): + _, mod = _enable_budget(tmp_path, monkeypatch) + N = 40 + barrier = threading.Barrier(N) + + def worker(i): + barrier.wait() + mod.reserve("t_shared", f"req_{i}", 10, budget=10_000) + mod.settle("t_shared", f"req_{i}", 5) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert mod.get_used("t_shared") == N * 5 + assert mod.get_pending_total("t_shared") == 0 + + +# ── 真并发超限:Codex §IV 强要求 · 精确 N 成功 / rest 失败 ───────── + + +def test_concurrent_reserve_precise_success_and_exceeded_counts(tmp_path, monkeypatch): + """40 线程 barrier reserve · budget 只允许 N 个成功 · 精确 N 成功 · 40-N 失败。 + + Codex §IV:证明无超扣、无 race race-double-count。 + """ + _, mod = _enable_budget(tmp_path, monkeypatch) + N_THREADS = 40 + PER_REQ = 10 + N_SUCCESS = 15 # 只允许 15 个成功 + BUDGET = N_SUCCESS * PER_REQ # 150 + barrier = threading.Barrier(N_THREADS) + successes = [] + exceeded = [] + success_lock = threading.Lock() + + def worker(i): + barrier.wait() + try: + mod.reserve("t_precise", f"r{i}", PER_REQ, budget=BUDGET) + with success_lock: + successes.append(i) + except mod.BudgetExceeded: + with success_lock: + exceeded.append(i) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(N_THREADS)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(successes) == N_SUCCESS, ( + f"expected exactly {N_SUCCESS} successes · got {len(successes)}" + ) + assert len(exceeded) == N_THREADS - N_SUCCESS + assert mod.get_pending_total("t_precise") == BUDGET # 全部成功者仍 pending + + # settle 全部成功者 · 无超扣 + for i in successes: + mod.settle("t_precise", f"r{i}", PER_REQ) + assert mod.get_used("t_precise") == BUDGET + assert mod.get_pending_total("t_precise") == 0 + + +# ── UTC 跨日 rollover 真测试(Codex §IV) ──────────────────────── + + +def test_utc_day_rollover_isolates_bucket(tmp_path, monkeypatch): + """monkeypatch `_today_utc` 模拟跨日 · 昨日 used 不影响今日 budget。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + + day1 = "2026-08-09" + day2 = "2026-08-10" + + # Day 1 · 用满 budget=100 的 80 + monkeypatch.setattr(mod, "_today_utc", lambda: day1) + mod.reserve("t_roll", "r_d1", 80, budget=100) + mod.settle("t_roll", "r_d1", 80) + assert mod.get_used("t_roll") == 80 + + # Day 2 · 昨日 used 不算入今日 · 今日仍可 reserve 满 100 + monkeypatch.setattr(mod, "_today_utc", lambda: day2) + # 新 bucket · used 归 0 + assert mod.get_used("t_roll") == 0 + mod.reserve("t_roll", "r_d2", 100, budget=100) + mod.settle("t_roll", "r_d2", 100) + assert mod.get_used("t_roll") == 100 + + # 回到 Day 1 · 昨日 bucket 仍 80(未被今日污染) + monkeypatch.setattr(mod, "_today_utc", lambda: day1) + assert mod.get_used("t_roll") == 80 + + +def test_settle_after_utc_rollover_charges_original_bucket(tmp_path, monkeypatch): + """跨零点 provider 调用必须结算创建 reservation 的原 UTC 日。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + current_day = {"value": "2026-08-09"} + monkeypatch.setattr(mod, "_today_utc", lambda: current_day["value"]) + + mod.reserve("t_cross", "req_cross", 80, budget=100) + current_day["value"] = "2026-08-10" + mod.settle("t_cross", "req_cross", 60) + + assert mod.get_used("t_cross") == 0 + assert mod.get_reservation("t_cross", "req_cross") is None + current_day["value"] = "2026-08-09" + assert mod.get_used("t_cross") == 60 + assert mod.get_pending_total("t_cross") == 0 + + +def test_release_after_utc_rollover_clears_original_bucket(tmp_path, monkeypatch): + """跨零点异常清理不得把昨日 pending reservation 永久遗留。""" + _, mod = _enable_budget(tmp_path, monkeypatch) + current_day = {"value": "2026-08-09"} + monkeypatch.setattr(mod, "_today_utc", lambda: current_day["value"]) + + mod.reserve("t_cross", "req_release_cross", 80, budget=100) + current_day["value"] = "2026-08-10" + assert mod.release("t_cross", "req_release_cross") is True + assert mod.get_reservation("t_cross", "req_release_cross") is None + current_day["value"] = "2026-08-09" + assert mod.get_pending_total("t_cross") == 0 + assert mod.get_used("t_cross") == 0 + + +# ── observer hooks 不 enforce(§I 契约) ───────────────────────── + + +def test_pre_api_request_observer_never_enforces(tmp_path, monkeypatch): + from hermes_cli.plugins import invoke_hook + + _, mod = _enable_budget(tmp_path, monkeypatch) + results = invoke_hook( + "pre_api_request", + api_request_id="r1", + approx_input_tokens=999999, + max_tokens=999999, + ) + assert results == [] + assert mod.get_used("t_any") == 0 + assert mod.get_pending_total("t_any") == 0 diff --git a/plugins/spike_p3_m0_cache/__init__.py b/plugins/spike_p3_m0_cache/__init__.py new file mode 100644 index 0000000000000..396562b682ab2 --- /dev/null +++ b/plugins/spike_p3_m0_cache/__init__.py @@ -0,0 +1,764 @@ +"""P3-M0 Spike · tenant-safe cache + orchestrator(llm_execution middleware). + +**Codex 第三轮 §I / §II / §III / §V 契约(修订)**: + +1. 注册 `llm_execution` middleware(`ctx.register_middleware("llm_execution", ...)`)· + 经过 `run_llm_execution_middleware`(`hermes_cli/middleware.py:187`)真正生产走位· + fail-open on callback(fork middleware 语义 · 见 middleware.py:_run_execution_chain) + +2. **控制流严格解耦 · cache eligibility ≠ budget enforcement**: + + 插件禁用 → next_call(不 lookup 不 reserve) + 插件启用 → + ├─ 硬依赖门:budget module 缺失 → fail-CLOSED(结构化 error response · 不 next_call) + ├─ tenant context 4 字段严格校验(缺/空/非白/超长)→ fail-CLOSED + ├─ budget config 校验(BudgetConfigError → fail-CLOSED · 不静默 None) + ├─ 非 chat_completions api_mode → fail-CLOSED(M0 明确缩小支持范围 · 见 §II) + ├─ 非 dict request → fail-CLOSED + │ + ├─ 判 cacheable_request(streaming/tools 非 cacheable · **但仍 reserve/settle**) + │ + ├─ cacheable_request → cache lookup + │ └─ HIT → 返回 cached + 顶层 usage=0(**不 reserve**) + │ + ├─ (MISS 或 非 cacheable_request)→ budget.reserve + │ ├─ raise BudgetExceeded → 结构化非空 error response(**不 next_call**) + │ └─ 成功 → next_call + │ ├─ 成功 → budget.settle(actual) + cache insert(若 req+resp 皆 cacheable) + │ └─ 异常 → budget.release + re-raise + +3. 严格 tenant fail-CLOSED:tenant_id / principal_id / permission_scope_version / + profile_version 任一缺失/空/非法 → **不 next_call** · 返回结构化 fail-closed response + · **禁**默认填 "system"/"0" 等安全值(Codex §III) + +4. **只**支持 `api_mode == "chat_completions"`:其他 3 mode(anthropic_messages / + bedrock_converse / codex_responses)fail-CLOSED · **不 next_call**(见 §II 明令)· + M0 缩小支持范围到 chat_completions · 其他 mode 需 M1+ 补 token 估算 + +5. 顶层 `usage` = billable_usage(全 0)· 原始生成 usage 只在 `cache_meta.origin_usage` + +6. 并发:threading.RLock 保护 LRU + insert 序 + +7. 配置: + - `plugins.spike_p3_m0_cache.enabled` **严格 bool**(v is True · 拒 "false"/1/等) + - `plugins.spike_p3_m0_cache.tenant.*`(4 字段 · str · 白名单 · len≤128) + - `plugins.spike_p3_m0_cache.tenant.daily_budget_tokens`(4 态 · BudgetConfigError 硬 fail-CLOSED) + - **不用** SPIKE_* env · **不**读 profile.yaml · **不**读 HERMES_DAILY_TOKEN_BUDGET + +8. **input token 估算**(Codex §V):不得静默 0 后宣称预算护栏成立。 + - 优先信 ctx.approx_input_tokens(若 int/float > 0) + - 否则从 request.messages content 字符数 / 4(粗略英文 heuristic · M0 approximation · 见文档) + - 加上 request.max_tokens 上限 · 得 tokens_needed +""" +from __future__ import annotations + +import copy +import hashlib +import importlib +import json +import logging +import re +import sys +import threading +from collections import OrderedDict +from typing import Any, Callable, Dict, Optional + +logger = logging.getLogger(__name__) + + +class ConfigLoadError(RuntimeError): + """The runtime config could not be read safely.""" + + +class ConfigShapeError(RuntimeError): + """The runtime config has an invalid shape for this plugin.""" + + +def _budget_module(): + """Locate the budget sibling module regardless of parent package. + + hermes_cli.plugins loads bundled plugins under ``hermes_plugins.*``; running + the plugins directly from a checkout may see ``plugins.*`` instead. Lazy + lookup avoids a hard ``from .. import`` that only works under one layout. + + **Codex §III**:budget 模块缺失时 middleware **不**得 fail-open · 由调用方(orchestrator) + 捕获 ImportError 走 fail-CLOSED response · 保持"cache 单独启用"场景可测。 + """ + parent = __name__.rsplit(".", 1)[0] if "." in __name__ else "" + if parent: + candidate = f"{parent}.spike_p3_m0_budget" + if candidate in sys.modules: + return sys.modules[candidate] + try: + return importlib.import_module(candidate) + except ImportError: + pass + raise ImportError( + "spike_p3_m0_budget module not loaded; enable spike-p3-m0-budget in plugins.enabled" + ) + + +CACHE_CONTRACT_VERSION = "v2" +CODEC_VERSION = "codec_v1" +CACHE_MAX_SIZE = 256 # LRU cap · 简单进程内 · M0 不 persistent + +# 允许 tenant_id/principal_id/permission_scope_version/profile_version 的字符白名单 +# (类比 Hermes_AI `_validate_tenant_id` · 保守集合) +_ID_ALLOWED_RE = re.compile(r"^[a-zA-Z0-9_\-]+$") +_ID_MAX_LEN = 128 + +# fail-closed 结构化 response 消息(非空 · 可展示 · finish_reason=content_filter 抑制 retry) +_BUDGET_EXCEEDED_MESSAGE = ( + "[请求已被拒绝 · tenant 每日预算已达上限 · 请稍后重试或联系管理员]" +) +_BUDGET_FAIL_CLOSED_MESSAGE = ( + "[请求已被拒绝 · 预算护栏未就绪 · 请联系管理员]" +) + + +# ── Module state · thread-safe LRU ──────────────────────────────────── + +_lock = threading.RLock() +_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + + +# ── Config loading(fork config.yaml · 不用 env) ───────────────────── + +def _load_config() -> Dict[str, Any]: + """Load fork config via ``hermes_cli.config.load_config``. + + Once the middleware has been registered, a config read failure is a + security failure, not evidence that the plugin is disabled. Returning an + empty mapping here would make ``_is_enabled`` false and silently bypass + budget enforcement through ``next_call``. + """ + try: + from hermes_cli.config import load_config + + cfg = load_config() + except Exception as exc: # noqa: BLE001 + raise ConfigLoadError("spike cache config could not be loaded") from exc + if cfg is None: + return {} + if not isinstance(cfg, dict): + raise ConfigShapeError("root config must be a mapping") + return cfg + + +def _get_plugin_config() -> Dict[str, Any]: + """Return the validated ``plugins.spike_p3_m0_cache`` mapping.""" + cfg = _load_config() + plugins_cfg = cfg.get("plugins") + if plugins_cfg is None: + return {} + if not isinstance(plugins_cfg, dict): + raise ConfigShapeError("plugins config must be a mapping") + plugin_cfg = plugins_cfg.get("spike_p3_m0_cache") + if plugin_cfg is None: + return {} + if not isinstance(plugin_cfg, dict): + raise ConfigShapeError("spike_p3_m0_cache config must be a mapping") + return plugin_cfg + + +def _is_enabled(plugin_cfg: Optional[Dict[str, Any]] = None) -> bool: + """`plugins.spike_p3_m0_cache.enabled` **严格 bool** · 只接受 literal `True`。 + + Codex §III 硬约束:拒 `bool("false") == True`(str 转 bool 陷阱)· 拒 int/list/dict。 + True → 启用;其他(False / None / "true" / 1 / [] / {} / ...)→ 视 as 禁用。 + """ + if plugin_cfg is None: + plugin_cfg = _get_plugin_config() + return plugin_cfg.get("enabled") is True + + +def _get_tenant_context( + plugin_cfg: Optional[Dict[str, Any]] = None, +) -> Optional[Dict[str, str]]: + """严格 fail-closed tenant context · 从 config `plugins.spike_p3_m0_cache.tenant` 读。 + + 所有 4 字段(tenant_id/principal_id/permission_scope_version/profile_version)必需· + 非空 · 类型 str · 字符白名单 · 任一违反 → 返回 None(fail-closed · log context_invalid). + + **禁**默认填 "system"/"0" 等安全值(Codex §III 明令)。 + + T2 首选:由 provisioning 写入该 tenant 独立 HERMES_HOME 的 config.yaml · fork + 进程仅信 config · 不信 prompt/tool_args。 + """ + if plugin_cfg is None: + plugin_cfg = _get_plugin_config() + tenant_block = plugin_cfg.get("tenant") or {} + if not isinstance(tenant_block, dict): + logger.info("spike-cache: tenant context invalid · not a mapping") + return None + required = ("tenant_id", "principal_id", "permission_scope_version", "profile_version") + resolved: Dict[str, str] = {} + for field in required: + v = tenant_block.get(field) + if v is None: + logger.info("spike-cache: tenant context invalid · missing field %s", field) + return None + if not isinstance(v, str): + logger.info("spike-cache: tenant context invalid · field %s type=%s(need str)", field, type(v).__name__) + return None + if not v.strip(): + logger.info("spike-cache: tenant context invalid · field %s empty/whitespace", field) + return None + if len(v) > _ID_MAX_LEN: + logger.info("spike-cache: tenant context invalid · field %s len=%d > %d", field, len(v), _ID_MAX_LEN) + return None + if not _ID_ALLOWED_RE.match(v): + logger.info("spike-cache: tenant context invalid · field %s non-whitelist chars", field) + return None + resolved[field] = v + return resolved + + +# ── Cacheability guards ───────────────────────────────────────────── + +def _api_mode_supported(api_mode: str) -> bool: + """M0 只支持 `chat_completions`(见 §II)· 其他 mode fail-CLOSED · 不 next_call。""" + return api_mode == "chat_completions" + + +def is_cacheable_request(request: Any, *, api_mode: str = "chat_completions") -> bool: + """判 request 是否可 lookup/insert cache(**与 budget 决策解耦**)。 + + Codex §II:streaming / tools 可以不缓存 · 但**不能不计预算**。 + """ + if not _api_mode_supported(api_mode): + return False + if not isinstance(request, dict): + return False + if request.get("stream"): + return False + tools = request.get("tools") or [] + if tools: + return False + return True + + +def is_cacheable_response(response: Any) -> bool: + """仅 finish=stop + 无 tool_calls · 支持 SDK 对象和 dict 形态。""" + try: + if isinstance(response, dict): + choices = response.get("choices") + else: + choices = getattr(response, "choices", None) + if not choices: + return False + first = choices[0] + msg = first.get("message") if isinstance(first, dict) else getattr(first, "message", None) + if not msg: + return False + tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + if tool_calls: + return False + finish = first.get("finish_reason") if isinstance(first, dict) else getattr(first, "finish_reason", None) + return finish == "stop" + except Exception: # noqa: BLE001 + return False + + +# ── Cache key builder(§2 schema v1)────────────────────────────── + +def build_cache_key( + request: Dict[str, Any], + tenant_ctx: Dict[str, str], + *, + model: str, + provider: str, + api_mode: str = "chat_completions", +) -> str: + """SHA-256 of deterministic semantic request components(schema v2). + + Besides identity, messages, and tool schema, every public request option + that can affect generation is hashed. This intentionally prefers a cache + miss over reusing a response produced with different sampling, output, + reasoning, or provider-specific options. + """ + messages = request.get("messages") or [] + messages_bytes = json.dumps(messages, sort_keys=True, ensure_ascii=False).encode("utf-8") + tools_bytes = json.dumps(request.get("tools") or [], sort_keys=True, ensure_ascii=False).encode("utf-8") + generation_options = { + key: value + for key, value in request.items() + if key not in {"messages", "tools", "stream", "_kb_collection", "_kb_version"} + and not key.startswith("_") + } + generation_options_bytes = json.dumps( + generation_options, + sort_keys=True, + ensure_ascii=False, + ).encode("utf-8") + # system prompt hash · 从 messages 首 system 消息内容 + sys_msg_content = "" + for m in messages: + if isinstance(m, dict) and m.get("role") == "system": + sys_msg_content = json.dumps(m.get("content"), sort_keys=True, ensure_ascii=False) + break + parts = { + "tenant_id": tenant_ctx["tenant_id"], + "principal_id": tenant_ctx["principal_id"], + "permission_scope_version": tenant_ctx["permission_scope_version"], + "provider": provider or "", + "model": model or "", + "api_mode": api_mode, + "system_prompt_version": hashlib.sha256(sys_msg_content.encode("utf-8")).hexdigest()[:16], + "tool_schema_version": hashlib.sha256(tools_bytes).hexdigest()[:16], + "generation_options_version": hashlib.sha256(generation_options_bytes).hexdigest(), + "profile_version": tenant_ctx["profile_version"], + "knowledge_collection": str(request.get("_kb_collection") or ""), + "knowledge_collection_version": str(request.get("_kb_version") or ""), + "normalized_semantic_messages": hashlib.sha256(messages_bytes).hexdigest(), + "cache_contract_version": CACHE_CONTRACT_VERSION, + } + canonical = json.dumps(parts, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +# ── Response归一化 · codec_v1(仅 chat_completions dict 形态)────── + +def _extract_cacheable_response(response: Any) -> Dict[str, Any]: + """Normalize provider response → cacheable dict(codec_v1)· 只 chat_completions dict.""" + def _get(obj, k, default=None): + return obj.get(k, default) if isinstance(obj, dict) else getattr(obj, k, default) + origin_usage_obj = _get(response, "usage") or {} + if not isinstance(origin_usage_obj, dict): + origin_usage_obj = { + "prompt_tokens": getattr(origin_usage_obj, "prompt_tokens", 0) or 0, + "completion_tokens": getattr(origin_usage_obj, "completion_tokens", 0) or 0, + } + choices = copy.deepcopy(_get(response, "choices")) + return { + "model": _get(response, "model"), + "choices": choices, + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, # 顶层 usage = billable_usage · 全 0 + "codec_version": CODEC_VERSION, + "cache_meta": { + "hit": True, + "cache_contract_version": CACHE_CONTRACT_VERSION, + "origin_usage": copy.deepcopy(origin_usage_obj), + "billable_usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "saved_usage": copy.deepcopy(origin_usage_obj), + "cache_lookup_cost": None, # in-memory · 通常 0 + }, + } + + +# ── Input token 估算(Codex §V:不得静默 0) ────────────────────── + +def _estimate_input_tokens_from_messages(request: Dict[str, Any]) -> int: + """从 messages content 字符数估算 input tokens · **M0 approximation**。 + + heuristic:总字符数 ÷ 4(rough 英文 char-per-token)· 中文/emoji 会低估 · + 但对 M0 预算护栏"防明显超支"目标足够 · 生产 M1+ 需接 tiktoken/tokenizer 精算 + (out-of-band 契约字段 · 见 docs/fork/budget-hook.md §5b)。 + + 保底:若 messages 全空/无 content · 返回 0 · 由调用方结合 max_tokens 判断。 + """ + if not isinstance(request, dict): + return 0 + msgs = request.get("messages") or [] + if not isinstance(msgs, list): + return 0 + total_chars = 0 + for m in msgs: + if not isinstance(m, dict): + continue + content = m.get("content") + if isinstance(content, str): + total_chars += len(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict): + t = part.get("text") + if isinstance(t, str): + total_chars += len(t) + return total_chars // 4 + + +def _estimate_needed_tokens(request: Any, ctx: Dict[str, Any]) -> int: + """总预留 = input 估算 + max_tokens · **不静默 0**(Codex §V)。 + + 优先信 ctx.approx_input_tokens(若 int/float > 0);否则从 messages 估算 · + 加上 request.max_tokens(或 ctx.max_tokens)· 得 tokens_needed。 + """ + explicit = ctx.get("approx_input_tokens") + if isinstance(explicit, bool): + explicit = None + if isinstance(explicit, (int, float)) and explicit > 0: + approx_input = int(explicit) + else: + approx_input = _estimate_input_tokens_from_messages(request if isinstance(request, dict) else {}) + max_tokens_raw = None + if isinstance(request, dict): + max_tokens_raw = request.get("max_tokens") + if max_tokens_raw is None: + max_tokens_raw = ctx.get("max_tokens") + if isinstance(max_tokens_raw, bool): + raise ValueError("max_tokens must not be bool") + try: + max_tokens = int(max_tokens_raw or 0) + except (TypeError, ValueError) as exc: + raise ValueError("max_tokens must be an integer") from exc + if max_tokens < 0: + raise ValueError("max_tokens must be >= 0") + estimated = approx_input + max_tokens + if estimated <= 0: + raise ValueError("token estimate unavailable") + return estimated + + +# ── Structured error responses(Codex §V:非空、安全、可展示) ───── + +def _fail_closed_response(model: str, *, reason_code: str, **extras: Any) -> Dict[str, Any]: + """结构化 fail-closed response · finish_reason=content_filter 抑制 retry/fallback。 + + Codex §V:BudgetExceeded / 预算护栏未就绪 时返回必须**非空** · 安全可展示 · + 不触发 retry。上层 orchestrator 若追加 retry 决策 · 需读 `cache_meta.reason_code` + 做过滤(M1+ 契约:orchestrator 遇 content_filter + cache_meta.reason_code 明确 · 不 retry)。 + """ + return { + "model": model or "", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": _BUDGET_EXCEEDED_MESSAGE if reason_code == "budget_exceeded" else _BUDGET_FAIL_CLOSED_MESSAGE, + }, + "finish_reason": "content_filter", + }], + "usage": {"prompt_tokens": 0, "completion_tokens": 0}, + "cache_meta": { + "hit": False, + "budget_exceeded": reason_code == "budget_exceeded", + "budget_fail_closed": reason_code != "budget_exceeded", + "reason_code": reason_code, + **extras, + }, + } + + +# ── Orchestrator middleware(cache + budget 显式组合 · 解耦) ───── + +def cache_and_budget_middleware( + request: Any, + next_call: Callable[[Any], Any], + **ctx: Any, +) -> Any: + """`llm_execution` middleware · 严格 Codex 第三轮 §II 控制流(解耦 cache 与 budget)。 + + 见模块 docstring §2 完整控制流。 + """ + raw_api_request_id = ctx.get("api_request_id") + raw_api_mode = ctx.get("api_mode") + raw_provider = ctx.get("provider") + raw_model = ctx.get("model") + context_values = (raw_api_request_id, raw_api_mode, raw_provider, raw_model) + if any(value is not None and not isinstance(value, str) for value in context_values): + return _fail_closed_response("", reason_code="invalid_execution_context") + api_request_id = raw_api_request_id or "" + api_mode = raw_api_mode or "chat_completions" + provider = raw_provider or "" + model = raw_model or "" + + # 1. Resolve config exactly once. A registered middleware whose config + # cannot be read/validated must not be mistaken for a disabled plugin: + # the outer execution chain is intentionally fail-open for ordinary + # plugins, so leaking an exception here would call the provider directly. + try: + plugin_cfg = _get_plugin_config() + except ConfigLoadError: + logger.error("spike-cache: config load failed · fail-CLOSED") + return _fail_closed_response(model, reason_code="config_load_failed") + except ConfigShapeError: + logger.error("spike-cache: config shape invalid · fail-CLOSED") + return _fail_closed_response(model, reason_code="config_shape_invalid") + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: unexpected config resolution failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="config_internal_error") + + # 2. Only an explicitly disabled plugin may pass through. Missing/false + # ``enabled`` is a deliberate off switch; read/shape failures above are + # never converted into this branch. + try: + enabled = _is_enabled(plugin_cfg) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: enabled-state resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="config_enabled_error") + if not enabled: + return next_call(request) + + # 3. Budget module 硬依赖门(Codex §III:cache 单独启用不得因 budget 缺失 fail-open) + try: + budget_mod = _budget_module() + except ImportError: + logger.error( + "spike-cache: budget module missing · enable spike-p3-m0-budget in plugins.enabled · fail-CLOSED", + ) + return _fail_closed_response(model, reason_code="budget_module_missing") + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: budget module resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="budget_module_error") + + # 4. api_mode 支持范围收紧(Codex §II:非 chat_completions fail-CLOSED · 不静默 pass-through) + try: + api_mode_supported = _api_mode_supported(api_mode) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: api-mode classification failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="api_mode_error") + if not api_mode_supported: + logger.warning( + "spike-cache: unsupported api_mode=%s · M0 only supports chat_completions · fail-CLOSED", + api_mode, + ) + return _fail_closed_response(model, reason_code="unsupported_api_mode", api_mode=api_mode) + + # 5. request 类型门(non-dict 无法 estimate token · fail-CLOSED) + if not isinstance(request, dict): + logger.warning("spike-cache: request not a dict · fail-CLOSED · type=%s", type(request).__name__) + return _fail_closed_response(model, reason_code="non_dict_request") + + # Provider and model are mandatory cache identity fields. Empty values + # would collapse unrelated upstream attempts into the same key. + if not provider or not model: + logger.warning("spike-cache: provider/model context missing · fail-CLOSED") + return _fail_closed_response(model, reason_code="missing_provider_or_model") + + # 6. tenant context 严格 fail-CLOSED(Codex §III:enforcement 开启时 next_call == 0) + try: + tenant_ctx = _get_tenant_context(plugin_cfg) + tenant_id = tenant_ctx["tenant_id"] if tenant_ctx is not None else None + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: tenant context resolution failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="tenant_context_error") + if tenant_ctx is None: + return _fail_closed_response(model, reason_code="tenant_context_invalid") + assert tenant_id is not None + + # 7. Budget config 校验 · BudgetConfigError 硬 fail-CLOSED(Codex §III:不转 None 静默继续) + tenant_cfg = plugin_cfg.get("tenant") + # A valid tenant context guarantees this is a mapping. + raw_budget = tenant_cfg.get("daily_budget_tokens") + try: + budget_value = budget_mod.validate_daily_budget(raw_budget) + except budget_mod.BudgetConfigError: + logger.error("spike-cache: daily_budget_tokens config invalid · fail-CLOSED") + return _fail_closed_response(model, reason_code="budget_config_invalid") + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: budget config validation failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="budget_config_error") + + # 8. Cache eligibility 判定(与 budget 决策解耦 · Codex §II step 4) + try: + cacheable_req = is_cacheable_request(request, api_mode=api_mode) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: request classification failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="request_classification_error") + + # 9. 若可缓存 → lookup · HIT 直接返回(不 reserve). Any internal + # pre-provider failure is converted locally to fail-CLOSED so it cannot + # escape into the outer middleware chain's fail-open policy. + cache_key: Optional[str] = None + if cacheable_req: + try: + cache_key = build_cache_key( + request, + tenant_ctx, + model=model, + provider=provider, + api_mode=api_mode, + ) + with _lock: + hit = _cache.get(cache_key) + if hit is not None: + _cache.move_to_end(cache_key) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: cache lookup/key failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="cache_lookup_error") + if hit is not None: + # cache hit · 不 reserve · 顶层 usage=0 · shallow copy 防调用方 mutate + return copy.deepcopy(hit) + + # 10. MISS 或非 cacheable · budget=None 明确表示不启用护栏,因此既不 + # 需要 api_request_id,也不做 token estimation/reserve/settle。 + reservation_active = budget_value is not None + tokens_needed: Optional[int] = None + if reservation_active: + if not api_request_id: + logger.warning("spike-cache: empty api_request_id · fail-CLOSED · cannot track reservation") + return _fail_closed_response(model, reason_code="missing_api_request_id") + try: + tokens_needed = _estimate_needed_tokens(request, ctx) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: token estimation failed · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="token_estimation_error") + try: + if reservation_active: + assert tokens_needed is not None + budget_mod.reserve(tenant_id, api_request_id, tokens_needed, budget=budget_value) + except budget_mod.BudgetExceeded: + logger.warning( + "spike-cache: budget exceeded · tenant=%s api_request_id=%s tokens_needed=%s budget=%s", + tenant_id, api_request_id, tokens_needed, budget_value, + ) + return _fail_closed_response( + model, + reason_code="budget_exceeded", + ) + except Exception as exc: # noqa: BLE001 + # Defensive cleanup in case a future reserve implementation mutates + # state before raising. Never expose exception text to the user. + if reservation_active: + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass + logger.error( + "spike-cache: budget reserve internal failure · type=%s · fail-CLOSED", + type(exc).__name__, + ) + return _fail_closed_response(model, reason_code="budget_reserve_error") + + # 12. next_call · success settle actual / exception release. This narrow + # try block deliberately preserves downstream exception identity. + try: + response = next_call(request) + except Exception: + if reservation_active: + try: + budget_mod.release(tenant_id, api_request_id) + except Exception as cleanup_exc: # noqa: BLE001 + logger.error( + "spike-cache: reservation release failed after provider error · type=%s", + type(cleanup_exc).__name__, + ) + raise + + # 13. Settle 实际用量(所有非异常路径都 settle · cacheable/非 cacheable 无关) + if reservation_active: + assert tokens_needed is not None + actual = _extract_actual_usage(response) + if actual is None: + actual = tokens_needed + try: + budget_mod.settle(tenant_id, api_request_id, actual) + except Exception as exc: # noqa: BLE001 + # Provider cost has already been incurred. Do not throw into the + # outer fail-open chain after next_call; clean pending state best + # effort, record only the exception type, and preserve the response. + try: + budget_mod.release(tenant_id, api_request_id) + except Exception: # noqa: BLE001 + pass + logger.error( + "spike-cache: budget settle internal failure · type=%s", + type(exc).__name__, + ) + + # 14. 只有 req + resp 皆 cacheable 才 insert. Cache bookkeeping is + # best-effort after a paid provider response and must never replace it. + try: + cacheable_resp = ( + cacheable_req + and cache_key is not None + and is_cacheable_response(response) + ) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: response classification failed after provider response · type=%s", + type(exc).__name__, + ) + cacheable_resp = False + if cacheable_resp: + try: + cached = _extract_cacheable_response(response) + with _lock: + _cache[cache_key] = cached + _cache.move_to_end(cache_key) + while len(_cache) > CACHE_MAX_SIZE: + _cache.popitem(last=False) + except Exception as exc: # noqa: BLE001 + logger.error( + "spike-cache: cache insert failed after provider response · type=%s", + type(exc).__name__, + ) + + return response + + +def _extract_actual_usage(response: Any) -> Optional[int]: + """从 response.usage 抽 token 数;缺失/非法返回 None 触发 reservation fallback。""" + try: + if isinstance(response, dict): + usage = response.get("usage") + else: + usage = getattr(response, "usage", None) + if usage is None: + return None + if isinstance(usage, dict): + if "prompt_tokens" not in usage and "completion_tokens" not in usage: + return None + prompt = usage.get("prompt_tokens", 0) + completion = usage.get("completion_tokens", 0) + else: + if not hasattr(usage, "prompt_tokens") and not hasattr(usage, "completion_tokens"): + return None + prompt = getattr(usage, "prompt_tokens", 0) + completion = getattr(usage, "completion_tokens", 0) + if ( + isinstance(prompt, bool) + or isinstance(completion, bool) + or not isinstance(prompt, int) + or not isinstance(completion, int) + or prompt < 0 + or completion < 0 + ): + return None + return prompt + completion + except Exception: # noqa: BLE001 + return None + + +def clear_cache() -> None: + """测试用 · 清空 LRU + reset budget bucket。""" + with _lock: + _cache.clear() + + +def cache_size() -> int: + with _lock: + return len(_cache) + + +def register(ctx: Any) -> None: + """PluginContext register · 挂 llm_execution middleware(orchestrator).""" + ctx.register_middleware("llm_execution", cache_and_budget_middleware) diff --git a/plugins/spike_p3_m0_cache/plugin.yaml b/plugins/spike_p3_m0_cache/plugin.yaml new file mode 100644 index 0000000000000..c659f446c8898 --- /dev/null +++ b/plugins/spike_p3_m0_cache/plugin.yaml @@ -0,0 +1,6 @@ +name: spike-p3-m0-cache +version: "0.0.1" +description: "P3-M0 Spike · tenant-safe cache + budget orchestrator · registers llm_execution middleware (single seam) · calls sibling spike-p3-m0-budget service · memory-only LRU · fail-closed without tenant context · non-streaming + tool-free + successful terminal responses only · usage split into origin/billable/saved · off by default (opt-in via plugins.enabled)." +author: "Hermes_AI × NousResearch/hermes-agent Spike" +provides_hooks: + - "llm_execution (middleware)" diff --git a/plugins/spike_p3_m0_cache/tests/__init__.py b/plugins/spike_p3_m0_cache/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py new file mode 100644 index 0000000000000..baf792cddcf4c --- /dev/null +++ b/plugins/spike_p3_m0_cache/tests/test_cache_middleware.py @@ -0,0 +1,1083 @@ +"""P3-M0 Spike · cache + budget orchestrator middleware 测试(**真实生产走位**). + +**Codex 第三轮 §I / §II / §III / §V / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 走真实 `PluginManager.discover_and_load` +- `has_middleware("llm_execution") == True` +- 通过 `run_llm_execution_middleware` 触发 · **不**直接调 `cache_and_budget_middleware` +- **不用** `SPIKE_*` env(fork AGENTS.md red-line 3)· **不污染**全局 manager +- **cache eligibility ≠ budget enforcement 严格解耦**: + * streaming / tools 非 cacheable · **但仍 reserve/settle** + * 非 chat_completions api_mode → fail-CLOSED(next_call == 0)· M0 缩小支持范围 + * budget 模块缺失 / tenant context 缺失 / budget config 非法 → fail-CLOSED · next_call == 0 +- cache miss + cacheable req → reserve/settle 各 1 次 +- cache hit → 不 reserve · 顶层 usage=0 +- provider exception → release 且原样传播 +- 严格 bool `enabled`(bool("false")==True 陷阱阻断) +""" +from __future__ import annotations + +import pytest + +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + get_discovered_module, + install_fresh_manager, + make_run_llm_execution, + write_config, +) + + +CACHE_KEY = "spike-p3-m0-cache" +BUDGET_KEY = "spike-p3-m0-budget" + + +# ── shared valid tenant block ───────────────────────────────────────── + +_VALID_TENANT = { + "tenant_id": "t_alpha", + "principal_id": "p_alpha", + "permission_scope_version": "1", + "profile_version": "1", +} + + +def _enable_cache( + tmp_path, + monkeypatch, + *, + daily_budget=None, + tenant=_VALID_TENANT, + enabled=True, + include_budget=True, +): + tenant_cfg = dict(tenant) + if daily_budget is not None: + tenant_cfg["daily_budget_tokens"] = daily_budget + enabled_list = [CACHE_KEY] + if include_budget: + enabled_list.append(BUDGET_KEY) + write_config( + tmp_path, + enabled_list, + spike_p3_m0_cache={ + "enabled": enabled, + "tenant": tenant_cfg, + }, + ) + manager = install_fresh_manager(monkeypatch, tmp_path) + cache_mod = get_discovered_module(manager, CACHE_KEY) + cache_mod.clear_cache() + budget_mod = None + if include_budget: + budget_mod = get_discovered_module(manager, BUDGET_KEY) + budget_mod.reset_state() + return manager, cache_mod, budget_mod + + +def _is_fail_closed(resp): + """Structured fail-closed response 特征:cache_meta 含 budget_exceeded 或 budget_fail_closed。""" + if not isinstance(resp, dict): + return False + meta = resp.get("cache_meta") or {} + return bool(meta.get("budget_exceeded") or meta.get("budget_fail_closed")) + + +# ── Registration invariants(§I 强要求) ───────────────────────────── + + +def test_discover_registers_llm_execution_middleware(tmp_path, monkeypatch): + from hermes_cli.plugins import has_middleware + + manager, _, _ = _enable_cache(tmp_path, monkeypatch) + assert has_middleware("llm_execution") is True + assert len(manager._middleware.get("llm_execution", [])) == 1 + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_middleware + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_middleware("llm_execution") is False + + +def test_enabled_bool_strict_true_only(tmp_path, monkeypatch): + """Codex §III:`enabled` 严格 bool True · 拒 str/int/其他(bool('false')==True 陷阱)。""" + # enabled = "false"(str)· 应视 as 禁用 · 直通 next_call + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled="false", + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + # 未启用 → 直通 next · 未 reserve + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +@pytest.mark.parametrize("enabled_value", [1, "true", "yes", [1], {"x": 1}]) +def test_enabled_non_bool_treated_as_disabled(tmp_path, monkeypatch, enabled_value): + """任何非 literal-True 值(int / str / list / dict)· 视 as 禁用 · 直通 next。""" + _, _, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, enabled=enabled_value, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 1 + assert not _is_fail_closed(resp) + + +# ── Cache miss(cacheable request) → reserve+settle 各 1 次 ───── + + +def test_miss_reserves_and_settles_once(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + call_count = {"next": 0} + + def next_call(_req): + call_count["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 50}, + } + + run = make_run_llm_execution( + api_request_id="req_miss_1", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + approx_input_tokens=200, + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + + assert call_count["next"] == 1 + assert resp["choices"][0]["message"]["content"] == "hi" + # settle 累加 actual usage = 150 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 150 + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Cache hit → budget reserve 0 次 · 顶层 usage=0(§I 强要求) ───── + + +def test_hit_no_reserve_and_zero_top_usage(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 200, "completion_tokens": 80}, + } + + run = make_run_llm_execution( + api_request_id="req_a", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + approx_input_tokens=300, + ) + req = {"messages": [{"role": "user", "content": "same"}]} + + # 首次 miss · settle 280 + run(req, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 + + # 二次 hit · next_call 0 次 · used 不变(reserve 0 次) + run2 = make_run_llm_execution( + api_request_id="req_b", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + ) + resp = run2(req, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 280 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp["cache_meta"]["hit"] is True + assert resp["cache_meta"]["billable_usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + assert resp["cache_meta"]["origin_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["saved_usage"] == {"prompt_tokens": 200, "completion_tokens": 80} + assert resp["cache_meta"]["cache_contract_version"] == cache_mod.CACHE_CONTRACT_VERSION + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +# ── Budget 超支 → next_call 0 次(§I / §V 强要求 · 非空结构化响应) ─ + + +def test_budget_exceeded_blocks_next_call_returns_safe_response(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100 + ) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"model": "m", "choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="req_over", provider="openai", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, + next_call, + ) + # 超支明确 short-circuit · next 未调 + assert counters["next"] == 0 + # 非空结构化 error response · 抑制 retry · Codex §V + assert resp["cache_meta"]["budget_exceeded"] is True + assert resp["cache_meta"]["reason_code"] == "budget_exceeded" + # 用户可见响应不得暴露租户标识、预算值或内部估算。 + assert "tenant_id" not in resp["cache_meta"] + assert "budget" not in resp["cache_meta"] + assert "tokens_needed" not in resp["cache_meta"] + # 非空 assistant message(可展示给用户) + assert resp["choices"][0]["message"]["role"] == "assistant" + assert resp["choices"][0]["message"]["content"] # 非空字符串 + # finish_reason 抑制 retry(content_filter) + assert resp["choices"][0]["finish_reason"] == "content_filter" + # 顶层 usage=0 + assert resp["usage"] == {"prompt_tokens": 0, "completion_tokens": 0} + # 无 pending reservation(reserve 抛异常 · 未记账) + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + + +def test_budget_exceeded_on_streaming_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:streaming 请求 + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_stream", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "stream": True, "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 # streaming 也被 reserve 门阻断 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_budget_exceeded_on_tools_also_blocks_next_call(tmp_path, monkeypatch): + """Codex §V:tools request + 预算超限 · next_call 也必须 0 次(解耦证明)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=50 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r_tools", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run( + {"messages": [{"role": "user", "content": "x"}], "tools": [{"name": "search"}], "max_tokens": 100}, + next_call, + ) + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + + +def test_non_cacheable_request_still_reserves_and_settles(tmp_path, monkeypatch): + """Codex §II:streaming/tools 非 cacheable · **但仍 reserve/settle**(解耦)。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "streamed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 60, "completion_tokens": 40}} + + # streaming request(不缓存 · 但必须 reserve) + run = make_run_llm_execution( + api_request_id="r_s1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 1 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 100 # settle 60+40 + # 不 cache · 二次同请求仍 miss(需 fresh api_request_id) + run2 = make_run_llm_execution( + api_request_id="r_s2", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + run2({"messages": [{"role": "user", "content": "x"}], "stream": True}, next_call) + assert counters["next"] == 2 # 再调 next + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 # 再 settle 100 + + +# ── Provider exception → reservation 释放 + 传播 ──────────────────── + + +def test_provider_exception_releases_reservation(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=10_000 + ) + + class ProviderBoom(RuntimeError): + pass + + def next_call(_req): + raise ProviderBoom("upstream 500") + + run = make_run_llm_execution( + api_request_id="req_boom", provider="openai", model="m", api_mode="chat_completions", + approx_input_tokens=100, + ) + with pytest.raises(ProviderBoom, match="upstream 500"): + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) + + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 0 + assert budget_mod.get_reservation(_VALID_TENANT["tenant_id"], "req_boom") is None + + +def test_provider_exception_identity_survives_release_failure(tmp_path, monkeypatch): + """清理失败不得遮蔽 provider 原异常。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + class ProviderBoom(RuntimeError): + pass + + def next_call(_req): + raise ProviderBoom("provider-original") + + def release_boom(*_args, **_kwargs): + raise RuntimeError("cleanup-failed") + + monkeypatch.setattr(budget_mod, "release", release_boom) + run = make_run_llm_execution( + api_request_id="req_release_boom", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=100, + ) + with pytest.raises(ProviderBoom, match="provider-original"): + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 100}, next_call) + + +def test_null_budget_skips_reservation_and_settle(tmp_path, monkeypatch): + """null 是合法无限态:允许 provider,且不制造 unknown-settle 错误。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=None) + calls = {"reserve": 0, "settle": 0, "next": 0} + + monkeypatch.setattr( + budget_mod, + "reserve", + lambda *_args, **_kwargs: calls.__setitem__("reserve", calls["reserve"] + 1), + ) + monkeypatch.setattr( + budget_mod, + "settle", + lambda *_args, **_kwargs: calls.__setitem__("settle", calls["settle"] + 1), + ) + + def next_call(_req): + calls["next"] += 1 + return { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2}, + } + + run = make_run_llm_execution( + api_request_id="req_null_budget", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=10, + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert resp["choices"][0]["message"]["content"] == "ok" + assert calls == {"reserve": 0, "settle": 0, "next": 1} + + +def test_null_budget_requires_neither_request_id_nor_token_estimate(tmp_path, monkeypatch): + """null 明确关闭预算护栏;miss 不应被预算专用字段阻断。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=None) + calls = {"reserve": 0, "settle": 0, "next": 0} + monkeypatch.setattr( + budget_mod, + "reserve", + lambda *_args, **_kwargs: calls.__setitem__("reserve", calls["reserve"] + 1), + ) + monkeypatch.setattr( + budget_mod, + "settle", + lambda *_args, **_kwargs: calls.__setitem__("settle", calls["settle"] + 1), + ) + + def next_call(_req): + calls["next"] += 1 + return { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2}, + } + + run = make_run_llm_execution( + api_request_id="", + provider="openai", + model="m", + api_mode="chat_completions", + ) + response = run( + {"messages": [], "max_tokens": "not-needed-with-null-budget"}, + next_call, + ) + assert response["choices"][0]["message"]["content"] == "ok" + assert calls == {"reserve": 0, "settle": 0, "next": 1} + + +def test_missing_usage_settles_reserved_estimate(tmp_path, monkeypatch): + """provider 未回 usage 时按预留值结算,禁止按 0 洗预算。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + def next_call(_req): + return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}]} + + run = make_run_llm_execution( + api_request_id="req_missing_usage", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=123, + max_tokens=77, + ) + run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 77}, next_call) + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 + assert budget_mod.get_pending_total(_VALID_TENANT["tenant_id"]) == 0 + + +@pytest.mark.parametrize( + "usage", + [ + {"prompt_tokens": "5", "completion_tokens": 2}, + {"prompt_tokens": 1.5, "completion_tokens": 2}, + {"prompt_tokens": -1, "completion_tokens": 2}, + {"prompt_tokens": True, "completion_tokens": 2}, + ], +) +def test_invalid_usage_types_settle_reserved_estimate(tmp_path, monkeypatch, usage): + """非严格非负整数 usage 不可信,必须按 reservation 保守结算。""" + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + run = make_run_llm_execution( + api_request_id="req_invalid_usage", + provider="openai", + model="m", + api_mode="chat_completions", + approx_input_tokens=123, + max_tokens=77, + ) + run( + {"messages": [{"role": "user", "content": "x"}], "max_tokens": 77}, + lambda _req: { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": usage, + }, + ) + assert budget_mod.get_used(_VALID_TENANT["tenant_id"]) == 200 + + +def test_cache_hit_returns_deep_copy(tmp_path, monkeypatch): + """调用方修改一次 hit 结果不得污染后续缓存。""" + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + calls = {"next": 0} + + def next_call(_req): + calls["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "stable"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}, + } + + req = {"messages": [{"role": "user", "content": "same"}]} + make_run_llm_execution( + api_request_id="copy_miss", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit = make_run_llm_execution( + api_request_id="copy_hit_1", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + first_hit["choices"][0]["message"]["content"] = "poisoned" + first_hit["cache_meta"]["origin_usage"]["prompt_tokens"] = 999 + + second_hit = make_run_llm_execution( + api_request_id="copy_hit_2", provider="openai", model="gpt-4o", + api_mode="chat_completions", approx_input_tokens=10, + )(req, next_call) + assert calls["next"] == 1 + assert second_hit["choices"][0]["message"]["content"] == "stable" + assert second_hit["cache_meta"]["origin_usage"]["prompt_tokens"] == 5 + + +def test_cache_hit_does_not_require_budget_request_id(tmp_path, monkeypatch): + """命中不调用 provider/预算,因此不应要求 api_request_id。""" + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + calls = {"next": 0} + request = {"messages": [{"role": "user", "content": "same"}]} + + def next_call(_req): + calls["next"] += 1 + return { + "model": "gpt-4o", + "choices": [{"message": {"content": "cached"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}, + } + + make_run_llm_execution( + api_request_id="cache_seed", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + approx_input_tokens=10, + )(request, next_call) + hit = make_run_llm_execution( + api_request_id="", + provider="openai", + model="gpt-4o", + api_mode="chat_completions", + )(request, next_call) + assert calls["next"] == 1 + assert hit["cache_meta"]["hit"] is True + + +# ── Tenant context / api_mode / non-dict / budget config fail-CLOSED ─ + + +@pytest.mark.parametrize( + "bad_tenant", + [ + {"tenant_id": "", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "t", "principal_id": "p", "permission_scope_version": "1"}, + {"tenant_id": "t/../evil", "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + {"tenant_id": "a" * 200, "principal_id": "p", "permission_scope_version": "1", "profile_version": "1"}, + ], + ids=["empty-tenant", "empty-principal", "missing-profile-version", "non-whitelist-chars", "over-max-len"], +) +def test_invalid_tenant_fail_closed_blocks_next_call(tmp_path, monkeypatch, bad_tenant): + """Codex §III:enforcement 开启时 · 缺失/非法 tenant · **next_call == 0** · 结构化 fail-closed。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=100, tenant=bad_tenant + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + approx_input_tokens=500, + ) + resp = run({"messages": [{"role": "user", "content": "x"}], "max_tokens": 500}, next_call) + assert counters["next"] == 0 # fail-CLOSED · 严格 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" + assert resp["choices"][0]["message"]["content"] # 非空可展示 error + assert resp["choices"][0]["finish_reason"] == "content_filter" + + +def test_no_tenant_config_fail_closed(tmp_path, monkeypatch): + """config.yaml 无 tenant 段 · fail-CLOSED(不走 next_call)。""" + write_config( + tmp_path, + [CACHE_KEY, BUDGET_KEY], + spike_p3_m0_cache={"enabled": True}, # 无 tenant 段 + ) + install_fresh_manager(monkeypatch, tmp_path) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "hi"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "tenant_context_invalid" + + +def test_non_chat_completions_mode_fail_closed(tmp_path, monkeypatch): + """Codex §II:非 chat_completions api_mode 明确 fail-CLOSED(不静默 pass-through)。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="responses", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "unsupported_api_mode" + assert resp["cache_meta"]["api_mode"] == "responses" + + +def test_non_dict_request_fail_closed(tmp_path, monkeypatch): + """非 dict request 无法 estimate token · fail-CLOSED。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return None + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run(["not", "a", "dict"], next_call) # list · 非 dict + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "non_dict_request" + + +@pytest.mark.parametrize("bad_budget", [True, False, -1, "1000", 1.5]) +def test_invalid_budget_config_fail_closed(tmp_path, monkeypatch, bad_budget): + """Codex §III:BudgetConfigError 硬 fail-CLOSED · **不**转 None 静默继续。""" + manager, cache_mod, budget_mod = _enable_cache( + tmp_path, monkeypatch, daily_budget=bad_budget + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "budget_config_invalid" + + +def test_missing_api_request_id_fail_closed(tmp_path, monkeypatch): + """空 api_request_id · reserve 无法追踪 · fail-CLOSED。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "missing_api_request_id" + + +@pytest.mark.parametrize( + ("provider", "model"), + [("", "m"), ("openai", "")], +) +def test_missing_provider_or_model_fail_closed( + tmp_path, monkeypatch, provider, model, +): + """缓存身份字段缺失时不得生成可碰撞的空值 key。""" + _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + + response = make_run_llm_execution( + api_request_id="req_identity", + provider=provider, + model=model, + api_mode="chat_completions", + )({"messages": [{"role": "user", "content": "x"}]}, next_call) + assert counters["next"] == 0 + assert response["cache_meta"]["reason_code"] == "missing_provider_or_model" + + +def test_non_string_execution_context_fail_closed(tmp_path, monkeypatch): + _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + counters = {"next": 0} + + response = make_run_llm_execution( + api_request_id="req_identity", + provider={"unexpected": "mapping"}, + model="m", + api_mode="chat_completions", + )( + {"messages": [{"role": "user", "content": "x"}]}, + lambda _req: counters.__setitem__("next", counters["next"] + 1), + ) + assert counters["next"] == 0 + assert response["cache_meta"]["reason_code"] == "invalid_execution_context" + + +# ── Cache alone(budget disabled)· 硬依赖门 fail-CLOSED ───────────── + + +def test_cache_alone_without_budget_fail_closed(tmp_path, monkeypatch): + """Codex §III:cache 单独启用时 budget 缺失 · **必须 fail-CLOSED** · 不 pass-through。""" + # 只启用 cache · 不启用 budget · 但 config tenant 完整 + manager, cache_mod, _ = _enable_cache( + tmp_path, monkeypatch, daily_budget=1000, include_budget=False, + ) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "x"}, "finish_reason": "stop"}], "usage": {}} + + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": "x"}]}, next_call) + # budget module 未加载 · 硬 fail-CLOSED + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_fail_closed"] is True + assert resp["cache_meta"]["reason_code"] == "budget_module_missing" + + +# ── Cache key schema stability(§2 v1 schema) ────────────────────── + + +def test_cache_key_stable_and_deterministic(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": [{"role": "user", "content": "hi"}]} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="gpt-4o", provider="openai") + assert k1 == k2 and len(k1) == 64 + + +def test_cache_key_changes_on_provider(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + ctx = dict(_VALID_TENANT) + k1 = cache_mod.build_cache_key(req, ctx, model="m", provider="openai") + k2 = cache_mod.build_cache_key(req, ctx, model="m", provider="anthropic") + assert k1 != k2 + + +@pytest.mark.parametrize( + ("field", "first", "second"), + [ + ("temperature", 0, 0.8), + ("top_p", 0.9, 1.0), + ("max_tokens", 64, 128), + ("response_format", {"type": "text"}, {"type": "json_object"}), + ("seed", 1, 2), + ("extra_body", {"reasoning": {"effort": "low"}}, {"reasoning": {"effort": "high"}}), + ], +) +def test_cache_key_changes_on_generation_options( + tmp_path, monkeypatch, field, first, second, +): + """相同 messages 但生成语义不同,必须产生不同 cache key。""" + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + ctx = dict(_VALID_TENANT) + base = {"messages": [{"role": "user", "content": "same"}]} + req_a = dict(base, **{field: first}) + req_b = dict(base, **{field: second}) + key_a = cache_mod.build_cache_key(req_a, ctx, model="m", provider="openai") + key_b = cache_mod.build_cache_key(req_b, ctx, model="m", provider="openai") + assert key_a != key_b + + +def test_cache_key_changes_on_tenant(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + req = {"messages": []} + a = dict(_VALID_TENANT, tenant_id="tA") + b = dict(_VALID_TENANT, tenant_id="tB") + ka = cache_mod.build_cache_key(req, a, model="m", provider="p") + kb = cache_mod.build_cache_key(req, b, model="m", provider="p") + assert ka != kb + + +# ── Tool-call response 不 insert · 下次仍 miss ──────────────────── + + +def test_tool_calls_response_not_inserted(tmp_path, monkeypatch): + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=10_000) + + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "choices": [{"message": {"tool_calls": [{"id": "1"}]}, "finish_reason": "tool_calls"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + } + + run1 = make_run_llm_execution(api_request_id="r1", provider="p", model="m", api_mode="chat_completions") + run2 = make_run_llm_execution(api_request_id="r2", provider="p", model="m", api_mode="chat_completions") + req = {"messages": [{"role": "user", "content": "call tool"}]} + run1(req, next_call) + run2(req, next_call) + assert counters["next"] == 2 + + +# ── Input token estimation (Codex §V:不静默 0) ──────────────────── + + +def test_input_tokens_estimated_from_messages_when_no_ctx(tmp_path, monkeypatch): + """无 ctx.approx_input_tokens · 从 messages char count / 4 估算 · 不静默 0。""" + manager, cache_mod, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return {"choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2}} + + # 未传 approx_input_tokens · content 长约 4000 字符 → 估算 ~1000 tokens 触发超支 + long_content = "x" * 4000 + run = make_run_llm_execution( + api_request_id="r1", provider="p", model="m", api_mode="chat_completions", + ) + resp = run({"messages": [{"role": "user", "content": long_content}], "max_tokens": 100}, next_call) + # 估算 = 4000/4 + 100 = 1100 · budget=1000 · 超支 + assert counters["next"] == 0 + assert resp["cache_meta"]["budget_exceeded"] is True + assert "tokens_needed" not in resp["cache_meta"] + + +@pytest.mark.parametrize("bad_max_tokens", [True, -1, "not-an-int"]) +def test_invalid_or_unknown_token_estimate_fails_closed( + tmp_path, monkeypatch, bad_max_tokens, +): + _, _, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + + run = make_run_llm_execution( + api_request_id="bad_estimate", + provider="p", + model="m", + api_mode="chat_completions", + ) + resp = run({"messages": [], "max_tokens": bad_max_tokens}, next_call) + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == "token_estimation_error" + + +# ── Registered middleware must not inherit the core fail-open policy ── + + +def _run_with_counter(): + counters = {"next": 0} + + def next_call(_req): + counters["next"] += 1 + return { + "choices": [{"message": {"content": "provider output"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + + run = make_run_llm_execution( + api_request_id="r_fail_closed", + provider="p", + model="m", + api_mode="chat_completions", + approx_input_tokens=10, + ) + return counters, next_call, run + + +def _assert_internal_failure_is_safe(resp, counters, reason_code): + assert counters["next"] == 0 + assert resp["cache_meta"]["reason_code"] == reason_code + assert resp["choices"][0]["message"]["content"] + rendered = repr(resp) + assert "private user input" not in rendered + assert "TOP_SECRET_INTERNAL_EXCEPTION" not in rendered + + +def test_registered_plugin_config_load_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + from hermes_cli import config as config_mod + + def boom(): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(config_mod, "load_config", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "config_load_failed") + + +@pytest.mark.parametrize( + "bad_config", + [ + {"plugins": []}, + {"plugins": {"spike_p3_m0_cache": []}}, + ], + ids=["plugins-not-mapping", "plugin-config-not-mapping"], +) +def test_registered_plugin_malformed_config_shape_fail_closed( + tmp_path, monkeypatch, bad_config, +): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + monkeypatch.setattr(cache_mod, "_load_config", lambda: bad_config) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "config_shape_invalid") + + +def test_tenant_context_internal_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(_plugin_cfg): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(cache_mod, "_get_tenant_context", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "tenant_context_error") + + +def test_cache_key_internal_error_fail_closed(tmp_path, monkeypatch): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(cache_mod, "build_cache_key", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "cache_lookup_error") + + +def test_budget_reserve_internal_error_fail_closed(tmp_path, monkeypatch): + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(budget_mod, "reserve", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "budget_reserve_error") + + +@pytest.mark.parametrize( + ("target", "reason_code"), + [ + ("_is_enabled", "config_enabled_error"), + ("_api_mode_supported", "api_mode_error"), + ("is_cacheable_request", "request_classification_error"), + ("_estimate_needed_tokens", "token_estimation_error"), + ], +) +def test_other_pre_provider_internal_errors_fail_closed( + tmp_path, monkeypatch, target, reason_code, +): + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(*_args, **_kwargs): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(cache_mod, target, boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, reason_code) + + +def test_budget_validation_internal_error_fail_closed(tmp_path, monkeypatch): + _, _, budget_mod = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(_value): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(budget_mod, "validate_daily_budget", boom) + counters, next_call, run = _run_with_counter() + resp = run( + {"messages": [{"role": "user", "content": "private user input"}]}, + next_call, + ) + _assert_internal_failure_is_safe(resp, counters, "budget_config_error") + + +def test_response_classification_error_preserves_provider_response(tmp_path, monkeypatch): + """provider 已成功后,缓存分类错误只能降级为不缓存。""" + _, cache_mod, _ = _enable_cache(tmp_path, monkeypatch, daily_budget=1000) + + def boom(_response): + raise RuntimeError("TOP_SECRET_INTERNAL_EXCEPTION") + + monkeypatch.setattr(cache_mod, "is_cacheable_response", boom) + expected = { + "choices": [{"message": {"content": "provider-success"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 3}, + } + run = make_run_llm_execution( + api_request_id="response_classification", + provider="p", + model="m", + api_mode="chat_completions", + approx_input_tokens=10, + ) + assert run({"messages": [{"role": "user", "content": "x"}]}, lambda _req: expected) is expected diff --git a/plugins/spike_p3_m0_fixtures/__init__.py b/plugins/spike_p3_m0_fixtures/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_fixtures/citation_cases.py b/plugins/spike_p3_m0_fixtures/citation_cases.py new file mode 100644 index 0000000000000..23917bea9da3d --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/citation_cases.py @@ -0,0 +1,392 @@ +"""P3-M0 · Shared strip-sources fixture(冻结 · Hermes_AI + fork 双仓共享). + +**跨仓 SHA-256 校验**:两仓 pytest read 同一份文件内容 · sha256 匹配 +才允许消费(见双仓 PR body 互链字段 "fixture SHA-256")· 任一侧修改 · 双仓 +PR 必须同步更新 hash · 否则跨仓不一致视为 BLOCKER。 + +**向后兼容基线**:Hermes_AI `hermes_devices/ops/audit.py::strip_sources` +(charter C4 · 已合入 main)· fork 侧 sanitize_presentation 至少覆盖该 +基线所有正例 + 加严 negative cases。 + +**覆盖面**: +1. 正例(必须 strip):[source:...] · [来源:...] · [source:...] · [来源:...] · + 独立行 source: / 来源: · 全角/半角括号 · 嵌套中文括号 +2. Negative(必须原样保留):bare [1] / [2][3] · Markdown footnote reference · + JSON 数组 [] · 数组下标 arr[0] · 年份 [2024] · 版本 [v1.2] · 代码块内 · + 非 source/来源 关键字的 [xxx] +3. Unicode 边界:CJK · emoji · surrogate pair · RTL +4. Streaming 字节切分点(S1 完整缓冲策略验证)· 每 1..N-1 byte 分片 +5. Interrupt 未闭合标记 · 不吞下一行 +6. Presentation vs raw transcript 边界(canonical 保留 · presentation strip) +7. Tool result 含来源 +8. History replay 转录 + +**M0 首轮 strip prototype 只实装 mode="final"** · 其他 mode(streaming/history/ +interim/tool)是 **skeleton pass-through**(fixture present · implementation +pending)。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class StripCase: + """一个 strip 测试样本。 + + Attributes: + name: 测试用例唯一 id(供 pytest parametrize)· 全 fixture 唯一 + raw: 未 strip 的原始字符串 + expected: strip 后期望的字符串(必须严格 == · 不允许"包含") + mode: 触发出口 · "final" | "history" | "streaming" | "interim" | "tool" + marker_type: 声称触及的 marker 分类 · 供 pytest 校验实际存在 + "source_inline" | "source_line" | "unclosed" | + "unicode" | "streaming_split" | "negative" | + "history_replay" | "tool_result_json" + notes: 说明(不参与断言) + """ + + name: str + raw: str + expected: str + mode: str + marker_type: str + notes: str = "" + + +# ============================================================================ +# 正例(必须 strip · 与 Hermes_AI ops/audit.py::strip_sources 向后兼容) +# ============================================================================ + +INLINE_SOURCE_CASES: list[StripCase] = [ + StripCase( + name="inline_source_ascii", + raw="Answer is X [source: doc_abc].", + expected="Answer is X.", + mode="final", + marker_type="source_inline", + notes="ASCII `[source: X]` · Hermes_AI baseline · 保末尾句点", + ), + StripCase( + name="inline_source_zh_colon", + raw="答案是 X [source:doc_abc]。", + expected="答案是 X。", + mode="final", + marker_type="source_inline", + notes="全角句点 + 半角冒号", + ), + StripCase( + name="inline_lai_yuan_half_width", + raw="答案是 X [来源: doc_abc]。", + expected="答案是 X。", + mode="final", + marker_type="source_inline", + notes="中文 `[来源: X]` · 半角括号", + ), + StripCase( + name="inline_lai_yuan_full_width", + raw="答案是 X 【来源:doc_abc】。", + expected="答案是 X。", + mode="final", + marker_type="source_inline", + notes="全角方括号 【】", + ), + StripCase( + name="inline_lai_yuan_full_width_colon", + raw="答案是 X 【来源:doc_abc】。", + expected="答案是 X。", + mode="final", + marker_type="source_inline", + notes="全角冒号 · 与半角冒号等效处理", + ), + StripCase( + name="inline_lai_yuan_paren_nested_zh", + raw="答案 【来源:FAQ(2026版)】。", + expected="答案。", + mode="final", + marker_type="source_inline", + notes="嵌套中文括号(2026版)· 一层嵌套 · 与 Hermes_AI baseline 兼容", + ), + StripCase( + name="inline_multiple_sources", + raw="点 A [source: a]。点 B [来源: b]。点 C [source:c]。", + expected="点 A。点 B。点 C。", + mode="final", + marker_type="source_inline", + notes="多个 inline source · 全部移除", + ), +] + +LINE_SOURCE_CASES: list[StripCase] = [ + StripCase( + name="line_source_ascii", + raw="Answer is X.\nsource: doc_abc", + expected="Answer is X.", + mode="final", + marker_type="source_line", + notes="独立行 `source: X` · 整行移除", + ), + StripCase( + name="line_lai_yuan_zh", + raw="答案是 X。\n来源:doc_abc", + expected="答案是 X。", + mode="final", + marker_type="source_line", + notes="独立行 `来源: X`", + ), + StripCase( + name="line_source_case_insensitive", + raw="Answer.\nSource: doc_abc", + expected="Answer.", + mode="final", + marker_type="source_line", + notes="`Source:`(首字母大写)· case-insensitive", + ), +] + + +# ============================================================================ +# Negative cases(必须原样保留 · Codex §VI 明令) +# ============================================================================ + +NEGATIVE_CASES: list[StripCase] = [ + StripCase( + name="neg_bare_bracket_number", + raw="See [1] and [2] for details.", + expected="See [1] and [2] for details.", + mode="final", + marker_type="negative", + notes="裸 `[N]` 无结构化 sources metadata · **不得**无条件 strip", + ), + StripCase( + name="neg_markdown_footnote", + raw="A footnote reference[^1] and definition below.", + expected="A footnote reference[^1] and definition below.", + mode="final", + marker_type="negative", + notes="Markdown footnote `[^N]` · 原样保留", + ), + StripCase( + name="neg_json_array", + raw='Data: {"items": [1, 2, 3]}.', + expected='Data: {"items": [1, 2, 3]}.', + mode="final", + marker_type="negative", + notes="JSON array · 括号不得 strip", + ), + StripCase( + name="neg_array_index", + raw="arr[0] and arr[10] access.", + expected="arr[0] and arr[10] access.", + mode="final", + marker_type="negative", + notes="数组下标 · 原样保留", + ), + StripCase( + name="neg_year_bracket", + raw="Published in [2024]. Version [v1.2].", + expected="Published in [2024]. Version [v1.2].", + mode="final", + marker_type="negative", + notes="年份/版本号 [2024] [v1.2] · 无 source 关键字 · 原样保留", + ), + StripCase( + name="neg_code_block_brackets", + raw="Code: `arr[i] = arr[j]` swap.", + expected="Code: `arr[i] = arr[j]` swap.", + mode="final", + marker_type="negative", + notes="代码块内的括号 · 原样保留", + ), + StripCase( + name="neg_non_source_keyword_bracket", + raw="Note [important]: check this.", + expected="Note [important]: check this.", + mode="final", + marker_type="negative", + notes="`[important]` 非 source/来源 · 原样保留", + ), +] + + +# ============================================================================ +# 未闭合标记(interrupt · 不吞下一行) +# ============================================================================ + +UNCLOSED_CASES: list[StripCase] = [ + StripCase( + name="unclosed_open_bracket_only", + raw="Answer is X [source: partial\nNext line intact.", + expected="Answer is X [source: partial\nNext line intact.", + mode="final", + marker_type="unclosed", + notes="未闭合 `[source:` · **保留下一行** · 不吞", + ), + StripCase( + name="unclosed_zh_bracket_at_end", + raw="答案 【来源:doc", + expected="答案 【来源:doc", + mode="interim", + marker_type="unclosed", + notes="流式 interrupt · 未闭合中括 · 保留(等待更多 chunk)", + ), +] + + +# ============================================================================ +# Unicode 边界 +# ============================================================================ + +UNICODE_CASES: list[StripCase] = [ + StripCase( + name="unicode_cjk_full", + raw="发货时间是 48 小时 [source: policy]。", + expected="发货时间是 48 小时。", + mode="final", + marker_type="unicode", + notes="CJK + inline source", + ), + StripCase( + name="unicode_emoji_with_source", + raw="🚀 Launched! [source: press_release] 🎉", + expected="🚀 Launched! 🎉", + mode="final", + marker_type="unicode", + notes="Emoji 多字节 · source strip 不断字符", + ), + StripCase( + name="unicode_surrogate_pair", + raw="𝐇𝐞𝐥𝐥𝐨 [source: mathml] 𝕎𝕠𝕣𝕝𝕕", + expected="𝐇𝐞𝐥𝐥𝐨 𝕎𝕠𝕣𝕝𝕕", + mode="final", + marker_type="unicode", + notes="Astral plane 字符 · surrogate pair 不切裂", + ), + StripCase( + name="unicode_rtl_arabic", + raw="مرحبا [source: greeting] بالعالم", + expected="مرحبا بالعالم", + mode="final", + marker_type="unicode", + notes="RTL 阿拉伯文 · 方向保持", + ), +] + + +# ============================================================================ +# Streaming 字节切分点(S1 完整缓冲策略验证) +# ============================================================================ + +STREAMING_BYTE_SPLIT_CASES: list[StripCase] = [ + StripCase( + name="stream_split_ascii", + raw="A [source: a] B [source: b] end.", + expected="A B end.", + mode="streaming", + marker_type="streaming_split", + notes="ASCII · 每字节 boundary 全测(见 enumerate_byte_split_chunks)", + ), + StripCase( + name="stream_split_cjk", + raw="答案 [source: doc] 是 42。", + expected="答案 是 42。", + mode="streaming", + marker_type="streaming_split", + notes="CJK · UTF-8 多字节 · 切分点可能落在字符中间 · 缓冲重组", + ), + StripCase( + name="stream_split_emoji_boundary", + raw="Done 🎉 [source: r] end", + expected="Done 🎉 end", + mode="streaming", + marker_type="streaming_split", + notes="Emoji 字节边界 · UTF-8 aware buffering", + ), +] + + +# ============================================================================ +# Tool result 含来源 +# ============================================================================ + +TOOL_RESULT_CASES: list[StripCase] = [ + StripCase( + name="tool_result_with_inline_source", + raw='{"answer": "X [source: doc_a]", "sources": ["doc_a"]}', + expected='{"answer": "X", "sources": ["doc_a"]}', + mode="tool", + marker_type="tool_result_json", + notes="JSON tool_result 内 answer 含 inline source · strip answer · 保留 sources 数组", + ), +] + + +# ============================================================================ +# History replay(session.list/resume 转录 · presentation boundary) +# ============================================================================ + +HISTORY_REPLAY_CASES: list[StripCase] = [ + StripCase( + name="history_assistant_msg_source", + raw="Previous: The result is 42 [source: analysis]. [来源: report] concurs.", + expected="Previous: The result is 42. concurs.", + mode="history", + marker_type="history_replay", + notes="History replay 出口 · presentation sanitize · raw canonical 不改", + ), +] + + +# ============================================================================ +# 汇总(供 pytest parametrize) +# ============================================================================ + +ALL_CASES: list[StripCase] = ( + INLINE_SOURCE_CASES + + LINE_SOURCE_CASES + + NEGATIVE_CASES + + UNCLOSED_CASES + + UNICODE_CASES + + STREAMING_BYTE_SPLIT_CASES + + TOOL_RESULT_CASES + + HISTORY_REPLAY_CASES +) + + +VALID_MARKER_TYPES: frozenset[str] = frozenset({ + "source_inline", + "source_line", + "negative", + "unclosed", + "unicode", + "streaming_split", + "tool_result_json", + "history_replay", +}) + +VALID_MODES: frozenset[str] = frozenset({ + "final", "history", "streaming", "interim", "tool", +}) + + +def cases_by_mode(mode: str) -> list[StripCase]: + """按出口模式筛选 cases · 供各 hook prototype 测试消费。""" + return [c for c in ALL_CASES if c.mode == mode] + + +def cases_by_marker(marker_type: str) -> list[StripCase]: + """按 marker 分类筛选 · 供分层断言。""" + return [c for c in ALL_CASES if c.marker_type == marker_type] + + +def enumerate_byte_split_chunks(raw: str) -> list[tuple[bytes, bytes]]: + """将 raw 转 UTF-8 后 · 按每 1..N-1 字节切分点返回 (head, tail) 对。 + + 专供 streaming S1 缓冲策略测试:每一切分点送 buffered sanitizer · 最终 + emit 必须等于 sanitize(raw)。切在多字节字符中间(如 UTF-8 continuation + byte)· sanitizer **必须等待更多字节** · 不得强解码为 replacement char。 + """ + encoded = raw.encode("utf-8") + n = len(encoded) + return [(encoded[:i], encoded[i:]) for i in range(1, n)] diff --git a/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py b/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py new file mode 100644 index 0000000000000..50689c02f63ce --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/hermes_ai_baseline.py @@ -0,0 +1,58 @@ +"""P3-M0 · Vendored copy of Hermes_AI `hermes_devices/ops/audit.py::strip_sources`. + +**Purpose**:cross-repo import is not always available in fork CI; +this file mirrors the Hermes_AI baseline **verbatim** so the differential test +(`plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py`) can +compare fork sanitizer output against it and lock the exact divergence. + +**Source of truth**: + Repo: https://github.com/bri12afdsarker96-lgtm/Hermes_AI + File: `hermes_devices/ops/audit.py` + Symbols mirrored: `_INLINE_RE`, `_LINE_RE`, `strip_sources` + Baseline commit: PR #27 merge `aa5c3c4f4f862b6e7bf4a9bc97f5dfeb35489e55` + (P2-0.1 段① · charter C4) + +**Sync discipline**:when the Hermes_AI baseline changes, this file MUST be +resynced in the same PR, and both repos' PR bodies must reference the drift. +The differential test's `test_baseline_regex_verbatim` byte-checks the strings. +""" +from __future__ import annotations + +import re + + +# ── mirrored from Hermes_AI hermes_devices/ops/audit.py:74-91 (verbatim) ── + +# Citation markers we strip from user-bound text. Deliberately conservative: +# only bracketed/labelled "source/来源" fragments — ordinary brackets survive. +# The body class excludes \n (an unclosed marker must NOT swallow following +# lines of real reply text — that falls through to _LINE_RE instead) and +# allows one nested bracket pair (来源:FAQ(2026版)) without leaving residue. +_INLINE_RE = re.compile( + r"[\[【((]\s*(?:source|来源)\s*[::]" + r"(?:[^\[\]【】()()\n]|[((][^()()\n]*[))]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】))]", + re.IGNORECASE, +) +_LINE_RE = re.compile(r"^[ \t]*(?:source|来源)\s*[::].*$", re.IGNORECASE | re.MULTILINE) + + +def strip_sources(text: str) -> str: + """Remove citation markers from a user-bound reply (charter C4).""" + if not text: + return text + out = _INLINE_RE.sub("", text) + out = _LINE_RE.sub("", out) + out = re.sub(r"[ \t]+(?=\n|$)", "", out) # trailing spaces left by removals + out = re.sub(r"\n{3,}", "\n\n", out) + return out.strip() + + +# Verbatim source strings — locked by test_baseline_regex_verbatim so any drift +# from the Hermes_AI baseline flips the differential test loudly. +BASELINE_INLINE_PATTERN = ( + r"[\[【((]\s*(?:source|来源)\s*[::]" + r"(?:[^\[\]【】()()\n]|[((][^()()\n]*[))]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】))]" +) +BASELINE_LINE_PATTERN = r"^[ \t]*(?:source|来源)\s*[::].*$" diff --git a/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py new file mode 100644 index 0000000000000..e2eceed0b3599 --- /dev/null +++ b/plugins/spike_p3_m0_fixtures/pluginmgr_helper.py @@ -0,0 +1,140 @@ +"""P3-M0 Spike · shared test helper · fresh PluginManager + tmp HERMES_HOME. + +**Codex §VIII 契约**:所有 3 spike 插件测试必须走真实 `PluginManager.discover_and_load` ++ 临时 `config.yaml` 启用 · `has_middleware("llm_execution") == True` ++ 通过 `run_llm_execution_middleware` / `invoke_hook` 触发 · **不用** `SPIKE_*` env ++ **不污染**全局 manager(fixture teardown 严格还原)。 + +**为什么用 fresh PluginManager 而非复用全局**: +- 全局 `_plugin_manager` 单例 · 一次 discover 后所有插件都 cached +- 测试需要:临时 config 启用 → discover → 验证 → teardown 还原 +- 用 fresh instance + monkeypatch global · 严格隔离 · zero pollution +""" +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +import yaml + + +def write_config(hermes_home: Path, enabled: List[str], **plugin_cfg: Any) -> None: + """Write ``config.yaml`` under ``hermes_home`` enabling the given plugins. + + ``plugin_cfg`` keyword args become entries under ``plugins.``. Example: + write_config(tmp, ["spike-p3-m0-cache"], spike_p3_m0_cache={"enabled": True, ...}) + """ + hermes_home.mkdir(parents=True, exist_ok=True) + cfg: Dict[str, Any] = { + "plugins": { + "enabled": list(enabled), + **plugin_cfg, + }, + } + (hermes_home / "config.yaml").write_text( + yaml.safe_dump(cfg, sort_keys=True, allow_unicode=True), encoding="utf-8" + ) + + +def install_fresh_manager(monkeypatch, hermes_home: Path, request=None): + """Replace the global PluginManager with a fresh instance for this test. + + Steps: + 1. Set ``HERMES_HOME`` env → temp dir. + 2. Force ``hermes_cli.config._load_config_impl`` cache clear. + 3. Create fresh :class:`hermes_cli.plugins.PluginManager`. + 4. Monkeypatch ``hermes_cli.plugins._plugin_manager`` → fresh instance. + 5. Call ``discover_and_load(force=True)``. + + Fixture teardown (monkeypatch scope) restores env + module attr. + Returns the fresh manager. + + **Codex 第三轮 §VI**:测试可选调用 :func:`assert_no_plugin_module_leak` (见下) + 在 teardown 阶段快照对比 `hermes_plugins.*`,证明 discover 引入的模块在 + fixture 结束后全部清除,不污染后续测试。 + """ + from hermes_cli import plugins as plugins_mod + + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + # 强制 config cache 失效 · load_config 内部 mtime_ns 键 · 换 HERMES_HOME 自然 miss + try: + from hermes_cli import config as config_mod + + if hasattr(config_mod, "_CONFIG_CACHE"): + monkeypatch.setattr(config_mod, "_CONFIG_CACHE", {}, raising=False) + except Exception: + pass + + fresh = plugins_mod.PluginManager() + monkeypatch.setattr(plugins_mod, "_plugin_manager", fresh) + + # 快照:进入 fixture 前 sys.modules 里已经存在的 hermes_plugins.* 集合 + pre_existing = {k for k in list(sys.modules) if k.startswith("hermes_plugins")} + # 卸载 hermes_plugins.* 命名空间(避免上次 discover 的旧 module 状态污染) + for name in list(pre_existing): + monkeypatch.delitem(sys.modules, name, raising=False) + + fresh.discover_and_load(force=True) + + # Codex 第三轮 §VI:teardown 显式清除 discover 引入的 hermes_plugins.* 新键, + # 确保 fixture 结束后 sys.modules 恢复到"进入 fixture 前"的精确状态。 + # 若 request 传入 · 用 pytest addfinalizer(monkeypatch 无此 API);否则记录 + # 一个显式 helper 供测试手工清理与验证。 + if request is not None: + def _cleanup_new_plugin_modules(): + for name in list(sys.modules): + if name.startswith("hermes_plugins") and name not in pre_existing: + sys.modules.pop(name, None) + request.addfinalizer(_cleanup_new_plugin_modules) + + return fresh + + +def snapshot_plugin_modules() -> set: + """Snapshot of currently-loaded ``hermes_plugins.*`` module names. + + Used by :func:`assert_no_plugin_module_leak` and the standalone test + ``test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue`` to prove + that `install_fresh_manager` + monkeypatch teardown removes every module it + caused to be inserted. + """ + return {k for k in sys.modules if k.startswith("hermes_plugins")} + + +def assert_no_plugin_module_leak(before: set) -> None: + """Assert no new ``hermes_plugins.*`` entries remain in sys.modules. + + Call **after** monkeypatch fixture teardown to verify cleanup. + """ + after = snapshot_plugin_modules() + leaked = after - before + assert not leaked, ( + f"hermes_plugins.* module leak after fixture teardown: {sorted(leaked)} · " + "install_fresh_manager teardown did not clean up loaded plugin modules" + ) + + +def get_discovered_module(manager, plugin_key: str): + """Return the imported module object of a plugin by its plugin.yaml `name`.""" + loaded = manager._plugins.get(plugin_key) + if loaded is None: + raise KeyError(f"plugin {plugin_key!r} not discovered · loaded keys: {list(manager._plugins)}") + if loaded.module is None: + raise RuntimeError(f"plugin {plugin_key!r} discovered but module is None · error={loaded.error!r}") + return loaded.module + + +def make_run_llm_execution(**default_ctx): + """Return a helper that invokes `run_llm_execution_middleware` with default ctx. + + Simplifies tests: ``run = make_run_llm_execution(api_request_id="r1", ...)`` + then ``run(request, next_call, provider="p")``. + """ + from hermes_cli.middleware import run_llm_execution_middleware + + def _run(request, next_call, **override_ctx): + ctx = {**default_ctx, **override_ctx} + return run_llm_execution_middleware(request, next_call, **ctx) + + return _run diff --git a/plugins/spike_p3_m0_strip/__init__.py b/plugins/spike_p3_m0_strip/__init__.py new file mode 100644 index 0000000000000..880e57ed54af1 --- /dev/null +++ b/plugins/spike_p3_m0_strip/__init__.py @@ -0,0 +1,84 @@ +"""P3-M0 Spike · strip-sources prototype(exit ① 普通最终响应). + +**Codex §VI fail-closed 契约**: +- sanitizer 异常 → 返回**安全占位** · 记录 audit · **不**返回 None(否则原文透出) +- 无改动 · 返回 None(标准 hook "first non-None str wins" 契约) + +**范围**: +- 仅出口 ① `transform_llm_output` @ `agent/turn_finalizer.py:561` +- streaming/interim/interrupt/history-replay/tool-result 五出口 fixture 冻结 + · **本 spike 不实装**(fixture present · implementation pending) +- `_sanitize_surrogates`(fresh baseline turn_finalizer 已加)独立 · 不冲突 + +**插件启用**:通过 fork `config.yaml` 的 `plugins.enabled` 列表 · **不**用 env。 +""" +from __future__ import annotations + +import logging +from typing import Any, Optional + +from . import sanitize as _sanitize_mod # module reference · 供 monkeypatch fail-closed 测试 + +logger = logging.getLogger(__name__) + +# fail-closed 占位 · 出现 sanitizer 崩时返回 · 提示用户重试 +FAIL_CLOSED_PLACEHOLDER = ( + "[response filtered · sanitizer error · please retry]" +) + + +def on_transform_llm_output( + *, + response_text: str, + session_id: str = "", + model: str = "", + platform: str = "", + **_kwargs: Any, +) -> Optional[str]: + """`transform_llm_output` hook · fail-closed on sanitizer exception. + + Contract(`hermes_cli/plugins.py::VALID_HOOKS`): + - 返回 non-empty str → 覆盖 `final_response`(first-wins 跨 plugins) + - 返回 None / "" → 保留 `final_response` 不变 + + Codex §VI · fail-closed 补丁: + - 无异常 · 无改动 → 返回 None + - 无异常 · 有改动 → 返回 stripped str + - 异常 → 返回 `FAIL_CLOSED_PLACEHOLDER`(**不返回原文**)· 记录 audit + """ + text = response_text or "" + try: + stripped = _sanitize_mod.sanitize_presentation(text, mode="final") + except Exception as exc: # noqa: BLE001 — fail-closed · 拒原文泄漏 + logger.error( + "spike-p3-m0-strip: sanitizer FAILED · fail-closed " + "(session_id=%s model=%s platform=%s error_class=%s)", + session_id, model, platform, type(exc).__name__, + ) + # audit event · 便于监控告警 + _emit_audit( + event="strip.sanitizer_failed", + session_id=session_id, + model=model, + platform=platform, + error_class=type(exc).__name__, + ) + return FAIL_CLOSED_PLACEHOLDER + if stripped == text: + return None # 无改动 + return stripped + + +def _emit_audit(**payload: Any) -> None: + """轻量 audit hook · 通过 lifecycle invoke_hook 触发观察者(如 langfuse/nemo_relay)。""" + try: + from hermes_cli.lifecycle import invoke_hook + + invoke_hook("api_request_error", **payload) # 复用现有 API error 通道 + except Exception: # noqa: BLE001 — audit never breaks hook + pass + + +def register(ctx: Any) -> None: + """PluginContext hook · called at plugin discovery.""" + ctx.register_hook("transform_llm_output", on_transform_llm_output) diff --git a/plugins/spike_p3_m0_strip/plugin.yaml b/plugins/spike_p3_m0_strip/plugin.yaml new file mode 100644 index 0000000000000..740bf16ed47fa --- /dev/null +++ b/plugins/spike_p3_m0_strip/plugin.yaml @@ -0,0 +1,6 @@ +name: spike-p3-m0-strip +version: "0.0.1" +description: "P3-M0 Spike · strip-sources prototype · transform_llm_output hook · fail-closed on sanitizer exception (returns safe placeholder, NOT original text). Covers exit ① normal final response only. Streaming ② / interim ③ / history-replay ④ / tool-result ⑤ exits are fixture-frozen (pending). Configured via plugins.enabled in config.yaml (no env vars). WIP · do not enable in production." +author: "Hermes_AI × NousResearch/hermes-agent Spike" +provides_hooks: + - transform_llm_output diff --git a/plugins/spike_p3_m0_strip/sanitize.py b/plugins/spike_p3_m0_strip/sanitize.py new file mode 100644 index 0000000000000..dac512ba06912 --- /dev/null +++ b/plugins/spike_p3_m0_strip/sanitize.py @@ -0,0 +1,89 @@ +r"""Presentation-boundary sanitizer(spike · mode="final" 实装 · 其他 skeleton). + +**向后兼容基线(有明确增强 · 见"差异说明")**: +- Hermes_AI `hermes_devices/ops/audit.py::strip_sources`(charter C4 · fixture SHA-256 双仓锁死) + +**Codex §VI 契约**: +- 只 strip 明确的 `[source|来源: ...]` / `【来源:...】` / 独立行 `source:` `来源:` +- 允许一层嵌套中文括号(与 Hermes_AI 一致) +- **不得**无条件 strip 裸 `[N]`(需结构化 sources metadata 配对 · 本 M0 spike 不做) +- 未闭合标记不吞下一行 +- **fail-closed** on sanitizer exception:返回安全占位 · **不**返回原文 + +**与 Hermes_AI baseline 的差异(Codex 第三轮 §VI · §7 明文锁定)**: + +fork sanitizer 相对 baseline 做了两处**用户体验增强**,由 differential test 锁死: + +| 差异 | Hermes_AI baseline | fork sanitizer | 用户可见效果 | +|---|---|---|---| +| `_INLINE_SOURCE_RE` 前导 | `[\[【((]...`(不吃前导空白) | `[ \t]?[\[【((]...`(吃 1 个前导**水平**空白) | `"Answer [source:x]." → baseline: "Answer ." · fork: "Answer."` | +| 结果末尾清理 | `.strip()`(掐首尾空白) | `.rstrip()`(仅掐尾) | 保留首行 indentation(如 `" Answer"`)· 只去尾空 | + +**关键约束(Codex 第三轮 §7 补证)**: +- 前导消耗**只**取 `[ \t]`(空格 / tab)· **不**用 `\s?`(否则会吞 `\n` 拼接下一行) +- 若 `Line A\n[source: k]Next` · 期望 `Line A\nNext`(fork 与 baseline 均保留换行 · 无行拼接) +- 空白差异**不扩大** source 删除范围:非 source 括号 `[normal]` 一律保留 + +**契约锁定**: +- baseline strip 的每个 source marker · fork 也必须 strip(**覆盖包含关系**) +- baseline 不 strip 的(negative case)· fork 也不 strip(**负例一致**) +- 差异**仅**在于两点 UX 增强(前导**水平**空白、首行 indent 保留 vs 全 strip)· 由 + `tests/test_strip_differential_vs_hermes_ai.py` 一一断言 + +**冻结 · pending 出口**: +- mode="streaming":skeleton pass-through(pending · S1 完整缓冲 · M1+) +- mode="history":skeleton pass-through(pending · presentation boundary · M1+) +- mode="interim":skeleton pass-through(pending · M1+) +- mode="tool":skeleton pass-through(pending · M1+) +""" +from __future__ import annotations + +import re + +# ── 与 Hermes_AI ops/audit.py::_INLINE_RE 差 1 个前导 `[ \t]?` ────────── +# 前导用 `[ \t]?` 而非 `\s?` · 避免吞掉 `\n` 拼接下一行(Codex §7 反例证明)。 +# body class 完全一致 · 允许一层嵌套中文/半角括号(如 [来源: FAQ(2026版)]). +_INLINE_SOURCE_RE = re.compile( + r"[ \t]?[\[【((]\s*(?:source|来源)\s*[::]" + r"(?:[^\[\]【】()()\n]|[((][^()()\n]*[))]|[\[【][^\[\]【】\n]*[\]】])*" + r"[\]】))]", + re.IGNORECASE, +) + +# ── 独立行 `source:` / `来源:` · 整行(含换行)移除 · 与 baseline 完全一致 ─ +_LINE_SOURCE_RE = re.compile( + r"^[ \t]*(?:source|来源)\s*[::].*$", + re.IGNORECASE | re.MULTILINE, +) + +# ── Trailing whitespace / 连续空行清理(sub 完的后处理 · 与 baseline 一致)─ +_TRAILING_WS_RE = re.compile(r"[ \t]+(?=\n|$)") +_MULTI_NEWLINE_RE = re.compile(r"\n{3,}") + + +class SanitizerError(RuntimeError): + """Sanitizer 内部错误 · 触发 fail-closed(见 hook `__init__.py`)。""" + + +def sanitize_presentation(text: str, *, mode: str) -> str: + """Strip source markers · 按 mode 返回 sanitized text. + + M0 首轮:仅实装 mode="final"。其他 mode 返回原字符串(skeleton pass-through · + fixture present · implementation pending)。 + """ + if mode != "final": + return text + return _sanitize_final(text) + + +def _sanitize_final(text: str) -> str: + """出口 ① · 普通最终响应 sanitize · Hermes_AI ops/audit.py 兼容 + UX 增强。""" + if not text: + return text + out = _INLINE_SOURCE_RE.sub("", text) + out = _LINE_SOURCE_RE.sub("", out) + # 清理 sub 留下的行尾空格 · 3+ 连续空行合并为 2 + out = _TRAILING_WS_RE.sub("", out) + out = _MULTI_NEWLINE_RE.sub("\n\n", out) + # rstrip 而非 strip:保留首行 indentation(见模块 docstring "差异说明") + return out.rstrip() diff --git a/plugins/spike_p3_m0_strip/tests/__init__.py b/plugins/spike_p3_m0_strip/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py new file mode 100644 index 0000000000000..6d9ca68ef12d8 --- /dev/null +++ b/plugins/spike_p3_m0_strip/tests/test_strip_differential_vs_hermes_ai.py @@ -0,0 +1,226 @@ +r"""P3-M0 · fork sanitizer vs Hermes_AI baseline · differential contract lock. + +**Codex 第三轮 §VI 强制**: +- fork sanitizer 与 Hermes_AI `ops/audit.py::strip_sources` 差异**必须**明文锁定 +- 差异**仅**允许:①`\s?` 前导空白吸收 · ②`.rstrip()` 保留首行 indentation +- 覆盖包含关系:baseline strip 的 · fork 必 strip +- 负例一致:baseline 保留的 · fork 必保留 +- 用输入/输出行为证明覆盖包含关系,不锁死正则实现文本 +""" +from __future__ import annotations + +import pytest + +from plugins.spike_p3_m0_fixtures import hermes_ai_baseline +from plugins.spike_p3_m0_fixtures.citation_cases import ALL_CASES, cases_by_marker +from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + +# ── 1. 覆盖包含关系:baseline strip 的 · fork 必 strip ───────── + + +_SOURCE_MARKER_LEFT_BRACKETS = ("[source", "[来源", "[SOURCE", "[Source", + "【source", "【来源", "(source", "(source", + "(来源", "(来源") + + +def _has_source_marker(text: str) -> bool: + """Detect any remaining `source:` / `来源:` inline marker(differential coverage).""" + lowered = text.lower() + for prefix in ("[source", "[来源", "【source", "【来源", "(source", "(source", + "(来源", "(来源"): + if prefix.lower() in lowered: + return True + return False + + +@pytest.mark.parametrize( + "case", + [c for c in ALL_CASES if c.mode == "final" and c.marker_type != "negative"], + ids=lambda c: c.name, +) +def test_fork_strips_everything_baseline_strips(case): + """Positive case:baseline 移除的每个 marker · fork 也必移除(**覆盖包含关系**)。 + + 契约锁定的两点断言(§VI): + A. 两侧输出**均无** source marker 残留(覆盖完整性) + B. 两侧输出去除**所有空白**后 byte-identical(差异仅在空白 · UX 增强范围内) + """ + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + + # A. 覆盖完整性(仅对 baseline 实际 strip 的 case 生效) + # 未闭合标记(§VI:不吞下一行)· baseline 与 fork 均保留 · 跳过此断言 + if baseline_out != case.raw: + assert not _has_source_marker(baseline_out), ( + f"case={case.name} · baseline 输出仍含 source marker · fixture 缺失或 baseline drift · " + f"baseline={baseline_out!r}" + ) + assert not _has_source_marker(fork_out), ( + f"case={case.name} · fork 输出仍含 source marker · 覆盖不完整 · fork={fork_out!r}" + ) + + # B. 差异仅在空白(核心保证:内容 byte-identical after \s+ 剥离) + import re as _re + baseline_nowhite = _re.sub(r"\s+", "", baseline_out) + fork_nowhite = _re.sub(r"\s+", "", fork_out) + assert baseline_nowhite == fork_nowhite, ( + f"case={case.name} · fork 与 baseline 在去除空白后仍不 byte-identical\n" + f" raw: {case.raw!r}\n" + f" baseline: {baseline_out!r}\n" + f" fork: {fork_out!r}\n" + f" baseline_nowhite: {baseline_nowhite!r}\n" + f" fork_nowhite: {fork_nowhite!r}\n" + "差异必须限于 §VI 声明的两点(前导空白、首尾空白)· " + "否则更新 sanitize.py 或 fixture · 双仓 PR body 同步" + ) + + +# ── 4. Negative case 一致性:baseline 不 strip · fork 也不 strip ─ + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_identical(case): + """Negative case:baseline 与 fork 都必须原样保留(strict equality)。""" + baseline_out = hermes_ai_baseline.strip_sources(case.raw) + fork_out = sanitize_presentation(case.raw, mode="final") + assert baseline_out == case.raw, ( + f"baseline unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} baseline={baseline_out!r}" + ) + assert fork_out == case.raw, ( + f"fork unexpectedly stripped negative case {case.name!r} · " + f"raw={case.raw!r} fork={fork_out!r}" + ) + + +# ── 5. UX 差异边界样例(明文示例断言) ───────────────────────── + + +def test_ux_diff_inline_leading_space(): + """UX 差异 ①:`\\s?` 前导空白吸收 · 单空格样例。""" + raw = "Answer is X [source: doc]." + assert hermes_ai_baseline.strip_sources(raw) == "Answer is X ." + assert sanitize_presentation(raw, mode="final") == "Answer is X." + + +def test_ux_diff_preserves_leading_indent(): + """UX 差异 ②:`.rstrip()` 保留首行 indentation。""" + raw = " Answer with [source: x]" + assert hermes_ai_baseline.strip_sources(raw) == "Answer with" # baseline .strip() + assert sanitize_presentation(raw, mode="final") == " Answer with" # fork .rstrip() + + +# ── 6. Codex §7 反例证明:前导差异不吞换行、不扩大删除范围 ───── + + +def test_leading_space_does_not_eat_newline_before_marker(): + """前一行的 `\\n` **不得**被前导消耗吞掉 · 否则会拼接下一行(§7 反例)。 + + 这是当前实施 `[ \\t]?` 而非 `\\s?` 的核心理由。曾用的 `\\s?` 匹配 `\\n` · + `"Line A\\n[source: kb]Next"` → `"Line ANext"`(BUG)· 现改为 `[ \\t]?` · + 结果应保持换行结构:`"Line A\\nNext"`。 + """ + raw = "Line A\n[source: kb]Next word" + fork_out = sanitize_presentation(raw, mode="final") + # baseline 不吃前导 · 直接删 marker · 结果保持换行 · 后接 Next word + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 关键断言:换行未被吞 · 无行拼接 + assert "\n" in fork_out, ( + f"fork 前导消耗吞掉了 `\\n` · 拼接了下一行 · got={fork_out!r} · " + "前导 regex 必须是 `[ \\t]?` 不是 `\\s?`" + ) + assert fork_out == "Line A\nNext word", ( + f"fork 处理跨行 marker 错 · raw={raw!r} got={fork_out!r} expected='Line A\\nNext word'" + ) + # 同时 · fork 与 baseline 保持一致(此 case 前导不消耗 · 空白差异不体现) + assert fork_out == baseline_out + + +def test_leading_space_does_not_eat_double_newline_before_marker(): + """双换行边界同理 · 前导消耗**不吃** `\\n` · 段落分隔保留。""" + raw = "Paragraph A ends here.\n\n[source: kb]Paragraph B starts." + fork_out = sanitize_presentation(raw, mode="final") + # 双 \n 应保留(不吞任何 \n · 段落分隔完整) + assert "\n\n" in fork_out, ( + f"fork 吞掉了段落分隔 · got={fork_out!r}" + ) + assert fork_out == "Paragraph A ends here.\n\nParagraph B starts." + + +def test_leading_space_only_absorbs_single_space_or_tab(): + """前导 `[ \\t]?` 只吸收 1 个空格 / tab · 不吸收其他 whitespace。""" + # 空格 · 吸收 + assert sanitize_presentation("X [source: k]Y", mode="final") == "XY" + # tab · 吸收 + assert sanitize_presentation("X\t[source: k]Y", mode="final") == "XY" + # 换行 · 不吸收(§7 硬约束) + assert sanitize_presentation("X\n[source: k]Y", mode="final") == "X\nY" + # 回车 · 不吸收(避免 Windows 场景意外拼接) + assert sanitize_presentation("X\r[source: k]Y", mode="final") == "X\rY" + + +def test_whitespace_diff_does_not_widen_source_deletion_range(): + """空白差异**不**扩大 source 删除范围:非 source 括号一律保留。 + + Codex §7:确保 `\\s?` → `[ \\t]?` 与 `.rstrip()` 的两处 UX 增强· + 不会让 fork 意外删掉 baseline 保留的内容。 + """ + # 非 source 括号(`[normal]`)紧跟 source marker · 只 source 被删 + raw = "Result: [normal] [source: kb] end." + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + assert "[normal]" in fork_out, f"fork 误删了 [normal] · got={fork_out!r}" + assert "[normal]" in baseline_out + # source marker 已删 + assert "[source" not in fork_out.lower() + # 两侧内容(去空白)一致 + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) + + +def test_trailing_whitespace_rstrip_scope_matches_baseline_content(): + """`.rstrip()` 只掐尾空白 · 不改变文本主体 · 与 baseline `.strip()` 内容一致(仅首空白 diff)。""" + raw = "Answer here [source: k]\n\n " + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # 两侧 trailing 皆掐 · 主体一致 + assert fork_out == "Answer here" + assert baseline_out == "Answer here" + + +def test_leading_whitespace_only_diff_between_baseline_and_fork(): + """首行 indentation 场景 · fork 保留 · baseline 掐 · 主体 byte-identical。""" + raw = " Answer [source: k]" + fork_out = sanitize_presentation(raw, mode="final") + baseline_out = hermes_ai_baseline.strip_sources(raw) + # fork 保留首 3 空格 · baseline 全 strip + assert fork_out == " Answer" + assert baseline_out == "Answer" + # 去空白后主体 byte-identical + import re as _re + assert _re.sub(r"\s+", "", fork_out) == _re.sub(r"\s+", "", baseline_out) == "Answer" + + +def test_middle_line_source_marker_preserves_surrounding_newlines(): + """中间行的 inline source marker · 两侧换行完整保留 · 无行拼接。""" + raw = "Line 1\nLine 2 [source: k]\nLine 3" + fork_out = sanitize_presentation(raw, mode="final") + assert fork_out == "Line 1\nLine 2\nLine 3" + + +def test_marker_at_start_of_line_preserves_prev_line_terminator(): + """行首 marker 场景 · 前一行的 `\\n` 保留 · 只删 marker + 前导水平空白(若有)。""" + raw_no_lead = "Line 1\n[source: k]Line 2 content" + raw_with_lead = "Line 1\n \t[source: k]Line 2 content" + + out_no_lead = sanitize_presentation(raw_no_lead, mode="final") + out_with_lead = sanitize_presentation(raw_with_lead, mode="final") + + # 无 leading horizontal ws · 结果 "Line 1\nLine 2 content" + assert out_no_lead == "Line 1\nLine 2 content" + # 有 leading horizontal ws · marker 前导吸收 1 个 · 剩下 " \t"? + # 实际:`[ \t]?[source: k]` 匹配 `\t[source: k]` · 前面的 " " 保留 · 结果 "Line 1\n Line 2 content" + # 关键点:换行未被吞 · 有 \n + assert "\n" in out_with_lead + assert out_with_lead == "Line 1\n Line 2 content" diff --git a/plugins/spike_p3_m0_strip/tests/test_strip_final.py b/plugins/spike_p3_m0_strip/tests/test_strip_final.py new file mode 100644 index 0000000000000..97249d35dfe59 --- /dev/null +++ b/plugins/spike_p3_m0_strip/tests/test_strip_final.py @@ -0,0 +1,289 @@ +"""P3-M0 Spike · strip prototype 出口 ① 测试(**真实 PluginManager · invoke_hook**). + +**Codex §VI / §VIII 契约**: +- 通过临时 `config.yaml` 启用插件 · 真实 `PluginManager.discover_and_load` +- 通过 `hermes_cli.plugins.invoke_hook("transform_llm_output", ...)` 触发 +- 覆盖: + * fixture SHA-256 双仓一致(cross-repo lock) + * final-mode 全 fixture case parametrize · sanitize_presentation 直调 + * 有改动 hook 返回 stripped str · 无改动返回 None + * sanitizer 异常 → hook 返回 FAIL_CLOSED_PLACEHOLDER(**fail-closed** · 不返回原文) + * negative case:bare [N] / JSON / footnote ref 等**必须**原样保留 + * streaming byte-split · UTF-8 缓冲 · sanitize 结果与整体一致 +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from plugins.spike_p3_m0_fixtures.citation_cases import ( + ALL_CASES, + StripCase, + cases_by_marker, + cases_by_mode, + enumerate_byte_split_chunks, +) +from plugins.spike_p3_m0_fixtures.pluginmgr_helper import ( + assert_no_plugin_module_leak, + get_discovered_module, + install_fresh_manager, + snapshot_plugin_modules, + write_config, +) + + +STRIP_KEY = "spike-p3-m0-strip" +_HERE = Path(__file__).parent +_FIXTURE_DIR = _HERE.parent.parent / "spike_p3_m0_fixtures" + +# 与 Hermes_AI `tests/test_p3_m0_fixture_gate.py::EXPECTED_SHA256` 完全一致 +# **规范 LF 字节** SHA-256(Git normalized · 见 fork .gitattributes 声明) +EXPECTED_FIXTURE_SHA256 = ( + "47155c24dfd58019d1c7abdf3870e30099da29f56fe9535d4054d16b50168666" +) + + +def _canonical_lf_bytes(path: Path) -> bytes: + """Read file bytes with CRLF/CR line endings normalized to LF. + + Cross-repo SHA gate contract locks the **canonical LF bytes** (not disk + bytes) so Windows autocrlf checkout does not produce a false SHA mismatch. + Paired with fork `.gitattributes text eol=lf` rule for this file. + """ + raw = path.read_bytes() + return raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +# ── Fixture SHA-256 校验(跨仓一致性 门槛 · 规范 LF 字节) ───── + + +def test_fixture_sha256_matches_hermes_ai(): + fixture_path = _FIXTURE_DIR / "citation_cases.py" + actual = hashlib.sha256(_canonical_lf_bytes(fixture_path)).hexdigest() + assert actual == EXPECTED_FIXTURE_SHA256, ( + f"fixture canonical LF SHA-256 mismatch:\n expected: {EXPECTED_FIXTURE_SHA256}\n " + f"actual: {actual}\n跨仓 fixture 不一致 · 需双仓同步更新" + ) + + +def test_fixture_sha256_crlf_input_normalizes(): + """CRLF checkout(Windows autocrlf 模拟)仍匹配 SHA gate · 契约锁 LF 字节。""" + fixture_path = _FIXTURE_DIR / "citation_cases.py" + lf_bytes = fixture_path.read_bytes() + if b"\r\n" in lf_bytes: + pytest.skip("checkout 本就是 CRLF · 直接由 test_fixture_sha256_matches_hermes_ai 覆盖") + crlf_bytes = lf_bytes.replace(b"\n", b"\r\n") + normalized = crlf_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + assert hashlib.sha256(normalized).hexdigest() == EXPECTED_FIXTURE_SHA256, ( + "CRLF → LF 规范化后 SHA 与冻结值不符" + ) + + +# ── Discovery / registration(§VIII) ──────────────────────────── + + +def _enable_strip(tmp_path, monkeypatch): + write_config(tmp_path, [STRIP_KEY]) + manager = install_fresh_manager(monkeypatch, tmp_path) + mod = get_discovered_module(manager, STRIP_KEY) + return manager, mod + + +def test_discover_registers_transform_llm_output_hook(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + manager, mod = _enable_strip(tmp_path, monkeypatch) + assert has_hook("transform_llm_output") is True + + +def test_pluginmgr_helper_cleanup_leaves_no_sys_modules_residue(tmp_path): + """Codex 第三轮 §VI:证明 fixture teardown 后 sys.modules 完全恢复(names + object ids)。 + + 比 name-only 快照更强的契约:每个 hermes_plugins.* 键的 module OBJECT 在 teardown + 后必须还是**同一实例**(delitem+monkeypatch.undo 恢复原对象)。若 discover 新建的 + module 对象泄漏 · id() 会不同 · 直接暴露污染。 + """ + import sys + import pytest as _pytest + + before_names = snapshot_plugin_modules() + before_ids = {k: id(sys.modules[k]) for k in before_names} + + mp = _pytest.MonkeyPatch() + finalizers = [] + + class _FakeRequest: + def addfinalizer(self, fn): + finalizers.append(fn) + + try: + write_config(tmp_path, [STRIP_KEY]) + install_fresh_manager(mp, tmp_path, request=_FakeRequest()) + # 验证 discover 确实加载了(要么新键 · 要么替换了 object id) + during_names = snapshot_plugin_modules() + during_ids = {k: id(sys.modules[k]) for k in during_names} + discover_loaded = (during_names - before_names) or { + k for k in before_names & during_names if during_ids.get(k) != before_ids.get(k) + } + assert discover_loaded, "install_fresh_manager 未加载 · 也未替换任何 hermes_plugins.*" + finally: + # 手工 teardown:先 finalizers · 再 monkeypatch undo + for fn in reversed(finalizers): + fn() + mp.undo() + + # 快照完全恢复:names 与 object ids 都必须与 before 一致 + after_names = snapshot_plugin_modules() + assert after_names == before_names, ( + f"name leak · new keys: {sorted(after_names - before_names)} · " + f"missing keys: {sorted(before_names - after_names)}" + ) + for k in before_names: + assert id(sys.modules[k]) == before_ids[k], ( + f"hermes_plugins module {k!r} object identity changed after teardown · " + "discover-inserted new object leaked past finalizer + monkeypatch.undo" + ) + + +def test_disabled_plugin_not_loaded(tmp_path, monkeypatch): + from hermes_cli.plugins import has_hook + + write_config(tmp_path, []) + install_fresh_manager(monkeypatch, tmp_path) + assert has_hook("transform_llm_output") is False + + +# ── final-mode sanitize_presentation 全 fixture parametrize ──── + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_sanitize_final_mode(case: StripCase): + """全 final mode fixture case · sanitize_presentation(mode='final') 正确。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + got = sanitize_presentation(case.raw, mode="final") + assert got == case.expected, ( + f"case={case.name}\n raw: {case.raw!r}\n " + f"expected: {case.expected!r}\n got: {got!r}\n notes: {case.notes}" + ) + + +# ── Negative case:bare [N] / JSON / footnote 必须原样保留(§VI 硬约束) ── + + +@pytest.mark.parametrize("case", cases_by_marker("negative"), ids=lambda c: c.name) +def test_negative_cases_preserved(case: StripCase): + """negative case:sanitize 不得吞掉 · 原样返回。""" + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + got = sanitize_presentation(case.raw, mode="final") + assert got == case.raw, ( + f"NEGATIVE case={case.name} 被误 strip · raw={case.raw!r} got={got!r}" + ) + + +# ── Hook 契约(通过 invoke_hook · 真实 PluginManager 走位) ──────── + + +@pytest.mark.parametrize("case", cases_by_mode("final"), ids=lambda c: c.name) +def test_transform_llm_output_hook_via_invoke_hook(tmp_path, monkeypatch, case: StripCase): + from hermes_cli.plugins import invoke_hook + + _enable_strip(tmp_path, monkeypatch) + results = invoke_hook( + "transform_llm_output", + response_text=case.raw, + session_id="s_test", + model="m", + platform="cli", + ) + if case.raw == case.expected: + assert results == [], f"unchanged input · hook 应返回 None(过滤后空 list)· case={case.name}" + else: + assert results == [case.expected], ( + f"case={case.name} · expected [{case.expected!r}] · got {results!r}" + ) + + +def test_hook_empty_input_returns_none(tmp_path, monkeypatch): + from hermes_cli.plugins import invoke_hook + + _enable_strip(tmp_path, monkeypatch) + assert invoke_hook("transform_llm_output", response_text="") == [] + + +# ── fail-CLOSED on sanitizer exception(§VI 硬约束) ───────────── + + +def test_hook_fail_closed_on_sanitizer_exception(tmp_path, monkeypatch, caplog): + """sanitizer 崩 · hook 返回 FAIL_CLOSED_PLACEHOLDER · **不**返回原文。 + + Codex §VI:sanitizer 失败必须拒原文透出 · 返回安全占位符 · 记录 audit。 + """ + from hermes_cli.plugins import invoke_hook + + _, strip_mod = _enable_strip(tmp_path, monkeypatch) + from plugins.spike_p3_m0_fixtures.pluginmgr_helper import get_discovered_module + # 从 discover 后的 module 修改 sanitize 依赖(fresh manager 下的 module) + sanitize_dep = strip_mod._sanitize_mod + + internal_secret = "TOP_SECRET_SANITIZER_PAYLOAD" + + def boom(*_args, **_kwargs): + raise RuntimeError(internal_secret) + + monkeypatch.setattr(sanitize_dep, "sanitize_presentation", boom) + + dangerous_text = "SECRET INTERNAL DATA · SHOULD NOT LEAK · [source: internal]" + results = invoke_hook( + "transform_llm_output", + response_text=dangerous_text, + session_id="s_test", + model="m", + platform="cli", + ) + # 关键断言:不返回原文 · 返回占位符 + assert len(results) == 1 + assert results[0] == strip_mod.FAIL_CLOSED_PLACEHOLDER + assert dangerous_text not in results[0], "fail-closed 泄漏原文 · 违 §VI 硬约束" + assert internal_secret not in caplog.text + + +# ── 其他 mode 契约冻结(skeleton pass-through · M1+ 实装) ────── + + +@pytest.mark.parametrize("mode", ["history", "streaming", "interim", "tool"]) +def test_non_final_modes_skeleton_pass_through(mode: str): + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + text = "Answer with citation [source: kb_2024]. More text." + got = sanitize_presentation(text, mode=mode) + assert got == text, f"mode={mode} 应 pass-through(M0 skeleton · 见 §VI 冻结)" + + +# ── Streaming byte-split · S1 完整缓冲后 sanitize 与整体一致 ─── + + +@pytest.mark.parametrize("case", cases_by_marker("streaming_split"), ids=lambda c: c.name) +def test_streaming_full_buffer_matches_final(case: StripCase): + """S1 策略验证:任意字节切分点分片 → 缓冲拼回 → UTF-8 decode → final sanitize + → 结果必须等于对整体 raw 直接 final sanitize 的结果(与 case.expected 一致)。 + """ + from plugins.spike_p3_m0_strip.sanitize import sanitize_presentation + + expected = sanitize_presentation(case.raw, mode="final") + assert expected == case.expected, ( + f"streaming_split case={case.name} · 直接 final sanitize 与 fixture expected 不一致 · " + f"expected={case.expected!r} got={expected!r}" + ) + # 每个切分点:两段拼接 · decode · sanitize · 与 expected 一致 + for head, tail in enumerate_byte_split_chunks(case.raw): + buf = head + tail + text = buf.decode("utf-8") + got = sanitize_presentation(text, mode="final") + assert got == expected, ( + f"streaming byte-split 破坏 sanitize · case={case.name} " + f"head_len={len(head)} · got={got!r} expected={expected!r}" + )