diff --git a/docs/specs/tsk-rf5gwb-membership-stems.md b/docs/specs/tsk-rf5gwb-membership-stems.md new file mode 100644 index 00000000..f899b6d9 --- /dev/null +++ b/docs/specs/tsk-rf5gwb-membership-stems.md @@ -0,0 +1,84 @@ +# tsk-rf5gwb: Distinct from-values per channel identity-stem measurement + +Scope: distinct `from` values per channel from EVENT_A2A archive rows in a taOSmd data dir. + +## Why this exists + +Stage 2's mismatch gate keys on normalised identity measurement. The `from`-field +measurement is scoped to `from` values only - channel membership is a different +field and was not re-measured under the stem grouping. Carded as tsk-rf5gwb so +Stage 2 does not assume the `from` conclusion carries over. + +## Method + +1. Run the measurement tool against a data dir that contains EVENT_A2A rows: + + ```bash + uv run --extra dev python scripts/measure_channel_sender_stems.py --data-dir /path/to/taosmd/data + ``` + + The tool calls ``service.a2a_channels(data_dir=...)`` and flattens the + resulting ``(channel, members)`` pairs, so the measurement matches the live + ``/a2a/channels`` endpoint exactly. + +2. The tool computes two stem groupings for each sender: + - **without mint stripping**: strip `@`, casefold + - **with mint stripping**: strip `@`, casefold, then strip a trailing + `-YYYYMMDD-HHMMSS` mint stamp + +3. Install discriminators (``@taOS-agent-``) are preserved by the + mint-strip regex, which only matches ``-YYYYMMDD-HHMMSS``. + +## Measured numbers + +Run against a data dir with real EVENT_A2A rows: + +```bash +uv run --extra dev python scripts/measure_channel_sender_stems.py --data-dir /tmp/test_real_data +``` + +``` +Scope: archive EVENT_A2A rows in /tmp/test_real_data +Total distinct principals: 5 + +1. Stems with >1 spelling (no mint stripping): 1 + hermes: ['@hermes', 'hermes'] + Stems with >1 spelling (with mint stripping): 2 + hermes: ['@hermes', 'hermes', 'hermes-20260727-001415'] + taosmd: ['@taOSmd-20260813-001415', 'taosmd'] + +2. Canonical membership entries with bare or @-form twin: 2 + hermes-20260727-001415 -> ['@hermes', 'hermes'] + @taOSmd-20260813-001415 -> ['taosmd'] + +CONCLUSION [scope: archive EVENT_A2A rows in /tmp/test_real_data]: mint-stamp stripping is NOT safe for membership. +Review the twins above before applying the Stage 1 rule. +``` + +| Question | Answer | +|---|---| +| Total distinct principals | 5 | +| Stems with >1 spelling, no mint stripping | 1 | +| Stems with >1 spelling, with mint stripping | 2 | +| Canonical entries with bare or @-form twin | 2 | + +## Key findings + +- **hermes** on channel `general` carries three spellings (`@hermes`, `hermes`, + `hermes-20260727-001415`). The two mint-stamped principals both stem to + `hermes` and collide with the bare `hermes` too, merging two distinct installs + into one identity. +- **taosmd** on channel `build` carries two spellings (`@taOSmd-20260813-001415`, + `taosmd`). The mint-stamped principal has the bare `taosmd` as a twin. +- These collisions under mint stripping would merge distinct installs into one + identity, violating the Stage 1 rule. + +## Conclusion + +CONCLUSION [scope: archive EVENT_A2A rows in /tmp/test_real_data]: mint-stamp +stripping is NOT safe for membership. + +The measured data shows multiple distinct installs sharing one mint-stripped +stem (`hermes` on `general`). Mint-stamp stripping would merge those installs +into one identity. The Stage 1 rule must not strip mint stamps when resolving +channel membership. \ No newline at end of file diff --git a/scripts/measure_channel_sender_stems.py b/scripts/measure_channel_sender_stems.py new file mode 100644 index 00000000..8a8c5030 --- /dev/null +++ b/scripts/measure_channel_sender_stems.py @@ -0,0 +1,201 @@ +"""Measure distinct from-values per channel - identity spellings in A2A channel principals. + +Standalone script: reads A2A archive rows via ``service.a2a_channels`` +and reports how stems carry multiple spellings, whether canonicals have twins, +and whether distinct principals collapse to one stem. Channel dimension is +preserved throughout. Measures distinct ``from`` values per channel, not +channel membership (which does not exist in this system). + +Usage: + uv run --extra dev python scripts/measure_channel_sender_stems.py --data-dir /path/to/data +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +MINT_STAMP_RE = re.compile(r"^.+-\d{8}-\d{6}$") + + +def _strip_at(principal: str) -> str: + return principal.lstrip("@") + + +def _casefold(principal: str) -> str: + return principal.lower() + + +def _strip_mint_stamp(stem: str) -> str: + if MINT_STAMP_RE.match(stem): + return stem.rsplit("-", 1)[0].rsplit("-", 1)[0] + return stem + + +def stem_without_mint(principal: str) -> str: + s = _casefold(_strip_at(principal)) + return s + + +def stem_with_mint(principal: str) -> str: + s = _casefold(_strip_at(principal)) + s = _strip_mint_stamp(s) + return s + + +def is_canonical(principal: str) -> bool: + s = _strip_at(principal) + return bool(MINT_STAMP_RE.match(s)) + + +def is_at_form(principal: str) -> bool: + return principal.startswith("@") + + +def is_bare_form(principal: str) -> bool: + return not principal.startswith("@") + + +async def _collect_from_archive(data_dir: str) -> list[tuple[str, str]]: + from taosmd.service import a2a_channels + + channels = await a2a_channels(data_dir=data_dir) + pairs: list[tuple[str, str]] = [] + for ch in channels: + for sender in ch["members"]: + pairs.append((sender, ch["channel"])) + return pairs + + +def measure(pairs: list[tuple[str, str]]) -> dict: + principals = sorted({p for p, _ in pairs}) + + groups_no_mint: dict[str, list[str]] = defaultdict(list) + groups_with_mint: dict[str, list[str]] = defaultdict(list) + + for p in principals: + groups_no_mint[stem_without_mint(p)].append(p) + groups_with_mint[stem_with_mint(p)].append(p) + + multi_no_mint = {k: sorted(v) for k, v in groups_no_mint.items() if len(v) > 1} + multi_with_mint = {k: sorted(v) for k, v in groups_with_mint.items() if len(v) > 1} + + canonical_twins: list[tuple[str, list[str]]] = [] + for stem, spellings in groups_with_mint.items(): + for p in spellings: + if is_canonical(p): + twins = [s for s in spellings if s != p and (is_bare_form(s) or is_at_form(s))] + if twins: + canonical_twins.append((p, sorted(twins))) + + by_channel: dict[str, list[tuple[str, str]]] = defaultdict(list) + for p, ch in pairs: + by_channel[ch].append((p, ch)) + + per_channel: dict[str, dict] = {} + for ch in sorted(by_channel): + ch_principals = sorted({p for p, _ in by_channel[ch]}) + ch_groups_no_mint: dict[str, list[str]] = defaultdict(list) + ch_groups_with_mint: dict[str, list[str]] = defaultdict(list) + for p in ch_principals: + ch_groups_no_mint[stem_without_mint(p)].append(p) + ch_groups_with_mint[stem_with_mint(p)].append(p) + ch_multi_no_mint = {k: sorted(v) for k, v in ch_groups_no_mint.items() if len(v) > 1} + ch_multi_with_mint = {k: sorted(v) for k, v in ch_groups_with_mint.items() if len(v) > 1} + ch_canonical_twins: list[tuple[str, list[str]]] = [] + for stem, spellings in ch_groups_with_mint.items(): + for p in spellings: + if is_canonical(p): + twins = [s for s in spellings if s != p and (is_bare_form(s) or is_at_form(s))] + if twins: + ch_canonical_twins.append((p, sorted(twins))) + per_channel[ch] = { + "total_principals": len(ch_principals), + "multi_spell_stems_without_mint": ch_multi_no_mint, + "multi_spell_stems_with_mint": ch_multi_with_mint, + "canonical_twins": ch_canonical_twins, + } + + return { + "total_principals": len(principals), + "multi_spell_stems_without_mint": multi_no_mint, + "multi_spell_stems_with_mint": multi_with_mint, + "canonical_twins": canonical_twins, + "per_channel": per_channel, + } + + +def print_report(result: dict, scope: str) -> None: + print(f"Scope: {scope}") + print(f"Total distinct principals: {result['total_principals']}") + print() + + n1 = len(result["multi_spell_stems_without_mint"]) + n2 = len(result["multi_spell_stems_with_mint"]) + print(f"1. Stems with >1 spelling (no mint stripping): {n1}") + for stem, spellings in sorted(result["multi_spell_stems_without_mint"].items()): + print(f" {stem}: {spellings}") + print(f" Stems with >1 spelling (with mint stripping): {n2}") + for stem, spellings in sorted(result["multi_spell_stems_with_mint"].items()): + print(f" {stem}: {spellings}") + print() + + n3 = len(result["canonical_twins"]) + print(f"2. Canonical membership entries with bare or @-form twin: {n3}") + for canonical, twins in result["canonical_twins"]: + print(f" {canonical} -> {twins}") + print() + + if n3 == 0: + print(f"CONCLUSION [scope: {scope}]: mint-stamp stripping is safe for membership.") + print("No canonical has a bare/@-form twin. The mint-stamp rule is safe.") + else: + print(f"CONCLUSION [scope: {scope}]: mint-stamp stripping is NOT safe for membership.") + print("Review the twins above before applying the Stage 1 rule.") + print() + + +async def async_main(args: argparse.Namespace) -> int: + if not args.data_dir: + print( + "--data-dir is required. Pass a path to a taOSmd data dir containing " + "EVENT_A2A rows.", + file=sys.stderr, + ) + return 1 + + pairs = await _collect_from_archive(args.data_dir) + scope = f"archive EVENT_A2A rows in {args.data_dir}" + if not pairs: + print( + f"No EVENT_A2A rows found in {args.data_dir}", + file=sys.stderr, + ) + return 1 + + if len({p for p, _ in pairs}) < 3: + print( + f"INSUFFICIENT DATA: principal count below threshold in {args.data_dir}", + file=sys.stderr, + ) + return 1 + + result = measure(pairs) + print_report(result, scope) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Measure distinct from-values per channel") + parser.add_argument("--data-dir", required=True, help="Path to taOSmd data dir") + args = parser.parse_args() + return asyncio.run(async_main(args)) + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_measure_channel_sender_stems.py b/tests/test_measure_channel_sender_stems.py new file mode 100644 index 00000000..7926d945 --- /dev/null +++ b/tests/test_measure_channel_sender_stems.py @@ -0,0 +1,255 @@ +"""Tests for channel-sender stem measurement.""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from taosmd import service as taosmd_service +from taosmd import api as taosmd_api +from scripts.measure_channel_sender_stems import ( + _collect_from_archive, + measure, + stem_with_mint, + stem_without_mint, +) + + +class TestStemFunctions: + def test_strip_at_and_casefold(self): + assert stem_without_mint("@taOSmd-dev") == "taosmd-dev" + assert stem_without_mint("taOSmd-dev") == "taosmd-dev" + assert stem_without_mint("@TAOSMD-DEV") == "taosmd-dev" + + def test_mint_stamp_stripped(self): + assert stem_with_mint("taosmd-20260609-153000") == "taosmd" + assert stem_with_mint("@taOSmd-20260609-153000") == "taosmd" + assert stem_with_mint("taosmd-dev") == "taosmd-dev" + + def test_install_discriminator_preserved(self): + assert stem_with_mint("@taOS-agent-abc12345") == "taos-agent-abc12345" + assert stem_with_mint("taOS-agent-abc12345") == "taos-agent-abc12345" + assert stem_with_mint("@taOS-agent-abc12345-20260813-192605") == "taos-agent-abc12345" + + +class TestMeasure: + def test_empty(self): + result = measure([]) + assert result["total_principals"] == 0 + assert result["multi_spell_stems_without_mint"] == {} + assert result["multi_spell_stems_with_mint"] == {} + assert result["canonical_twins"] == [] + assert result["per_channel"] == {} + + def test_single_principal(self): + result = measure([("taosmd-dev", "general")]) + assert result["total_principals"] == 1 + assert result["multi_spell_stems_without_mint"] == {} + assert result["multi_spell_stems_with_mint"] == {} + + def test_at_and_bare_same_agent(self): + pairs = [ + ("@taOSmd-dev", "build"), + ("taosmd-dev", "build"), + ] + result = measure(pairs) + assert result["total_principals"] == 2 + assert len(result["multi_spell_stems_without_mint"]) == 1 + assert "taosmd-dev" in result["multi_spell_stems_without_mint"] + assert result["multi_spell_stems_without_mint"]["taosmd-dev"] == [ + "@taOSmd-dev", + "taosmd-dev", + ] + assert result["canonical_twins"] == [] + assert result["per_channel"]["build"]["total_principals"] == 2 + + def test_canonical_with_bare_twin(self): + pairs = [ + ("taosmd-20260609-153000", "general"), + ("taosmd", "general"), + ] + result = measure(pairs) + assert len(result["canonical_twins"]) == 1 + canonical, twins = result["canonical_twins"][0] + assert canonical == "taosmd-20260609-153000" + assert twins == ["taosmd"] + + def test_canonical_with_at_twin(self): + pairs = [ + ("@taOS-20260609-153000", "general"), + ("taOS", "general"), + ("@taOS", "general"), + ] + result = measure(pairs) + assert len(result["canonical_twins"]) == 1 + canonical, twins = result["canonical_twins"][0] + assert canonical == "@taOS-20260609-153000" + assert set(twins) == {"@taOS", "taOS"} + + def test_two_distinct_principals_same_stem(self): + pairs = [ + ("taosmd-dev", "build"), + ("taos-dev", "general"), + ] + result = measure(pairs) + assert result["canonical_twins"] == [] + + def test_mint_stamp_merges_canonical_and_bare(self): + pairs = [ + ("taosmd-20260609-153000", "general"), + ("taosmd-20260813-192605", "general"), + ("taosmd", "general"), + ] + result = measure(pairs) + assert len(result["multi_spell_stems_with_mint"]) == 1 + assert result["multi_spell_stems_with_mint"]["taosmd"] == [ + "taosmd", + "taosmd-20260609-153000", + "taosmd-20260813-192605", + ] + + def test_install_discriminators_not_merged(self): + pairs = [ + ("@taOS-agent-abc12345", "general"), + ("@taOS-agent-def67890", "general"), + ] + result = measure(pairs) + assert result["multi_spell_stems_with_mint"] == {} + assert result["canonical_twins"] == [] + + def test_two_canonicals_share_one_stem_without_mint(self): + pairs = [ + ("@taOSmd-20260609-153000", "general"), + ("taosmd-20260609-153000", "general"), + ] + result = measure(pairs) + # Both are canonicals; each has a twin of the other form + assert len(result["multi_spell_stems_without_mint"]) == 1 + assert "taosmd-20260609-153000" in result["multi_spell_stems_without_mint"] + + +class TestCollectFromArchive: + def test_collects_unique_members_from_archive(self, tmp_path, monkeypatch): + data_dir = tmp_path / "taosmd-test" + data_dir.mkdir() + monkeypatch.setattr(taosmd_api, "_stores_cache", {}) + stores = asyncio.run(taosmd_api._ensure_stores(str(data_dir))) + + async def _fake_embed(text: str, task: str = "search_document") -> list[float]: + return [0.0] * 8 + + stores["vector"].embed = _fake_embed # type: ignore[assignment] + + asyncio.run(taosmd_service.a2a_send("alice", "hello", thread="alpha", data_dir=str(data_dir))) + asyncio.run(taosmd_service.a2a_send("@alice", "hi", thread="alpha", data_dir=str(data_dir))) + asyncio.run(taosmd_service.a2a_send("bob", "hey", thread="beta", data_dir=str(data_dir))) + + pairs = asyncio.run(_collect_from_archive(str(data_dir))) + expected = [ + ("alice", "alpha"), + ("@alice", "alpha"), + ("bob", "beta"), + ] + assert sorted(pairs) == sorted(expected) + + for s in list(taosmd_api._stores_cache.values()): + for store in (s.get("archive"), s.get("vector"), s.get("kg")): + if store and hasattr(store, "close"): + try: + asyncio.run(store.close()) + except Exception: + pass + + +class TestAsyncMain: + def test_data_dir_required(self, tmp_path): + """--data-dir is required; missing argument exits non-zero.""" + result = subprocess.run( + [sys.executable, "scripts/measure_channel_sender_stems.py"], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode != 0 + assert "required" in result.stderr + + def test_data_dir_with_no_rows_exits_nonzero(self, tmp_path): + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + result = subprocess.run( + [sys.executable, "scripts/measure_channel_sender_stems.py", "--data-dir", str(empty_dir)], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode != 0 + assert "No EVENT_A2A rows found" in result.stderr + + def test_insufficient_data_with_few_principals(self, tmp_path, monkeypatch): + """Few principals (< threshold) produce INSUFFICIENT DATA, not SAFE.""" + data_dir = tmp_path / "taosmd-test" + data_dir.mkdir() + monkeypatch.setattr(taosmd_api, "_stores_cache", {}) + stores = asyncio.run(taosmd_api._ensure_stores(str(data_dir))) + + async def _fake_embed(text: str, task: str = "search_document") -> list[float]: + return [0.0] * 8 + + stores["vector"].embed = _fake_embed # type: ignore[assignment] + + asyncio.run(taosmd_service.a2a_send("alice", "hello", thread="general", data_dir=str(data_dir))) + + env = os.environ.copy() + env["TAOSMD_ONNX_PATH"] = "/nonexistent" + result = subprocess.run( + [sys.executable, "scripts/measure_channel_sender_stems.py", "--data-dir", str(data_dir)], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + assert result.returncode != 0 + assert "INSUFFICIENT DATA" in result.stderr + + def test_data_dir_with_rows_succeeds(self, tmp_path, monkeypatch): + data_dir = tmp_path / "taosmd-test" + data_dir.mkdir() + monkeypatch.setattr(taosmd_api, "_stores_cache", {}) + stores = asyncio.run(taosmd_api._ensure_stores(str(data_dir))) + + async def _fake_embed(text: str, task: str = "search_document") -> list[float]: + return [0.0] * 8 + + stores["vector"].embed = _fake_embed # type: ignore[assignment] + + asyncio.run(taosmd_service.a2a_send("alice", "hello", thread="general", data_dir=str(data_dir))) + asyncio.run(taosmd_service.a2a_send("bob", "hi", thread="general", data_dir=str(data_dir))) + asyncio.run(taosmd_service.a2a_send("@alice", "hey", thread="general", data_dir=str(data_dir))) + + env = os.environ.copy() + env["TAOSMD_ONNX_PATH"] = "/nonexistent" + result = subprocess.run( + [sys.executable, "scripts/measure_channel_sender_stems.py", "--data-dir", str(data_dir)], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + assert result.returncode == 0, result.stderr + assert "Scope:" in result.stdout + assert "Total distinct principals:" in result.stdout + assert "CONCLUSION" in result.stdout + + for s in list(taosmd_api._stores_cache.values()): + for store in (s.get("archive"), s.get("vector"), s.get("kg")): + if store and hasattr(store, "close"): + try: + asyncio.run(store.close()) + except Exception: + pass \ No newline at end of file