From 1c973533ba63bb1a7b5fc500642bcbdbf8b87640 Mon Sep 17 00:00:00 2001 From: devagentic-dev Date: Fri, 22 May 2026 06:51:55 +0000 Subject: [PATCH] feat(memory): graph-memory adapter + migration script (Phase D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hermes side of the two-repo Phase D lift (TechDevGroup/devagentic#54). agent/devagentic_memory.py is the read-through adapter: when DEVAGENTIC_MEMORY_GRAPH=1, it POSTs userFactQuery to a running devagentic and returns the relevance-ranked facts; on any failure (gate off, network error, parse error, no user_id) returns [] so the caller's existing file-based memory fallback runs unchanged. scripts/migrate_memory_to_graph.py is the one-way migration — walks HERMES_HOME for MEMORY.md / USER.md / SOUL.md and POSTs each file as a kind:user-fact node via userFactCreate. --dry-run prints the plan; v0 granularity is one fact per file with tags=[origin:, migration:YYYY-MM-DD, ...]. Companion devagentic PR: TechDevGroup/devagentic# Co-Authored-By: Claude Opus 4.7 (1M context) --- agent/devagentic_memory.py | 204 +++++++++++++++++++++++++++ scripts/migrate_memory_to_graph.py | 174 +++++++++++++++++++++++ tests/test_devagentic_memory.py | 213 +++++++++++++++++++++++++++++ 3 files changed, 591 insertions(+) create mode 100644 agent/devagentic_memory.py create mode 100755 scripts/migrate_memory_to_graph.py create mode 100644 tests/test_devagentic_memory.py diff --git a/agent/devagentic_memory.py b/agent/devagentic_memory.py new file mode 100644 index 000000000000..81823c1165e5 --- /dev/null +++ b/agent/devagentic_memory.py @@ -0,0 +1,204 @@ +"""Phase D devagentic-graph memory adapter (devagentic issue #54). + +Talks to a running devagentic over HTTP to resolve user-facts from +graph nodes (kind:user-fact) before hermes falls back to its +~/.hermes/MEMORY.md / USER.md / SOUL.md files. + +Opt-in via env `DEVAGENTIC_MEMORY_GRAPH=1`. Default off keeps the +file-based flow byte-stable. The migration script +`scripts/migrate_memory_to_graph.py` populates the graph from +existing files; once enabled, `query_user_facts(query_text)` +returns relevance-ranked facts the caller can splice into a +system prompt or memory rollup. + +Env vars: + DEVAGENTIC_MEMORY_GRAPH set to `1` to enable graph reads + (default: off). When off, + query_user_facts always returns + []; callers fall straight to files. + DEVAGENTIC_BASE_URL devagentic base URL (default + http://127.0.0.1:6071/v1). Reused + from the devagentic-local provider + so a single configuration covers + skills, memory, and completions. + DEVAGENTIC_API_KEY bearer token forwarded to devagentic. + With DEVAGENTIC_TRUST_HEADER=1 on + devagentic, any non-empty value works. + +Failure semantics: every code path that touches the network is +wrapped in try/except. Resolver returns an empty list on any +failure (network, parse, no facts, etc.); callers MUST keep their +existing file fallback so a transient devagentic outage doesn't +brick memory retrieval. +""" +from __future__ import annotations + +import json +import logging +import os +import urllib.error +import urllib.request +from typing import Any, Optional + + +logger = logging.getLogger(__name__) + + +GRAPH_ENV = "DEVAGENTIC_MEMORY_GRAPH" + +# Network timeout. Generous on a local loopback; faster than that +# and a slow devagentic startup would race the caller's file +# fallback. +_DEFAULT_TIMEOUT = 8.0 + + +def graph_enabled() -> bool: + """True iff `DEVAGENTIC_MEMORY_GRAPH` is in `1|true|yes|on`.""" + return os.environ.get(GRAPH_ENV, "0").strip().lower() in ( + "1", "true", "yes", "on") + + +def _base_url() -> str: + raw = os.environ.get("DEVAGENTIC_BASE_URL", "http://127.0.0.1:6071/v1") + base = raw.rstrip("/") + if base.endswith("/v1"): + base = base[:-3] + return base + + +def _api_key() -> str: + return (os.environ.get("DEVAGENTIC_API_KEY") or "").strip() + + +def _user_id() -> Optional[str]: + """Resolve the X-User-Id to send. + + Same precedence as the devagentic-local provider plugin + (Phase G #50) and the skill adapter (Phase C #52): + 1. `DEVAGENTIC_USER_ID` env override. + 2. `hermes_cli.profiles.get_active_profile_name()`. + 3. None — caller doesn't inject the header; adapter returns + empty. + """ + override = (os.environ.get("DEVAGENTIC_USER_ID") or "").strip() + if override: + return override + try: + from hermes_cli.profiles import get_active_profile_name + name = (get_active_profile_name() or "").strip() + return name or None + except Exception as exc: # noqa: BLE001 + logger.debug("devagentic_memory: profile resolution failed: %s", exc) + return None + + +def _post_graphql(query: str, variables: dict, + *, timeout: float = _DEFAULT_TIMEOUT) -> Optional[dict]: + """POST a GraphQL query to devagentic. Returns parsed `data` + on success, None on any failure (network, non-200, parse error, + GraphQL error). Failures log at DEBUG only.""" + base = _base_url() + user = _user_id() + if not user: + logger.debug("devagentic_memory: no user_id resolved; skipping") + return None + body = json.dumps({"query": query, "variables": variables}).encode("utf-8") + req = urllib.request.Request( + f"{base}/graphql", data=body, method="POST") + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + req.add_header("X-User-Id", user) + api_key = _api_key() + if api_key: + req.add_header("Authorization", f"Bearer {api_key}") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8") + except (urllib.error.URLError, OSError, TimeoutError) as exc: + logger.debug("devagentic_memory: request failed: %s", exc) + return None + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + logger.debug("devagentic_memory: parse failed: %s", exc) + return None + if not isinstance(payload, dict): + return None + if payload.get("errors"): + logger.debug("devagentic_memory: graphql errors: %s", + payload.get("errors")) + return None + return payload.get("data") or None + + +def query_user_facts( + query: str, + *, + top_k: int = 10, + timeout: float = _DEFAULT_TIMEOUT, +) -> list[dict]: + """Relevance-rank user-facts from the devagentic graph. + + Returns a list of dicts shaped `{id, body, tags, source, + confidence}` — keys mirror the GraphQL UserFact type. Empty + list on: + * gate off (`DEVAGENTIC_MEMORY_GRAPH` unset / `0`) + * empty query + * no user_id resolvable + * network / parse error + * no matching facts + + Callers MUST keep their existing file fallback; this is a + passive read-through, not a hard dependency.""" + if not graph_enabled(): + return [] + if not query: + return [] + gql = ( + "query($u:String!,$q:String!,$k:Int)" + "{userFactQuery(userId:$u,query:$q,topK:$k)" + "{id body tags source confidence}}" + ) + user = _user_id() + if not user: + return [] + data = _post_graphql( + gql, {"u": user, "q": query, "k": top_k}, timeout=timeout) + if data is None: + return [] + facts = data.get("userFactQuery") or [] + if not isinstance(facts, list): + return [] + return [f for f in facts if isinstance(f, dict) and f.get("body")] + + +def create_user_fact( + body: str, + source: str, + *, + tags: Optional[list[str]] = None, + confidence: Optional[float] = None, + timeout: float = _DEFAULT_TIMEOUT, +) -> Optional[str]: + """Write a new `kind:user-fact` node to the devagentic graph. + Returns the new fact's head_id on success, None on any + failure. Used by `scripts/migrate_memory_to_graph.py`.""" + user = _user_id() + if not user: + return None + gql = ( + "mutation($u:String!,$b:String!,$s:String!,$t:[String!],$c:Float)" + "{userFactCreate(userId:$u,body:$b,source:$s,tags:$t,confidence:$c)" + "{id source}}" + ) + data = _post_graphql(gql, { + "u": user, "b": body, "s": source, + "t": list(tags) if tags else None, + "c": confidence, + }, timeout=timeout) + if data is None: + return None + fact = data.get("userFactCreate") + if not isinstance(fact, dict): + return None + return fact.get("id") diff --git a/scripts/migrate_memory_to_graph.py b/scripts/migrate_memory_to_graph.py new file mode 100755 index 000000000000..02ebf7766902 --- /dev/null +++ b/scripts/migrate_memory_to_graph.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Phase D one-way memory migration (devagentic issue #54). + +Reads `MEMORY.md`, `USER.md`, `SOUL.md` from `HERMES_HOME` (or a +custom dir via `--from`) and writes each one as a +`kind:user-fact` graph node in devagentic via `userFactCreate`. +After the migration, devagentic's `userFactQuery` returns facts +sourced from these files, and `agent.devagentic_memory` short- +circuits memory reads when `DEVAGENTIC_MEMORY_GRAPH=1`. + +Usage: + + python scripts/migrate_memory_to_graph.py # migrate + python scripts/migrate_memory_to_graph.py --dry-run # preview + python scripts/migrate_memory_to_graph.py --from /custom/path + +Granularity: one node per file (file-level migration v0). Each +file's full content becomes the `body` of one `kind:user-fact`; +tags identify the origin file (`origin:MEMORY.md`, +`origin:USER.md`, etc.) plus the migration date. Per-paragraph +or per-bullet granularity is documented as deferred — file-level +keeps the migration round-trippable and the supersede story +simple (refining one file == one supersede). + +Idempotency: append-only. Re-running creates fresh facts; the +old ones remain visible in the graph. To dedupe a re-migration, +manually `userFactSupersede(old_id, new_id)` for each pair, or +use `--supersede-prior` (flagged but not implemented in v0). + +Requires: + * A running devagentic at $DEVAGENTIC_BASE_URL (default + http://127.0.0.1:6071/v1). + * A bearer in $DEVAGENTIC_API_KEY (any value works when + devagentic runs in trust-header mode). + * X-User-Id resolution via the active hermes profile (or + $DEVAGENTIC_USER_ID override). +""" +from __future__ import annotations + +import argparse +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + + +# Files we attempt to migrate. Order matters only for the printed +# plan; the actual writes can happen in any order. +_MEMORY_FILES = ("MEMORY.md", "USER.md", "SOUL.md") + + +def _default_memory_dir() -> Path: + """Resolve the active hermes home dir. Prefers + `hermes_constants.get_hermes_home()` when importable; falls + back to `$HERMES_HOME` or `~/.hermes`.""" + try: + from hermes_constants import get_hermes_home + return get_hermes_home() + except Exception: + home = os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes") + return Path(home) + + +def _today_tag() -> str: + """`migration:YYYY-MM-DD` tag for traceability.""" + return f"migration:{datetime.now(tz=timezone.utc).strftime('%Y-%m-%d')}" + + +def _iter_memory_files(root: Path) -> list[Path]: + """Yield each MEMORY.md / USER.md / SOUL.md file in `root` that + exists and is non-empty.""" + out: list[Path] = [] + for fname in _MEMORY_FILES: + p = root / fname + if not p.is_file(): + continue + try: + if not p.read_text(encoding="utf-8").strip(): + continue + except OSError: + continue + out.append(p) + return out + + +def main() -> int: + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--from", dest="from_dir", type=Path, default=None, + help="hermes home dir to migrate from " + "(default: HERMES_HOME)") + p.add_argument("--dry-run", action="store_true", + help="print the migration plan without writing") + p.add_argument("--tag", action="append", default=[], + metavar="TAG", + help="extra tag to attach to every migrated fact; " + "repeatable") + p.add_argument("--confidence", type=float, default=0.9, + help="confidence value for migrated facts " + "(default: 0.9; user-facts from files are " + "high-trust but not perfect since they may " + "have aged)") + args = p.parse_args() + + root = args.from_dir or _default_memory_dir() + print(f"migration source: {root}") + files = _iter_memory_files(root) + if not files: + print(f"no MEMORY.md / USER.md / SOUL.md files found " + f"under {root}; nothing to do") + return 0 + print(f"discovered {len(files)} memory file(s): " + f"{[f.name for f in files]}") + + # Lazy import so --help works without the adapter on the path. + try: + from agent.devagentic_memory import create_user_fact + except Exception as exc: + print(f"ABORT: could not import agent.devagentic_memory: {exc}") + return 2 + + today = _today_tag() + extra_tags = list(args.tag) + + written: list[str] = [] + failures: list[tuple[str, str]] = [] + for path in files: + try: + body = path.read_text(encoding="utf-8") + except OSError as exc: + failures.append((str(path), f"read failed: {exc}")) + continue + if not body.strip(): + print(f" SKIP {path.name}: file empty") + continue + tags = [ + f"origin:{path.name}", + today, + "kind:hermes-memory-migration", + *extra_tags, + ] + source = f"hermes-migration:{path.name}" + if args.dry_run: + print(f" [DRY] would migrate {path.name} " + f"({len(body)} chars, tags={tags!r}, " + f"source={source!r})") + continue + fact_id = create_user_fact( + body=body, source=source, tags=tags, + confidence=args.confidence) + if fact_id is None: + failures.append((str(path), + "create_user_fact returned None " + "(check devagentic is reachable + " + "X-User-Id resolvable)")) + continue + print(f" wrote {path.name} → {fact_id}") + written.append(fact_id) + + print() + if args.dry_run: + print(f"dry-run complete; would have migrated " + f"{len(files)} fact(s)") + return 0 + print(f"migrated {len(written)} fact(s); " + f"{len(failures)} failure(s)") + for path, reason in failures: + print(f" FAIL {path}: {reason}") + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_devagentic_memory.py b/tests/test_devagentic_memory.py new file mode 100644 index 000000000000..2005aa8dd348 --- /dev/null +++ b/tests/test_devagentic_memory.py @@ -0,0 +1,213 @@ +"""Tests for the devagentic-graph memory adapter (devagentic #54). + +Phase D hermes-side. Verifies that: + + * `graph_enabled()` honors `DEVAGENTIC_MEMORY_GRAPH`. + * `query_user_facts()` returns [] when the gate is off, + regardless of fixture state. + * When enabled, returns the parsed `userFactQuery` array from + a mocked GraphQL response. + * Network failures, parse failures, empty query, GraphQL + errors, and missing user_id all return []. + * `create_user_fact()` posts the right GraphQL mutation and + returns the new fact id on success. + * User-id resolution uses `DEVAGENTIC_USER_ID` env override. + * Migration helpers `_iter_memory_files` + `_today_tag` work + as documented. +""" +from __future__ import annotations + +import importlib +import importlib.util +import re +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture +def adapter(): + """Import agent.devagentic_memory fresh per test so module-level + env reads inside helpers don't leak across cases.""" + import agent.devagentic_memory as mod + importlib.reload(mod) + return mod + + +@pytest.fixture +def migrate_script(): + """Import scripts/migrate_memory_to_graph.py by path so its + helpers are testable independent of the migration's main().""" + repo_root = Path(__file__).resolve().parents[1] + path = repo_root / "scripts" / "migrate_memory_to_graph.py" + spec = importlib.util.spec_from_file_location( + "migrate_memory_under_test", path) + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(mod) + return mod + + +# ─── Env gate ──────────────────────────────────────────────── + +def test_graph_enabled_truthy(adapter, monkeypatch): + for v in ("1", "true", "yes", "on", "TRUE"): + monkeypatch.setenv(adapter.GRAPH_ENV, v) + assert adapter.graph_enabled() is True, f"{v!r}" + + +def test_graph_enabled_falsy(adapter, monkeypatch): + for v in ("0", "false", "no", "off", ""): + monkeypatch.setenv(adapter.GRAPH_ENV, v) + assert adapter.graph_enabled() is False, f"{v!r}" + monkeypatch.delenv(adapter.GRAPH_ENV, raising=False) + assert adapter.graph_enabled() is False, "unset" + + +# ─── query_user_facts ──────────────────────────────────────── + +def test_query_user_facts_off_returns_empty(adapter, monkeypatch): + monkeypatch.delenv(adapter.GRAPH_ENV, raising=False) + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: pytest.fail( + "_post_graphql ran with gate off")) + assert adapter.query_user_facts("anything") == [] + + +def test_query_user_facts_returns_array(adapter, monkeypatch): + monkeypatch.setenv(adapter.GRAPH_ENV, "1") + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + fake = { + "userFactQuery": [ + {"id": "f1", "body": "alice prefers terse", + "tags": ["preference"], "source": "user-explicit", + "confidence": 1.0}, + {"id": "f2", "body": "alice's tz is UTC-5", + "tags": ["timezone"], "source": "hermes-curator", + "confidence": 0.9}, + ] + } + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: fake) + rows = adapter.query_user_facts("terse preference timezone", + top_k=5) + assert len(rows) == 2 + assert rows[0]["body"] == "alice prefers terse" + + +def test_query_user_facts_empty_query(adapter, monkeypatch): + monkeypatch.setenv(adapter.GRAPH_ENV, "1") + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: pytest.fail( + "_post_graphql ran on empty query")) + assert adapter.query_user_facts("") == [] + + +def test_query_user_facts_network_failure(adapter, monkeypatch): + monkeypatch.setenv(adapter.GRAPH_ENV, "1") + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: None) + assert adapter.query_user_facts("anything") == [] + + +def test_query_user_facts_filters_blank_bodies(adapter, monkeypatch): + monkeypatch.setenv(adapter.GRAPH_ENV, "1") + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + # Adapter should drop entries whose `body` is missing / empty. + fake = {"userFactQuery": [ + {"id": "good", "body": "valid body", "tags": [], + "source": "x", "confidence": 1.0}, + {"id": "blank", "body": "", "tags": []}, + {"id": "absent", "tags": []}, + ]} + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: fake) + rows = adapter.query_user_facts("x") + assert [r["id"] for r in rows] == ["good"] + + +def test_query_user_facts_no_user_id(adapter, monkeypatch): + monkeypatch.setenv(adapter.GRAPH_ENV, "1") + monkeypatch.delenv("DEVAGENTIC_USER_ID", raising=False) + fake_mod = type(sys)("hermes_cli.profiles") + fake_mod.get_active_profile_name = lambda: "" + monkeypatch.setitem(sys.modules, "hermes_cli.profiles", fake_mod) + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: pytest.fail( + "_post_graphql ran without user_id")) + assert adapter.query_user_facts("x") == [] + + +# ─── create_user_fact ──────────────────────────────────────── + +def test_create_user_fact_posts_and_returns_id(adapter, monkeypatch): + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + captured: dict = {} + + def _capture(query, variables, **kwargs): + captured["query"] = query + captured["variables"] = variables + return {"userFactCreate": {"id": "fact-xyz", + "source": "test-source"}} + monkeypatch.setattr(adapter, "_post_graphql", _capture) + fid = adapter.create_user_fact( + body="terse preferred", + source="user-explicit", + tags=["preference"], confidence=0.95) + assert fid == "fact-xyz" + assert "userFactCreate" in captured["query"] + v = captured["variables"] + assert v["u"] == "alice" + assert v["b"] == "terse preferred" + assert v["s"] == "user-explicit" + assert v["t"] == ["preference"] + assert v["c"] == 0.95 + + +def test_create_user_fact_no_user_id_returns_none( + adapter, monkeypatch): + monkeypatch.delenv("DEVAGENTIC_USER_ID", raising=False) + fake_mod = type(sys)("hermes_cli.profiles") + fake_mod.get_active_profile_name = lambda: "" + monkeypatch.setitem(sys.modules, "hermes_cli.profiles", fake_mod) + assert adapter.create_user_fact("x", "y") is None + + +def test_create_user_fact_failure_returns_none(adapter, monkeypatch): + monkeypatch.setenv("DEVAGENTIC_USER_ID", "alice") + monkeypatch.setattr(adapter, "_post_graphql", + lambda *a, **k: None) + assert adapter.create_user_fact("x", "y") is None + + +# ─── Migration helpers ────────────────────────────────────── + +def test_iter_memory_files(migrate_script, tmp_path): + (tmp_path / "MEMORY.md").write_text("memory content\n") + (tmp_path / "USER.md").write_text("user content\n") + (tmp_path / "SOUL.md").write_text("soul content\n") + files = migrate_script._iter_memory_files(tmp_path) + names = sorted(p.name for p in files) + assert names == ["MEMORY.md", "SOUL.md", "USER.md"] + + +def test_iter_memory_files_skips_empty(migrate_script, tmp_path): + (tmp_path / "MEMORY.md").write_text("memory content\n") + (tmp_path / "USER.md").write_text(" \n \n") # whitespace + files = migrate_script._iter_memory_files(tmp_path) + names = [p.name for p in files] + assert names == ["MEMORY.md"] + + +def test_iter_memory_files_skips_missing(migrate_script, tmp_path): + (tmp_path / "MEMORY.md").write_text("just memory") + files = migrate_script._iter_memory_files(tmp_path) + assert [p.name for p in files] == ["MEMORY.md"] + + +def test_today_tag_format(migrate_script): + tag = migrate_script._today_tag() + assert re.match(r"^migration:\d{4}-\d{2}-\d{2}$", tag), tag