Revise PR #312: the as_of/review_by coercion fix is correct but ships with zero tests and drops the producer half of the feature

changelog.d/tsk-agi5ph-collection-doc-currency.md

@@ -0,0 +1,5 @@
+### Fixed
+- `_format_hit` now defensively coerces `as_of` to float (ISO-8601 string timestamps degrade to `0.0` with `logger.warning`) and guards the `review_by` comparison on `isinstance(review_by, str)`, so malformed caller metadata from `ingest_batch` degrades gracefully instead of raising `ValueError`/`TypeError` on the search path. Non-string `review_by` values also log a warning (symmetric with the `as_of` guard) instead of silently yielding `is_past_review=False`.
+
+### Added
+- Markdown front matter (`doc_id`, `version` as int, `review_by` as validated ISO date) is now parsed at collection index time via `_parse_front_matter` and stored on chunk metadata as `indexed_at` plus the doc-currency keys. Search hits gain `is_current`, `as_of` (always float), `is_past_review`, and (on superseded rows) `superseded_by`. Front-matter parsing respects both `.md` and `.markdown` extensions and ignores document keys from thematic-break documents via a closing-delimiter line budget.

taosmd/api.py

@@ -449,12 +449,16 @@ def _format_hit(hit: dict) -> dict:
         user_md = dict(user_md)  # copy: never mutate the stored row metadata
         for key, value in preserved.items():
             user_md.setdefault(key, value)
+        # Warn (not assert) if critical provenance keys were lost during
+        # unwrap. A bare assert is stripped under ``-O`` and would convert a
+        # metadata-shape anomaly into a 500 on the search request path.
+        for key in ("archive_span_id", "agent", "project"):
+            if key in (md or {}) and key not in user_md:
+                logger.warning(
+                    "_format_hit lost critical key %r during metadata unwrap",
+                    key,
+                )
 
-    confidence = (
-        md.get("similarity")
-        if isinstance(md, dict) and md.get("similarity") is not None
-        else hit.get("source_score", 0.0)
-    )
     timestamp = (
         user_md.get("timestamp")
         if isinstance(user_md, dict) and user_md.get("timestamp") is not None
@@ -463,6 +467,51 @@ def _format_hit(hit: dict) -> dict:
         or 0
     )
 
+    # Collection doc-currency metadata lives in the inner user-metadata dict
+    # (that is how ingest_folder's chunk_md stores it), so read from user_md
+    # after the unwrap, not from the outer envelope.
+    # is_current is False for superseded/history rows (hidden_by set);
+    # always True on the default active-recall path.
+    # as_of is the indexing timestamp: indexed_at (float epoch) or the
+    # row's created_at/timestamp, coerced to float for a single type.
+    # is_past_review is True only when review_by is present and overdue.
+    if isinstance(user_md, dict):
+        is_current = not (isinstance(md, dict) and "hidden_by" in md)
+        as_of = user_md.get("indexed_at")
+        if as_of is None:
+            as_of = timestamp
+        try:
+            as_of = float(as_of)
+        except (TypeError, ValueError):
+            logger.warning(
+                "taosmd: could not convert as_of %r to float, defaulting to 0.0",
+                as_of,
+            )
+            as_of = 0.0
+        review_by = user_md.get("review_by")
+        has_review_by = review_by is not None
+        if has_review_by and not isinstance(review_by, str):
+            logger.warning(
+                "taosmd: review_by %r is not a string, is_past_review defaults to False",
+                review_by,
+            )
+        is_past_review = has_review_by and isinstance(review_by, str) and review_by < time.strftime("%Y-%m-%d")
+
+        user_md["is_current"] = is_current
+        user_md["as_of"] = as_of
+        if has_review_by:
+            user_md["is_past_review"] = is_past_review
+        # doc_id / version / review_by are already in user_md when present;
+        # no extraction needed. Only superseded_by comes from the envelope.
+        if isinstance(md, dict) and "hidden_by" in md:
+            user_md["superseded_by"] = md["hidden_by"]
+
+    confidence = (
+        md.get("similarity")
+        if isinstance(md, dict) and md.get("similarity") is not None
+        else hit.get("source_score", 0.0)
+    )
+
     return {
         "text": hit.get("text", ""),
         "source": hit.get("source", "unknown"),

taosmd/collections.py

@@ -41,6 +41,7 @@
 import secrets
 import sqlite3
 import time
+from datetime import date
 from pathlib import Path
 
 from . import _db
@@ -494,6 +495,78 @@ def _is_ignored(
     return ignored
 
 
+#: The only front-matter keys consumed by the doc-currency contract.
+_FRONT_MATTER_KEYS = frozenset({"doc_id", "version", "review_by"})
+
+#: How many lines deep we look for the closing ``---`` delimiter. Keeps a
+#: document that opens with a thematic break (``---`` followed by prose)
+#: from being scanned as front matter all the way to the next ``---``.
+_FRONT_MATTER_MAX_LINES = 50
+
+
+def _parse_front_matter(file_path: str) -> dict:
+    """Parse doc-currency front matter from a markdown file (stdlib-only).
+
+    Recognises the standard ``---`` delimiter pair: an opening ``---`` as the
+    first line of the file, a closing ``---``, and simple ``key: value`` lines
+    in between. Only ``doc_id`` (str), ``version`` (int), and ``review_by``
+    (ISO date str) are captured; unknown keys are skipped, a ``version`` that
+    cannot be parsed as ``int`` is dropped, and a ``review_by`` that is not a
+    valid ISO date is dropped. Absent keys are omitted from the result.
+    """
+    try:
+        text = Path(file_path).read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return {}
+    if not text.startswith("---\n"):
+        return {}
+    lines = text.splitlines(keepends=True)
+    fm_lines: list[str] = []
+    closing_found = False
+    budget = min(len(lines), _FRONT_MATTER_MAX_LINES + 1)
+    for i in range(1, budget):
+        stripped = lines[i].strip()
+        if stripped == "---":
+            closing_found = True
+            break
+        fm_lines.append(lines[i])
+    if not closing_found or not fm_lines:
+        return {}
+    result: dict = {}
+    for line in fm_lines:
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+        if key not in _FRONT_MATTER_KEYS:
+            continue
+        # Strip surrounding quotes.
+        if (
+            len(value) >= 2
+            and value[0] == value[-1]
+            and value[0] in ('"', "'")
+        ):
+            value = value[1:-1]
+        if key == "version":
+            try:
+                result["version"] = int(value)
+            except ValueError:
+                continue
+        elif key == "review_by":
+            try:
+                date.fromisoformat(value)
+            except ValueError:
+                continue
+            result["review_by"] = value
+        else:
+            result[key] = value
+    return result
+
+
 def _loader_for(path: Path):
     """Return an instance of the loader that explicitly claims ``path``,
     or ``None`` when no registered loader does.
@@ -819,20 +892,30 @@ async def ingest_folder(
                     continue
                 if rel in prior:
                     changed.append(rel)
+                # Parse front-matter from markdown files at index time.
+                fm: dict = {}
+                if abs_path.suffix.lower() in {".md", ".markdown"}:
+                    fm = _parse_front_matter(str(abs_path))
                 for i, chunk in enumerate(chunks):
                     chunk_id = hashlib.sha256(
                         f"{collection_id}:{rel}:{file_hash}:{i}".encode("utf-8")
                     ).hexdigest()
+                    chunk_md: dict = {
+                        "collection_id": collection_id,
+                        "file_path": rel,
+                        "source": "collection",
+                        "chunk_index": i,
+                        "file_hash": file_hash,
+                        "indexed_at": time.time(),
+                    }
+                    # Add front-matter parsed fields when present.
+                    for key in _FRONT_MATTER_KEYS:
+                        if key in fm:
+                            chunk_md[key] = fm[key]
                     items.append({
                         "text": chunk,
                         "id": chunk_id,
-                        "metadata": {
-                            "collection_id": collection_id,
-                            "file_path": rel,
-                            "source": "collection",
-                            "chunk_index": i,
-                            "file_hash": file_hash,
-                        },
+                        "metadata": chunk_md,
                     })
                 indexed.append((rel, file_hash))