Revise PR #242: bearer token still crosses origins on redirect, plus a missing route

taosmd/config.py

@@ -42,6 +42,8 @@
 # Key under which the registry auth token (taOS local/admin token) is stored.
 # Used to poll the auth-gated registry revoked feed; the pubkey feed is public.
 _REGISTRY_TOKEN_KEY = "registry_token"
+# Key under which the taOS Files base URL is stored.
+_FILES_URL_KEY = "files_url"
 # Who manages this taosmd instance: "standalone" (default) or "taos".
 _MANAGED_BY_KEY = "managed_by"
 # Override: serve the web dashboard even when managed_by=taos.
@@ -415,6 +417,49 @@ def set_registry_token(token: str, clear: bool = False, data_dir=None) -> None:
     _write(data, data_dir)
 
 
+def get_files_url(data_dir=None) -> str | None:
+    """Return the configured taOS Files base URL, or ``None`` if unset.
+
+    Resolution order (first non-empty wins):
+
+    1. ``TAOSMD_FILES_URL`` environment variable
+    2. ``files_url`` key in ``~/.taosmd/config.json``
+
+    When set, the ref-fetch helper resolves ``taos://`` refs against this
+    controller. When unset, the helper falls back to ``registry_url`` so a
+    single-controller install needs only one setting.
+    """
+    env = os.environ.get("TAOSMD_FILES_URL")
+    if env and env.strip():
+        return env.strip()
+    url = _read(data_dir).get(_FILES_URL_KEY)
+    if isinstance(url, str) and url.strip():
+        return url.strip()
+    return None
+
+
+def set_files_url(url: str, clear: bool = False, data_dir=None) -> None:
+    """Persist the taOS Files base URL (or clear it).
+
+    Args:
+        url: Base URL of the taOS controller serving the Files API, e.g.
+            ``"http://taos:8000"``. Ignored when ``clear`` is True.
+        clear: when True, remove the setting (ref-fetch falls back to
+            ``registry_url``).
+
+    Raises:
+        ValueError: when ``clear`` is False and ``url`` is not a non-empty string.
+    """
+    data = _read(data_dir)
+    if clear:
+        data.pop(_FILES_URL_KEY, None)
+    else:
+        if not isinstance(url, str) or not url.strip():
+            raise ValueError("url must be a non-empty string (or pass clear=True)")
+        data[_FILES_URL_KEY] = url.strip()
+    _write(data, data_dir)
+
+
 # ---------------------------------------------------------------------------
 # Remote server bearer token
 # ---------------------------------------------------------------------------
@@ -681,6 +726,8 @@ def set_collections_allowed_roots(roots, clear: bool = False, data_dir=None) ->
     "set_admin_token",
     "get_registry_url",
     "set_registry_url",
+    "get_files_url",
+    "set_files_url",
     "get_registry_token",
     "set_registry_token",
     "get_managed_by",

taosmd/http_server.py

@@ -91,6 +91,7 @@
                            Adds ``"vector_failures": int`` and ``"degraded": true`` to the
                            result when embedding failed for one or more items (archived,
                            repairable via reconcile), same as single ingest.
+``POST /refs/fetch``        ``{"ref": {"uri": "taos://<slug>/files/<path>", "sha256": "<hex>"}, "agent"}`` -> ``{"bytes": "<base64>", "sha256": "<hex>", "size": <int>}``
 ``POST /search``           ``{"query", "agent", "limit"?, "project"?, "also_include"?, "mode"?, "collection"?, "collections"?: [...], "collections_only"?: bool}`` -> ``{"hits": [...]}``
                            ``collection``/``collections`` add granted collections' indexed
                            content to the search (grants enforced per requesting agent;
@@ -937,6 +938,8 @@ def _dispatch(self, method: str) -> None:
                     self._handle_ingest()
                 elif method == "POST" and path == "/ingest/batch":
                     self._handle_ingest_batch()
+                elif method == "POST" and path == "/refs/fetch":
+                    self._handle_refs_fetch()
                 elif method == "GET" and path == "/projects":
                     self._handle_list_projects()
                 elif method == "GET" and path == "/shelves":
@@ -1104,6 +1107,19 @@ def _handle_ingest_batch(self) -> None:
             )
             self._send_json(200, result)
 
+        def _handle_refs_fetch(self) -> None:
+            body = self._read_json_body()
+            ref = body.get("ref")
+            agent = body.get("agent")
+            if not isinstance(ref, dict):
+                raise _BadRequest("'ref' (object) is required")
+            if not isinstance(agent, str) or not agent:
+                raise _BadRequest("'agent' (non-empty string) is required")
+            result = runner.run(
+                service.fetch_by_ref(ref=ref, agent=agent, data_dir=data_dir)
+            )
+            self._send_json(200, result)
+
         def _handle_search_post(self) -> None:
             body = self._read_json_body()
             query = body.get("query")

taosmd/ref_fetch.py

@@ -0,0 +1,154 @@
+"""Resolve and fetch taOS Files-backed refs with hash verification.
+
+A ref uri of the form ``taos://<project-slug>/files/<path>`` is resolved to
+``GET /api/projects/{slug}/files/{path}`` on the configured controller. The
+fetch helper verifies the returned bytes against the ref's ``sha256`` and
+returns the verified bytes or a typed error.
+
+No new server storage is introduced: this is purely a client-side helper.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import urllib.parse
+
+
+class RefFetchError(Exception):
+    """Base error for ref fetch failures."""
+
+
+class HashMismatchError(RefFetchError):
+    """Raised when the fetched bytes do not match the ref's sha256."""
+
+
+class NotFoundError(RefFetchError):
+    """Raised when the Files API reports the resource is missing."""
+
+
+class UnauthorizedError(RefFetchError):
+    """Raised when the Files API reports an auth failure."""
+
+
+_SCHEME_PREFIX = "taos://"
+_FILES_SEGMENT = "files/"
+
+
+def _reject_dot_segments(path: str) -> None:
+    """Raise ValueError if path contains . or .. segments (literal or encoded)."""
+    decoded = urllib.parse.unquote(path)
+    for segment in decoded.split("/"):
+        if segment in (".", ".."):
+            raise ValueError(
+                f"invalid taos ref uri: path contains dot segment {segment!r}"
+            )
+
+
+def resolve_ref_uri(ref, files_url: str) -> str:
+    """Map a taos:// ref uri to the concrete Files API fetch endpoint.
+
+    Args:
+        ref: A ref dict with at least a ``uri`` field.
+        files_url: Base URL of the taOS controller.
+
+    Returns:
+        The full URL for ``GET /api/projects/{slug}/files/{path}``.
+
+    Raises:
+        ValueError: If the uri scheme is not ``taos://`` or the shape is invalid.
+    """
+    uri = ref.get("uri") if isinstance(ref, dict) else None
+    if not isinstance(uri, str) or not uri.startswith(_SCHEME_PREFIX):
+        raise ValueError(
+            f"unsupported uri scheme: {uri!r} (only taos:// is accepted)"
+        )
+    rest = uri[len(_SCHEME_PREFIX):]
+    parts = rest.split("/", 1)
+    if len(parts) != 2 or not parts[1].startswith(_FILES_SEGMENT):
+        raise ValueError(
+            f"invalid taos ref uri: {uri!r} (expected taos://<slug>/files/<path>)"
+        )
+    slug = urllib.parse.quote(parts[0], safe="")
+    path = parts[1][len(_FILES_SEGMENT):]
+    if not path:
+        raise ValueError(
+            f"invalid taos ref uri: {uri!r} (path is empty)"
+        )
+    if path.startswith("/"):
+        raise ValueError(
+            f"invalid taos ref uri: {uri!r} (path must not be absolute)"
+        )
+    _reject_dot_segments(path)
+    encoded_path = urllib.parse.quote(path, safe="/")
+    base = files_url.rstrip("/") if files_url else files_url
+    return f"{base}/api/projects/{slug}/files/{encoded_path}"
+
+
+async def fetch_by_ref(ref, fetcher, agent, data_dir=None) -> bytes:
+    """Fetch bytes for a ref using an injected fetcher and verify the hash.
+
+    Args:
+        ref: A ref dict with ``uri`` and ``sha256`` fields.
+        fetcher: A callable ``fetcher(url: str, agent: str) -> bytes`` that
+            performs the HTTP GET and returns the raw response body. It should
+            raise :class:`NotFoundError` or :class:`UnauthorizedError` for
+            those HTTP status codes.
+        agent: The agent identity (passed to ``fetcher`` for auth context).
+        data_dir: Optional data directory used to resolve the files base URL.
+
+    Returns:
+        The verified raw bytes.
+
+    Raises:
+        ValueError: If the uri cannot be resolved.
+        HashMismatchError: If the fetched bytes' sha256 does not match ref.sha256.
+        NotFoundError: If the fetcher reports the resource is missing.
+        UnauthorizedError: If the fetcher reports an auth failure.
+        RefFetchError: For other fetch failures.
+    """
+    import asyncio
+
+    files_url = _get_files_url(data_dir)
+    url = resolve_ref_uri(ref, files_url)
+    loop = asyncio.get_running_loop()
+    raw = await loop.run_in_executor(None, fetcher, url, agent)
+    expected = ref.get("sha256") if isinstance(ref, dict) else None
+    if not expected:
+        raise RefFetchError("ref has no sha256")
+    actual = hashlib.sha256(raw).hexdigest()
+    if actual != expected:
+        raise HashMismatchError("sha256 mismatch")
+    return raw
+
+
+def _get_files_url(data_dir=None) -> str:
+    """Resolve the files base URL from env or config.
+
+    Falls back to ``registry_url`` when ``files_url`` is unset, so a
+    single-controller install needs only one setting.
+    """
+    env = os.environ.get("TAOSMD_FILES_URL")
+    if env and env.strip():
+        return env.strip()
+    from .config import get_files_url
+    url = get_files_url(data_dir)
+    if url:
+        return url
+    from .config import get_registry_url
+    url = get_registry_url(data_dir)
+    if url:
+        return url
+    raise RefFetchError(
+        "files_url is not configured: set TAOSMD_FILES_URL or files_url in config.json"
+    )
+
+
+__all__ = [
+    "RefFetchError",
+    "HashMismatchError",
+    "NotFoundError",
+    "UnauthorizedError",
+    "resolve_ref_uri",
+    "fetch_by_ref",
+]

taosmd/remote.py

@@ -401,4 +401,12 @@ async def task_remove_edge(
         )
 
 
+    async def fetch_by_ref(self, ref: dict, agent: str, **opts) -> dict:
+        """POST /refs/fetch: proxy a ref fetch to the remote server.
+
+        Returns ``{"bytes": <base64-str>, "sha256": <hash>, "size": <int>}``.
+        """
+        return await self._run("POST", "/refs/fetch", {"ref": ref, "agent": agent})
+
+
 __all__ = ["RemoteClient"]

taosmd/service.py

@@ -25,6 +25,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 
@@ -341,6 +342,65 @@ async def stats(*, agent: str, data_dir=None) -> dict:
     return out
 
 
+async def fetch_by_ref(ref: dict, *, agent: str, data_dir=None) -> dict:
+    """Fetch and verify bytes for a taOS Files-backed ref.
+
+    Thin wrapper over :func:`taosmd.ref_fetch.fetch_by_ref`. Resolves the
+    controller base URL from config, builds a fetcher that authenticates with
+    the registry token, and returns the verified bytes as a base64-encoded
+    string together with its sha256 and size.
+
+    Returns ``{"bytes": <base64-str>, "sha256": <hash>, "size": <int>}``.
+
+    Raises :class:`ValueError` for an unresolvable uri,
+    :class:`~taosmd.ref_fetch.HashMismatchError` for a hash mismatch,
+    :class:`~taosmd.ref_fetch.NotFoundError` for a 404, or
+    :class:`~taosmd.ref_fetch.UnauthorizedError` for a 401/403.
+    """
+    import base64
+
+    remote = _get_remote(data_dir)
+    if remote is not None:
+        return await remote.fetch_by_ref(ref, agent=agent)
+
+    from . import config as _config
+    from .ref_fetch import HashMismatchError, NotFoundError, RefFetchError, UnauthorizedError, fetch_by_ref as _fetch_by_ref
+
+    registry_token = _config.get_registry_token(data_dir)
+
+    def _fetcher(url: str, agent: str) -> bytes:
+        import urllib.error
+        import urllib.request
+
+        class _NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                return None
+
+        headers = {"Accept": "application/octet-stream"}
+        if registry_token:
+            headers["Authorization"] = f"Bearer {registry_token}"
+        req = urllib.request.Request(url, headers=headers, method="GET")
+        try:
+            opener = urllib.request.build_opener(_NoRedirect)
+            with opener.open(req, timeout=30) as resp:
+                return resp.read()
+        except urllib.error.HTTPError as exc:
+            if exc.code in (401, 403):
+                raise UnauthorizedError(f"HTTP {exc.code} from {url}") from exc
+            if exc.code == 404:
+                raise NotFoundError(f"HTTP 404 from {url}") from exc
+            raise
+        except urllib.error.URLError as exc:
+            raise RefFetchError(f"fetch failed for {url}: {exc}") from exc
+
+    raw = await _fetch_by_ref(ref, _fetcher, agent, data_dir=data_dir)
+    return {
+        "bytes": base64.b64encode(raw).decode("ascii"),
+        "sha256": hashlib.sha256(raw).hexdigest(),
+        "size": len(raw),
+    }
+
+
 async def a2a_send(
     sender: str,
     body: str,
@@ -1031,7 +1091,7 @@ async def collections_archive(collection_id: str, *, data_dir=None) -> dict:
 
 
 __all__ = ["ingest", "search", "pending_list", "pending_resolve", "reconcile", "stats",
-           "supersede", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members",
+           "supersede", "fetch_by_ref", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members",
            "task_create", "task_list", "task_ready", "task_prime",
            "task_update", "task_add_edge", "task_remove_edge", "task_projects",
            "admin_shelf_create", "admin_shelf_archive", "admin_shelf_unarchive",

tests/test_http_server.py

@@ -799,3 +799,38 @@ def test_graph_activations_endpoint(live_server):
     assert status == 200, body
     assert "activations" in body and isinstance(body["activations"], list)
     assert "now" in body
+
+
+def test_post_refs_fetch_returns_bytes(live_server, monkeypatch):
+    from taosmd import service as svc
+
+    async def fake_fetch_by_ref(ref, agent, data_dir=None):
+        return {"bytes": "aGVsbG8=", "sha256": "abc", "size": 5}
+
+    monkeypatch.setattr(svc, "fetch_by_ref", fake_fetch_by_ref)
+
+    status, body = _post(
+        f"{live_server}/refs/fetch",
+        {"ref": {"uri": "taos://proj/files/hello.txt", "sha256": "abc"}, "agent": "http-test"},
+    )
+    assert status == 200, body
+    assert body["bytes"] == "aGVsbG8="
+    assert body["size"] == 5
+
+
+def test_post_refs_fetch_missing_ref_returns_400(live_server):
+    status, body = _post(
+        f"{live_server}/refs/fetch",
+        {"agent": "http-test"},
+    )
+    assert status == 400
+    assert "ref" in body["error"]
+
+
+def test_post_refs_fetch_missing_agent_returns_400(live_server):
+    status, body = _post(
+        f"{live_server}/refs/fetch",
+        {"ref": {"uri": "taos://proj/files/hello.txt", "sha256": "abc"}},
+    )
+    assert status == 400
+    assert "agent" in body["error"]