Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions taosmd/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
# Key under which the registry auth token (taOS local/admin token) is stored.
# Used to poll the auth-gated registry revoked feed; the pubkey feed is public.
_REGISTRY_TOKEN_KEY = "registry_token"
# Key under which the taOS Files base URL is stored.
_FILES_URL_KEY = "files_url"
# Who manages this taosmd instance: "standalone" (default) or "taos".
_MANAGED_BY_KEY = "managed_by"
# Override: serve the web dashboard even when managed_by=taos.
Expand Down Expand Up @@ -415,6 +417,49 @@ def set_registry_token(token: str, clear: bool = False, data_dir=None) -> None:
_write(data, data_dir)


def get_files_url(data_dir=None) -> str | None:
"""Return the configured taOS Files base URL, or ``None`` if unset.

Resolution order (first non-empty wins):

1. ``TAOSMD_FILES_URL`` environment variable
2. ``files_url`` key in ``~/.taosmd/config.json``

When set, the ref-fetch helper resolves ``taos://`` refs against this
controller. When unset, the helper falls back to ``registry_url`` so a
single-controller install needs only one setting.
"""
env = os.environ.get("TAOSMD_FILES_URL")
if env and env.strip():
return env.strip()
url = _read(data_dir).get(_FILES_URL_KEY)
if isinstance(url, str) and url.strip():
return url.strip()
return None


def set_files_url(url: str, clear: bool = False, data_dir=None) -> None:
"""Persist the taOS Files base URL (or clear it).

Args:
url: Base URL of the taOS controller serving the Files API, e.g.
``"http://taos:8000"``. Ignored when ``clear`` is True.
clear: when True, remove the setting (ref-fetch falls back to
``registry_url``).

Raises:
ValueError: when ``clear`` is False and ``url`` is not a non-empty string.
"""
data = _read(data_dir)
if clear:
data.pop(_FILES_URL_KEY, None)
else:
if not isinstance(url, str) or not url.strip():
raise ValueError("url must be a non-empty string (or pass clear=True)")
data[_FILES_URL_KEY] = url.strip()
_write(data, data_dir)


# ---------------------------------------------------------------------------
# Remote server bearer token
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -681,6 +726,8 @@ def set_collections_allowed_roots(roots, clear: bool = False, data_dir=None) ->
"set_admin_token",
"get_registry_url",
"set_registry_url",
"get_files_url",
"set_files_url",
"get_registry_token",
"set_registry_token",
"get_managed_by",
Expand Down
153 changes: 153 additions & 0 deletions taosmd/ref_fetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""Resolve and fetch taOS Files-backed refs with hash verification.

A ref uri of the form ``taos://<project-slug>/files/<path>`` is resolved to
``GET /api/projects/{slug}/files/{path}`` on the configured controller. The
fetch helper verifies the returned bytes against the ref's ``sha256`` and
returns the verified bytes or a typed error.

No new server storage is introduced: this is purely a client-side helper.
"""

from __future__ import annotations

import hashlib
import os
import urllib.parse


class RefFetchError(Exception):
"""Base error for ref fetch failures."""


class HashMismatchError(RefFetchError):
"""Raised when the fetched bytes do not match the ref's sha256."""


class NotFoundError(RefFetchError):
"""Raised when the Files API reports the resource is missing."""


class UnauthorizedError(RefFetchError):
"""Raised when the Files API reports an auth failure."""


_SCHEME_PREFIX = "taos://"
_FILES_SEGMENT = "files/"


def _reject_dot_segments(path: str) -> None:
"""Raise ValueError if path contains . or .. segments (literal or encoded)."""
decoded = urllib.parse.unquote(path)
for segment in decoded.split("/"):
if segment in (".", ".."):
raise ValueError(
f"invalid taos ref uri: path contains dot segment {segment!r}"
)


def resolve_ref_uri(ref, files_url: str) -> str:
"""Map a taos:// ref uri to the concrete Files API fetch endpoint.

Args:
ref: A ref dict with at least a ``uri`` field.
files_url: Base URL of the taOS controller.

Returns:
The full URL for ``GET /api/projects/{slug}/files/{path}``.

Raises:
ValueError: If the uri scheme is not ``taos://`` or the shape is invalid.
"""
uri = ref.get("uri") if isinstance(ref, dict) else None
if not isinstance(uri, str) or not uri.startswith(_SCHEME_PREFIX):
raise ValueError(
f"unsupported uri scheme: {uri!r} (only taos:// is accepted)"
)
rest = uri[len(_SCHEME_PREFIX):]
parts = rest.split("/", 1)
if len(parts) != 2 or not parts[1].startswith(_FILES_SEGMENT):
raise ValueError(
f"invalid taos ref uri: {uri!r} (expected taos://<slug>/files/<path>)"
)
slug = urllib.parse.quote(parts[0], safe="")
path = parts[1][len(_FILES_SEGMENT):]
if not path:
raise ValueError(
f"invalid taos ref uri: {uri!r} (path is empty)"
)
if path.startswith("/"):
raise ValueError(
f"invalid taos ref uri: {uri!r} (path must not be absolute)"
)
_reject_dot_segments(path)
encoded_path = urllib.parse.quote(path, safe="/")
base = files_url.rstrip("/") if files_url else files_url

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[WARNING]: base = files_url.rstrip("/") if files_url else files_url does not validate None

When files_url is None (possible for direct callers of the public resolve_ref_uri), the subsequent f-string produces the malformed URL "None/api/projects/..." instead of failing fast. Consider validating files_url at the start of the function.


Reply with @kilocode-bot fix it to have Kilo Code address this issue.

return f"{base}/api/projects/{slug}/files/{encoded_path}"


async def fetch_by_ref(ref, fetcher, agent) -> bytes:
"""Fetch bytes for a ref using an injected fetcher and verify the hash.

Args:
ref: A ref dict with ``uri`` and ``sha256`` fields.
fetcher: A callable ``fetcher(url: str, agent: str) -> bytes`` that
performs the HTTP GET and returns the raw response body. It should
raise :class:`NotFoundError` or :class:`UnauthorizedError` for
those HTTP status codes.
agent: The agent identity (passed to ``fetcher`` for auth context).

Returns:
The verified raw bytes.

Raises:
ValueError: If the uri cannot be resolved.
HashMismatchError: If the fetched bytes' sha256 does not match ref.sha256.
NotFoundError: If the fetcher reports the resource is missing.
UnauthorizedError: If the fetcher reports an auth failure.
RefFetchError: For other fetch failures.
"""
import asyncio

files_url = _get_files_url()
url = resolve_ref_uri(ref, files_url)
loop = asyncio.get_running_loop()
raw = await loop.run_in_executor(None, fetcher, url, agent)
expected = ref.get("sha256") if isinstance(ref, dict) else None
if not expected:
raise RefFetchError("ref has no sha256")
actual = hashlib.sha256(raw).hexdigest()
if actual != expected:
raise HashMismatchError("sha256 mismatch")
return raw


def _get_files_url() -> str:
"""Resolve the files base URL from env or config.

Falls back to ``registry_url`` when ``files_url`` is unset, so a
single-controller install needs only one setting.
"""
env = os.environ.get("TAOSMD_FILES_URL")
if env and env.strip():
return env.strip()
from .config import get_files_url
url = get_files_url()
if url:
return url
from .config import get_registry_url
url = get_registry_url()
if url:
return url
raise RefFetchError(
"files_url is not configured: set TAOSMD_FILES_URL or files_url in config.json"
)


__all__ = [
"RefFetchError",
"HashMismatchError",
"NotFoundError",
"UnauthorizedError",
"resolve_ref_uri",
"fetch_by_ref",
]
8 changes: 8 additions & 0 deletions taosmd/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,4 +401,12 @@ async def task_remove_edge(
)


async def fetch_by_ref(self, ref: dict, agent: str, **opts) -> dict:
"""POST /refs/fetch: proxy a ref fetch to the remote server.

Returns ``{"bytes": <base64-str>, "sha256": <hash>, "size": <int>}``.
"""
return await self._run("POST", "/refs/fetch", {"ref": ref, "agent": agent})


__all__ = ["RemoteClient"]
62 changes: 61 additions & 1 deletion taosmd/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

from __future__ import annotations

import hashlib
import json
import logging

Expand Down Expand Up @@ -341,6 +342,65 @@ async def stats(*, agent: str, data_dir=None) -> dict:
return out


async def fetch_by_ref(ref: dict, *, agent: str, data_dir=None) -> dict:
"""Fetch and verify bytes for a taOS Files-backed ref.

Thin wrapper over :func:`taosmd.ref_fetch.fetch_by_ref`. Resolves the
controller base URL from config, builds a fetcher that authenticates with
the registry token, and returns the verified bytes as a base64-encoded
string together with its sha256 and size.

Returns ``{"bytes": <base64-str>, "sha256": <hash>, "size": <int>}``.

Raises :class:`ValueError` for an unresolvable uri,
:class:`~taosmd.ref_fetch.HashMismatchError` for a hash mismatch,
:class:`~taosmd.ref_fetch.NotFoundError` for a 404, or
:class:`~taosmd.ref_fetch.UnauthorizedError` for a 401/403.
"""
import base64

remote = _get_remote(data_dir)
if remote is not None:
return await remote.fetch_by_ref(ref, agent=agent)

from . import config as _config
from .ref_fetch import HashMismatchError, NotFoundError, RefFetchError, UnauthorizedError, fetch_by_ref as _fetch_by_ref

files_url = _config.get_files_url(data_dir)
if not files_url:
raise RefFetchError(
"files_url is not configured: set TAOSMD_FILES_URL or files_url in config.json"
)
registry_token = _config.get_registry_token(data_dir)

def _fetcher(url: str, agent: str) -> bytes:
import urllib.error
import urllib.request

headers = {"Accept": "application/octet-stream"}
if registry_token:
headers["Authorization"] = f"Bearer {registry_token}"
req = urllib.request.Request(url, headers=headers, method="GET")
try:
with urllib.request.urlopen(req, timeout=30) as resp:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[WARNING]: urllib.request.urlopen follows redirects by default

The PR title flags "bearer token still crosses origins on redirect". While Python 3.10+ strips the Authorization header on cross-origin redirects, the fetcher still follows the redirect blindly. Consider using a custom HTTPRedirectHandler to prevent cross-origin redirects or explicitly handle them.


Reply with @kilocode-bot fix it to have Kilo Code address this issue.

return resp.read()
except urllib.error.HTTPError as exc:
if exc.code in (401, 403):
raise UnauthorizedError(f"HTTP {exc.code} from {url}") from exc
if exc.code == 404:
raise NotFoundError(f"HTTP 404 from {url}") from exc
raise

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[WARNING]: raise re-raises raw urllib.error.HTTPError for non-401/403/404 HTTP errors

The docstring promises only RefFetchError subclasses or ValueError, but callers must also handle raw urllib.error.HTTPError for server errors (500, 502, etc.). Consider wrapping them in RefFetchError for API consistency.


Reply with @kilocode-bot fix it to have Kilo Code address this issue.

except urllib.error.URLError as exc:
raise RefFetchError(f"fetch failed for {url}: {exc}") from exc

raw = await _fetch_by_ref(ref, _fetcher, agent)
return {
"bytes": base64.b64encode(raw).decode("ascii"),
"sha256": hashlib.sha256(raw).hexdigest(),
"size": len(raw),
}


async def a2a_send(
sender: str,
body: str,
Expand Down Expand Up @@ -1031,7 +1091,7 @@ async def collections_archive(collection_id: str, *, data_dir=None) -> dict:


__all__ = ["ingest", "search", "pending_list", "pending_resolve", "reconcile", "stats",
"supersede", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members",
"supersede", "fetch_by_ref", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members",
"task_create", "task_list", "task_ready", "task_prime",
"task_update", "task_add_edge", "task_remove_edge", "task_projects",
"admin_shelf_create", "admin_shelf_archive", "admin_shelf_unarchive",
Expand Down
Loading