Repository navigation
Revise PR #242: bearer token still crosses origins on redirect, plus a missing route #262
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| """Resolve and fetch taOS Files-backed refs with hash verification. | ||
|
|
||
| A ref uri of the form ``taos://<project-slug>/files/<path>`` is resolved to | ||
| ``GET /api/projects/{slug}/files/{path}`` on the configured controller. The | ||
| fetch helper verifies the returned bytes against the ref's ``sha256`` and | ||
| returns the verified bytes or a typed error. | ||
|
|
||
| No new server storage is introduced: this is purely a client-side helper. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import hashlib | ||
| import os | ||
| import urllib.parse | ||
|
|
||
|
|
||
| class RefFetchError(Exception): | ||
| """Base error for ref fetch failures.""" | ||
|
|
||
|
|
||
| class HashMismatchError(RefFetchError): | ||
| """Raised when the fetched bytes do not match the ref's sha256.""" | ||
|
|
||
|
|
||
| class NotFoundError(RefFetchError): | ||
| """Raised when the Files API reports the resource is missing.""" | ||
|
|
||
|
|
||
| class UnauthorizedError(RefFetchError): | ||
| """Raised when the Files API reports an auth failure.""" | ||
|
|
||
|
|
||
| _SCHEME_PREFIX = "taos://" | ||
| _FILES_SEGMENT = "files/" | ||
|
|
||
|
|
||
| def _reject_dot_segments(path: str) -> None: | ||
| """Raise ValueError if path contains . or .. segments (literal or encoded).""" | ||
| decoded = urllib.parse.unquote(path) | ||
| for segment in decoded.split("/"): | ||
| if segment in (".", ".."): | ||
| raise ValueError( | ||
| f"invalid taos ref uri: path contains dot segment {segment!r}" | ||
| ) | ||
|
|
||
|
|
||
| def resolve_ref_uri(ref, files_url: str) -> str: | ||
| """Map a taos:// ref uri to the concrete Files API fetch endpoint. | ||
|
|
||
| Args: | ||
| ref: A ref dict with at least a ``uri`` field. | ||
| files_url: Base URL of the taOS controller. | ||
|
|
||
| Returns: | ||
| The full URL for ``GET /api/projects/{slug}/files/{path}``. | ||
|
|
||
| Raises: | ||
| ValueError: If the uri scheme is not ``taos://`` or the shape is invalid. | ||
| """ | ||
| uri = ref.get("uri") if isinstance(ref, dict) else None | ||
| if not isinstance(uri, str) or not uri.startswith(_SCHEME_PREFIX): | ||
| raise ValueError( | ||
| f"unsupported uri scheme: {uri!r} (only taos:// is accepted)" | ||
| ) | ||
| rest = uri[len(_SCHEME_PREFIX):] | ||
| parts = rest.split("/", 1) | ||
| if len(parts) != 2 or not parts[1].startswith(_FILES_SEGMENT): | ||
| raise ValueError( | ||
| f"invalid taos ref uri: {uri!r} (expected taos://<slug>/files/<path>)" | ||
| ) | ||
| slug = urllib.parse.quote(parts[0], safe="") | ||
| path = parts[1][len(_FILES_SEGMENT):] | ||
| if not path: | ||
| raise ValueError( | ||
| f"invalid taos ref uri: {uri!r} (path is empty)" | ||
| ) | ||
| if path.startswith("/"): | ||
| raise ValueError( | ||
| f"invalid taos ref uri: {uri!r} (path must not be absolute)" | ||
| ) | ||
| _reject_dot_segments(path) | ||
| encoded_path = urllib.parse.quote(path, safe="/") | ||
| base = files_url.rstrip("/") if files_url else files_url | ||
| return f"{base}/api/projects/{slug}/files/{encoded_path}" | ||
|
|
||
|
|
||
| async def fetch_by_ref(ref, fetcher, agent) -> bytes: | ||
| """Fetch bytes for a ref using an injected fetcher and verify the hash. | ||
|
|
||
| Args: | ||
| ref: A ref dict with ``uri`` and ``sha256`` fields. | ||
| fetcher: A callable ``fetcher(url: str, agent: str) -> bytes`` that | ||
| performs the HTTP GET and returns the raw response body. It should | ||
| raise :class:`NotFoundError` or :class:`UnauthorizedError` for | ||
| those HTTP status codes. | ||
| agent: The agent identity (passed to ``fetcher`` for auth context). | ||
|
|
||
| Returns: | ||
| The verified raw bytes. | ||
|
|
||
| Raises: | ||
| ValueError: If the uri cannot be resolved. | ||
| HashMismatchError: If the fetched bytes' sha256 does not match ref.sha256. | ||
| NotFoundError: If the fetcher reports the resource is missing. | ||
| UnauthorizedError: If the fetcher reports an auth failure. | ||
| RefFetchError: For other fetch failures. | ||
| """ | ||
| import asyncio | ||
|
|
||
| files_url = _get_files_url() | ||
| url = resolve_ref_uri(ref, files_url) | ||
| loop = asyncio.get_running_loop() | ||
| raw = await loop.run_in_executor(None, fetcher, url, agent) | ||
| expected = ref.get("sha256") if isinstance(ref, dict) else None | ||
| if not expected: | ||
| raise RefFetchError("ref has no sha256") | ||
| actual = hashlib.sha256(raw).hexdigest() | ||
| if actual != expected: | ||
| raise HashMismatchError("sha256 mismatch") | ||
| return raw | ||
|
|
||
|
|
||
| def _get_files_url() -> str: | ||
| """Resolve the files base URL from env or config. | ||
|
|
||
| Falls back to ``registry_url`` when ``files_url`` is unset, so a | ||
| single-controller install needs only one setting. | ||
| """ | ||
| env = os.environ.get("TAOSMD_FILES_URL") | ||
| if env and env.strip(): | ||
| return env.strip() | ||
| from .config import get_files_url | ||
| url = get_files_url() | ||
| if url: | ||
| return url | ||
| from .config import get_registry_url | ||
| url = get_registry_url() | ||
| if url: | ||
| return url | ||
| raise RefFetchError( | ||
| "files_url is not configured: set TAOSMD_FILES_URL or files_url in config.json" | ||
| ) | ||
|
|
||
|
|
||
| __all__ = [ | ||
| "RefFetchError", | ||
| "HashMismatchError", | ||
| "NotFoundError", | ||
| "UnauthorizedError", | ||
| "resolve_ref_uri", | ||
| "fetch_by_ref", | ||
| ] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,6 +25,7 @@ | |
|
|
||
| from __future__ import annotations | ||
|
|
||
| import hashlib | ||
| import json | ||
| import logging | ||
|
|
||
|
|
@@ -341,6 +342,65 @@ async def stats(*, agent: str, data_dir=None) -> dict: | |
| return out | ||
|
|
||
|
|
||
| async def fetch_by_ref(ref: dict, *, agent: str, data_dir=None) -> dict: | ||
| """Fetch and verify bytes for a taOS Files-backed ref. | ||
|
|
||
| Thin wrapper over :func:`taosmd.ref_fetch.fetch_by_ref`. Resolves the | ||
| controller base URL from config, builds a fetcher that authenticates with | ||
| the registry token, and returns the verified bytes as a base64-encoded | ||
| string together with its sha256 and size. | ||
|
|
||
| Returns ``{"bytes": <base64-str>, "sha256": <hash>, "size": <int>}``. | ||
|
|
||
| Raises :class:`ValueError` for an unresolvable uri, | ||
| :class:`~taosmd.ref_fetch.HashMismatchError` for a hash mismatch, | ||
| :class:`~taosmd.ref_fetch.NotFoundError` for a 404, or | ||
| :class:`~taosmd.ref_fetch.UnauthorizedError` for a 401/403. | ||
| """ | ||
| import base64 | ||
|
|
||
| remote = _get_remote(data_dir) | ||
| if remote is not None: | ||
| return await remote.fetch_by_ref(ref, agent=agent) | ||
|
|
||
| from . import config as _config | ||
| from .ref_fetch import HashMismatchError, NotFoundError, RefFetchError, UnauthorizedError, fetch_by_ref as _fetch_by_ref | ||
|
|
||
| files_url = _config.get_files_url(data_dir) | ||
| if not files_url: | ||
| raise RefFetchError( | ||
| "files_url is not configured: set TAOSMD_FILES_URL or files_url in config.json" | ||
| ) | ||
| registry_token = _config.get_registry_token(data_dir) | ||
|
|
||
| def _fetcher(url: str, agent: str) -> bytes: | ||
| import urllib.error | ||
| import urllib.request | ||
|
|
||
| headers = {"Accept": "application/octet-stream"} | ||
| if registry_token: | ||
| headers["Authorization"] = f"Bearer {registry_token}" | ||
| req = urllib.request.Request(url, headers=headers, method="GET") | ||
| try: | ||
| with urllib.request.urlopen(req, timeout=30) as resp: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [WARNING]: The PR title flags "bearer token still crosses origins on redirect". While Python 3.10+ strips the Reply with |
||
| return resp.read() | ||
| except urllib.error.HTTPError as exc: | ||
| if exc.code in (401, 403): | ||
| raise UnauthorizedError(f"HTTP {exc.code} from {url}") from exc | ||
| if exc.code == 404: | ||
| raise NotFoundError(f"HTTP 404 from {url}") from exc | ||
| raise | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [WARNING]: The docstring promises only Reply with |
||
| except urllib.error.URLError as exc: | ||
| raise RefFetchError(f"fetch failed for {url}: {exc}") from exc | ||
|
|
||
| raw = await _fetch_by_ref(ref, _fetcher, agent) | ||
| return { | ||
| "bytes": base64.b64encode(raw).decode("ascii"), | ||
| "sha256": hashlib.sha256(raw).hexdigest(), | ||
| "size": len(raw), | ||
| } | ||
|
|
||
|
|
||
| async def a2a_send( | ||
| sender: str, | ||
| body: str, | ||
|
|
@@ -1031,7 +1091,7 @@ async def collections_archive(collection_id: str, *, data_dir=None) -> dict: | |
|
|
||
|
|
||
| __all__ = ["ingest", "search", "pending_list", "pending_resolve", "reconcile", "stats", | ||
| "supersede", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members", | ||
| "supersede", "fetch_by_ref", "a2a_send", "a2a_feed", "a2a_channels", "a2a_members", | ||
| "task_create", "task_list", "task_ready", "task_prime", | ||
| "task_update", "task_add_edge", "task_remove_edge", "task_projects", | ||
| "admin_shelf_create", "admin_shelf_archive", "admin_shelf_unarchive", | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[WARNING]:
base = files_url.rstrip("/") if files_url else files_urldoes not validateNoneWhen
files_urlisNone(possible for direct callers of the publicresolve_ref_uri), the subsequent f-string produces the malformed URL"None/api/projects/..."instead of failing fast. Consider validatingfiles_urlat the start of the function.Reply with
@kilocode-bot fix itto have Kilo Code address this issue.