feat(bundle): run a bundle from the Hugging Face Hub with agent-env run hf://

README.md

@@ -44,6 +44,23 @@ uv sync --extra dev
 source .venv/bin/activate
 ```
 
+## Run a bundle from the Hub
+
+`agent-env run` also takes a bundle published in a [Hugging Face Hub](https://huggingface.co/datasets) dataset:
+
+```bash
+agent-env run hf://<org>/<repo>                     # the main branch
+agent-env run hf://<org>/<repo>@v1.0/bundles/hello  # a tag, branch or commit, and the bundle's folder
+```
+
+The revision resolves to a commit, and only that commit's bundle and the dataset card are downloaded, into
+`$XDG_CACHE_HOME/agentenv/hf/<org>/<repo>/<commit>` (`~/.cache` by default), where a later run of the commit finds
+them. Without a path, the run takes the only bundle under `bundles/`, or the one the card's `agentenv: default` names,
+else the repo's root. It prints the repo, commit and bundle, then runs as a folder does, with ids under
+`@hf/<org>/<repo>`, so a rerun of the same commit reuses what the last one wrote, and records the commit with each run.
+A bundle without a `bundle.toml` needs the plugins the card's `agentenv: bundles: <name>: plugins` lists. A gated or
+private dataset needs `hf auth login` or `HF_TOKEN`.
+
 ## Plugins
 
 A package of your own can add env types, providers, task steps and stores to agent-env, and `agent-env plugin` lists, checks, adds and removes installed plugins. The [plugin docs](https://www.agentenvframework.com/docs/plugins) show how.

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "click>=8.0.0",
     "cryptography>=42",  # the local transfer CA
     "httpx>=0.27.0",
+    "huggingface_hub>=1.0",  # agent-env run hf:// resolves and downloads a Hub bundle; 1.0 moved it onto httpx
     "jsonschema>=4.20",  # checks an app's settings against its Settings' schema before a deploy
     "litellm>=1.96.0,<1.97",
     "mcp>=1.25,<2",  # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway

src/agent_env/bundle/hub.py

@@ -0,0 +1,236 @@
+"""Bundles run straight from a Hugging Face Hub dataset repo, named ``hf://<org>/<repo>[@<revision>][/<path>]``.
+
+The revision, a branch, tag or commit (the main branch by default), resolves to a commit, and the bundle is picked from
+that commit's file list before anything is downloaded, so a results dataset's recordings never are. Only the bundle's
+files and the repo's card are downloaded, into a plain folder for the commit: the Hub's own cache keeps files as links
+into its blob store, and the bundle parser refuses a link that leaves the bundle. A commit never changes, so what was
+downloaded for it is reused; a reference pinned to the commit then needs no network at all. The bundle's ids are rooted
+at ``@hf/<org>/<repo>``, plus ``/<name>`` for a bundle under ``bundles/``, a local namespace of its own: the id says the
+bundle came from the Hub, and a rerun of the same commit reuses what the last one wrote. A gated or private repo is read
+with the Hub library's own login (``hf auth login`` or ``HF_TOKEN``).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote
+
+import httpx
+from huggingface_hub import HfApi
+from huggingface_hub.errors import RepositoryNotFoundError
+from huggingface_hub.repocard import metadata_load
+
+from agent_env.plugins._bundles import ALIASES_FILE
+from agent_env.plugins._requirements import unmet_problems
+from agent_env.store.ids import HF_PREFIX, validate_local_id
+
+from .parse import BUNDLE_TOML, BundleError, BundleKind
+from .requires import UnmetRequirements
+
+SCHEME = "hf://"
+BUNDLES = "bundles"
+CARD = "README.md"
+# The key a run's context records where its bundle came from under, in its metadata.
+SOURCE_KEY = "bundle_source"
+
+_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
+_SHA = re.compile(r"[0-9a-f]{40}")
+_KINDS = frozenset(kind.value for kind in BundleKind)
+_FORM = "hf://<org>/<repo>[@<revision>][/<path>]"
+
+
+@dataclass(frozen=True)
+class HubReference:
+    repo: str  # <org>/<repo>
+    revision: str | None  # None for the main branch
+    path: str | None  # the bundle's folder in the repo, when named
+
+
+@dataclass(frozen=True)
+class HubBundle:
+    """A bundle downloaded from the Hub: its folder, the id root it runs under, where it came from, and what the repo's
+    card says it needs."""
+
+    root: Path
+    id_root: str
+    source: Mapping[str, str]  # hub, repo, revision, sha, path: what each run's context records
+    plugins: tuple[str, ...]  # the card's, for a bundle without a bundle.toml
+    setup: str | None  # the card's command that sets the bundle up, shown when a run fails, never run
+
+    def check_plugins(self) -> None:
+        """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises
+        UnmetRequirements."""
+        problems, unmet = unmet_problems(self.plugins, f"{self.source['repo']}'s card")
+        if problems:
+            raise UnmetRequirements(problems, unmet) if unmet else BundleError(problems)
+
+
+@dataclass
+class _Commit:
+    """What this machine holds of a commit, kept in its folder: the commit's file list, and the bundles downloaded."""
+
+    files: list[str]
+    downloaded: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def path(folder: Path) -> Path:
+        return folder / ".cache" / "agentenv" / "commit.json"
+
+    @classmethod
+    def read(cls, folder: Path) -> _Commit | None:
+        try:
+            return cls(**json.loads(cls.path(folder).read_text()))
+        except FileNotFoundError:
+            return None
+
+    def save(self, folder: Path) -> None:
+        path = self.path(folder)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        partial = path.with_name(f"{path.name}.{os.getpid()}")
+        partial.write_text(json.dumps({"files": self.files, "downloaded": self.downloaded}))
+        os.replace(partial, path)
+
+
+def is_hub_reference(text: str) -> bool:
+    return text.startswith(SCHEME)
+
+
+def parse_reference(text: str) -> HubReference:
+    """``text``, ``hf://<org>/<repo>[@<revision>][/<path>]``, as its parts. The revision runs to the next ``/``, so one
+    holding a ``/`` is written percent-encoded, as the Hub's own ``hf://`` paths take it (``@refs%2Fpr%2F3``). Raises
+    BundleError."""
+    org, _, rest = text.removeprefix(SCHEME).partition("/")
+    name, _, path = rest.partition("/")
+    name, at, revision = name.partition("@")
+    parts = [part for part in path.split("/") if part]
+    if not (_NAME.fullmatch(org) and _NAME.fullmatch(name)) or (at and not revision):
+        raise BundleError([f"{text}: name a Hub dataset's bundle as {_FORM}"])
+    if any(part in (".", "..") for part in parts):
+        raise BundleError([f"{text}: the path in the repo can't hold '.' or '..'"])
+    return HubReference(f"{org}/{name}", unquote(revision) or None, "/".join(parts) or None)
+
+
+def cache_root() -> Path:
+    """Where downloaded commits are kept: ``$XDG_CACHE_HOME/agentenv/hf`` when that is an absolute path, else
+    ``~/.cache/agentenv/hf``."""
+    base = os.environ.get("XDG_CACHE_HOME", "")
+    return (Path(base) if os.path.isabs(base) else Path.home() / ".cache") / "agentenv" / "hf"
+
+
+def fetch_bundle(text: str, *, on_progress: Callable[[str], None] | None = None) -> HubBundle:
+    """The bundle ``text`` names, downloaded unless this machine has it. ``on_progress`` is given the line that says
+    which repo, commit and bundle it is, before anything is downloaded. Raises BundleError."""
+    reference = parse_reference(text)
+    try:
+        return _fetch(reference, HfApi(), on_progress or (lambda line: None))
+    except httpx.HTTPError as e:
+        error = BundleError([f"{text}: {str(e).strip() or type(e).__name__}"])
+        if isinstance(e, RepositoryNotFoundError):
+            error.add_note("a private or gated repo needs a token with access: log in with `hf auth login`, or set "
+                           "HF_TOKEN")
+        raise error from e
+
+
+def describe(source: Mapping[str, str]) -> str:
+    """Where a bundle came from, as a run prints it: ``<org>/<repo> at <sha12>, bundle <path>``."""
+    return f"{source['repo']} at {source['sha'][:12]}, bundle {source['path'] or 'at its root'}"
+
+
+def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> HubBundle:
+    sha = reference.revision if reference.revision and _SHA.fullmatch(reference.revision) else api.repo_info(
+        reference.repo, repo_type="dataset", revision=reference.revision).sha
+    folder = cache_root() / reference.repo / sha
+    commit = _Commit.read(folder)
+    if commit is None:
+        commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha))
+        if CARD in commit.files:
+            api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder)
+        commit.save(folder)
+    card = _card(folder)
+    path = _pick(reference, sha, commit.files, card)
+    id_root = _id_root(reference, path)
+    source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, "path": path}
+    say(describe(source))
+    if path not in commit.downloaded:
+        api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder,
+                              allow_patterns=_patterns(path))
+        commit.downloaded.append(path)
+        commit.save(folder)
+    name = path.rpartition("/")[2] or reference.repo.partition("/")[2]
+    needs = (card.get("bundles") or {}).get(name) or {}
+    declared = _within(path, BUNDLE_TOML) in commit.files
+    return HubBundle(folder / path, id_root, source, plugins=() if declared else tuple(needs.get("plugins") or ()),
+                     setup=needs.get("setup"))
+
+
+def _card(folder: Path) -> dict[str, Any]:
+    """The ``agentenv`` table of the card's metadata, ``{}`` without one."""
+    readme = folder / CARD
+    table = (metadata_load(readme) or {}).get("agentenv") if readme.is_file() else None
+    return table if isinstance(table, dict) else {}
+
+
+def _pick(reference: HubReference, sha: str, files: Sequence[str], card: Mapping[str, Any]) -> str:
+    """The folder in the repo of the bundle ``reference`` names, ``""`` for the root: its path; else the only bundle
+    under ``bundles/``, or the one the card names its default; else the root when it is a bundle. ``bundles/`` comes
+    first, since a dataset can keep a ``tasks/`` of data files at its root."""
+    where = f"{reference.repo} at {sha[:12]}"
+    if reference.path is not None:
+        if not _is_bundle(files, reference.path):
+            raise BundleError([f"{where}: {reference.path}/ isn't a bundle: it has none of {_kinds()}"])
+        return reference.path
+    names = sorted({parts[1] for file in files if len(parts := file.split("/")) > 2 and parts[0] == BUNDLES})
+    if len(names) == 1:
+        return _within(BUNDLES, names[0])
+    if card.get("default") in names:
+        return _within(BUNDLES, card["default"])
+    if names:
+        revision = f"@{reference.revision}" if reference.revision else ""
+        runs = ", ".join(f"{SCHEME}{reference.repo}{revision}/{BUNDLES}/{name}" for name in names)
+        raise BundleError([f"{where} holds several bundles under {BUNDLES}/ and its card names no default "
+                           f"(agentenv: default); run one of {runs}"])
+    if _is_bundle(files, ""):
+        return ""
+    raise BundleError([f"{where} holds no bundle: it has no {BUNDLES}/<name>/ folder, and none of {_kinds()} at its "
+                       "root"])
+
+
+def _is_bundle(files: Sequence[str], path: str) -> bool:
+    """Whether the folder at ``path`` in the repo holds a kind folder, as a bundle does."""
+    prefix = _within(path, "")
+    return any(file.startswith(prefix) and file[len(prefix):].split("/", 1)[0] in _KINDS
+               and "/" in file[len(prefix):] for file in files)
+
+
+def _patterns(path: str) -> list[str]:
+    """What to download of the bundle at ``path``: its folder, or for the repo's root its kind folders and files."""
+    if path:
+        return [f"{path}/*"]
+    return [*(f"{kind}/*" for kind in sorted(_KINDS)), BUNDLE_TOML, ALIASES_FILE]
+
+
+def _id_root(reference: HubReference, path: str) -> str:
+    """``@hf/<org>/<repo>``, then the bundle's name for one under ``bundles/``, or its path for one elsewhere."""
+    parts = path.split("/") if path else []
+    if len(parts) == 2 and parts[0] == BUNDLES:
+        parts = parts[1:]
+    id_root = "/".join([f"{HF_PREFIX}{reference.repo}", *parts])
+    try:
+        validate_local_id(id_root)
+    except ValueError as e:
+        raise BundleError([f"{SCHEME}{reference.repo}/{path}: can't be an id root: {e}"]) from None
+    return id_root
+
+
+def _within(folder: str, name: str) -> str:
+    return f"{folder}/{name}" if folder else name
+
+
+def _kinds() -> str:
+    return ", ".join(f"{kind.value}/" for kind in BundleKind)

src/agent_env/bundle/importer.py

@@ -62,7 +62,7 @@
 from agent_env.eval import Eval
 from agent_env.plugins import ARTIFACTS, ENVS, TASK_STEPS
 from agent_env.store.base import NotFoundError
-from agent_env.store.ids import LOCAL_PREFIX
+from agent_env.store.ids import LOCAL_PREFIX, is_local_id
 from agent_env.store.image_store.oci_registry_credentials import names_registry
 from agent_env.store.image_store.registry_api import pin_digest
 from agent_env.task.task import Task
@@ -608,15 +608,15 @@ def _fetch(url: str, dest: Path) -> None:
 
 
 def _output_names(sites: list[RefSite]) -> dict[tuple[EntityKind, str], str]:
-    """The name each output takes: as written, or an ``@local`` id's last segment, numbered when it is taken."""
+    """The name each output takes: as written, or a local id's last segment, numbered when it is taken."""
     written = {_key(site) for site in sites if site.ref.role is RefRole.OUTPUT}
-    taken = {_key(site) for site in sites if _key(site) not in written or not site.value.startswith(LOCAL_PREFIX)}
+    taken = {_key(site) for site in sites if _key(site) not in written or not is_local_id(site.value)}
     names: dict[tuple[EntityKind, str], str] = {}
     for site in sites:
         if site.ref.role is not RefRole.OUTPUT or _key(site) in names:
             continue
         name = site.value
-        if name.startswith(LOCAL_PREFIX):
+        if is_local_id(name):
             base = name = name.rsplit("/", 1)[-1]
             n = 1
             while (site.ref.kind, _nfc(name)) in taken:

src/agent_env/bundle/installed.py

@@ -7,39 +7,70 @@
 
 import difflib
 import os
+import unicodedata
 from collections.abc import Callable
 from pathlib import Path
+from typing import NamedTuple
 
 from packaging.utils import canonicalize_name
 
 from agent_env.plugins._bundles import BUNDLES, CORE, InstalledBundle, installed_bundles
 
+from .hub import HubBundle, fetch_bundle, is_hub_reference
 from .materialize import materialize
 from .parse import Bundle, BundleError, BundleKind, parse_bundle
 from .plan import check_bundle, plan_entities
 from .resolve import resolve_bundle
 
-__all__ = ["BUNDLES", "CORE", "InstalledBundle", "checked", "find_bundle", "installed_bundles", "run_name",
-           "write_installed"]
+__all__ = ["BUNDLES", "CORE", "InstalledBundle", "Located", "checked", "find_bundle", "installed_bundles",
+           "locate_bundle", "run_name", "write_installed"]
+
+
+class Located(NamedTuple):
+    root: Path
+    id_root: str | None  # an installed or Hub bundle's; a folder's ids come from its path
+    hub: HubBundle | None  # what a Hub bundle was downloaded as
 
 
 def locate_bundle(
-    text: str, installed: Callable[[], tuple[InstalledBundle, ...]] = installed_bundles,
-) -> tuple[Path, str | None]:
-    """The folder ``text`` names, and the id root of an installed bundle's, as ``agent-env run BUNDLE`` reads it: an
-    existing folder wins, and text that starts like a path is always one; anything else names a bundle ``installed``
-    lists. Raises BundleError."""
-    if path_like(text) or os.path.isdir(text):
-        return Path(text), None
+    text: str, installed: Callable[[], tuple[InstalledBundle, ...]] = installed_bundles, *,
+    on_progress: Callable[[str], None] | None = None,
+) -> Located:
+    """The folder ``text`` names, as ``agent-env run BUNDLE`` reads it: ``hf://`` names a bundle in a Hugging Face Hub
+    dataset, downloaded first (``on_progress`` is told which); an existing folder spelled exactly as written wins, and
+    text that starts like a path is always one; anything else names a bundle ``installed`` lists. Raises
+    BundleError."""
+    if is_hub_reference(text):
+        hub = fetch_bundle(text, on_progress=on_progress)
+        return Located(hub.root, hub.id_root, hub)
+    if path_like(text) or _spelled_folder(text):
+        return Located(Path(text), None, None)
     found = find_bundle(text, installed())
-    return found.root, found.id_root
+    return Located(found.root, found.id_root, None)
 
 
 def path_like(text: str) -> bool:
     """Text that names a folder even when there is none there."""
     return text.startswith((".", "/", "~"))
 
 
+def _spelled_folder(text: str) -> bool:
+    """Whether ``text`` names a folder each of whose names is spelled as written: on a case-insensitive filesystem
+    ``desktop`` finds ``Desktop``, which isn't the folder meant."""
+    if not os.path.isdir(text):
+        return False
+    above = Path()
+    for part in Path(text).parts:
+        if part not in (".", "..") and _nfc(part) not in {_nfc(name) for name in os.listdir(above)}:
+            return False
+        above /= part
+    return True
+
+
+def _nfc(name: str) -> str:
+    return unicodedata.normalize("NFC", name)
+
+
 def find_bundle(text: str, bundles: tuple[InstalledBundle, ...] | None = None) -> InstalledBundle:
     """The installed bundle ``text`` names, as ``<name>`` or ``<dist>/<name>``. Raises BundleError."""
     bundles = installed_bundles() if bundles is None else bundles