feat(harbor-nemo): add a NeMo registry backend for Harbor

docs/_scripts/lint_python_snippets.py

@@ -51,7 +51,9 @@
     "<!-- @nemo-docs: skip-python-type-check -->",
     "<!-- @nemo-nb: skip-type-check -->",
 }
-DEFAULT_IGNORED_TY_RULES = ("possibly-unbound-attribute",)
+# ``possibly-unbound-attribute`` was renamed upstream; passing the old name makes ty emit
+# ``warning[unknown-rule]``, which fails this check for every doc regardless of its snippets.
+DEFAULT_IGNORED_TY_RULES = ("possibly-missing-attribute",)
 
 
 @dataclass(frozen=True)

docs/evaluator/manage-tasks-tasksets.mdx

@@ -73,47 +73,82 @@ Reference the stored metric with a `MetricRef` (`workspace/name`, or a bare `nam
 the task's workspace). The service returns the stored `Task`.
 
 ```python
-from nemo_evaluator.api.schemas import MetadataItem, MetricRef, TaskInput, TaskInputs
+from nemo_evaluator.api.schemas import EvaluatorTaskDefinition, MetadataItem, MetricRef, TaskInput, TaskInputs
 
 task = TaskInput(
-    intent="Answer the user's geography question with the capital city.",
-    inputs=TaskInputs(instruction="What is the capital of France?"),
-    metrics=[MetricRef("default/answer-exact-match")],
+    spec=EvaluatorTaskDefinition(
+        kind="evaluator",
+        intent="Answer the user's geography question with the capital city.",
+        inputs=TaskInputs(instruction="What is the capital of France?"),
+        metrics=[MetricRef("default/answer-exact-match")],
+    ),
     metadata=[MetadataItem(key="suite", value="geography")],
 )
 
 stored = tasks.create("capital-of-france", task=task)
-print(stored.id, stored.metrics)
+print(stored.id, stored.spec.metrics)
 ```
 
 ### `TaskInput` fields
 
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `spec` | `TaskDefinition` | Yes | The task's content, discriminated by `kind` — see below. |
+| `metadata` | `list[MetadataItem]` | No | Key/value annotations. Keys must be unique. |
+| `tags` | `list[str]` | No | Tags to point at the revision this request publishes. `latest` is always applied server-side. |
+
+### Task kinds
+
+A task is an evaluation unit; its `kind` says which runner executes it. There are two:
+
+- `evaluator` — the task's content is fields you author, scored by platform metrics.
+- `harbor` — the task's content is a packaged directory of files, scored by Harbor's own reward.
+
+Both are stored as the same record type, so a taskset can group them and you manage every evaluation
+unit in one place regardless of which runner executes it.
+
+`EvaluatorTaskDefinition` (`kind="evaluator"`):
+
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
 | `intent` | `str` | Yes | Human-readable description of the desired agent behavior. |
 | `inputs` | `TaskInputs` | No | The task's recognized input fields. `instruction` is the agent's prompt; it falls back to `intent` when unset. |
+| `reference` | `dict[str, Any]` | No | Grader-only ground truth (held-out tests, expected outputs, rubric data). Surfaced to metrics but never seeded into the agent's workspace or shown to the agent. Held out from the *agent*, not from the API. |
 | `metrics` | `list[MetricRefOrInline]` | No | The metrics that score the task, as `MetricRef` references (`workspace/name`) to stored metrics. Pre-built inline metric bundles (`MetricInline`) are also accepted and are normalized to stored metrics on create. |
 | `views` | `dict[str, SemanticView]` | No | Optional reporting views mapping metric outputs into named semantic scores. |
-| `metadata` | `list[MetadataItem]` | No | Key/value annotations. Keys must be unique. |
-| `tags` | `list[str]` | No | Tags to point at the revision this request publishes. `latest` is always applied server-side. |
+
+`HarborTaskDefinition` (`kind="harbor"`):
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `archive_ref` | `str` | Yes | Files reference to the task's packaged directory (`workspace/fileset#path`). One fileset per task, so a task shared by several tasksets is stored once. |
+| `archive_digest` | `str` | Yes | Content hash Harbor computed over the task directory. |
+| `instruction` | `str` | No | The task's instruction text, when it has one. |
+| `config` | `dict` | No | Harbor's own task configuration (verifier, agent, environment, steps), stored as published. |
+
+<Note>
+Storing a Harbor task is supported; **running one from storage is not yet**. A taskset may group both
+kinds, but expanding a `harbor` member is rejected with `422` before the run starts, whatever target
+you submit against. Harbor evaluations continue to run through the existing dataset-driven path.
+</Note>
 
 <Note>
 A stored task holds **metric references only**. Any inline metric bundle you pass on create is stored
 as a content-addressed *derived* metric, and the task record is normalized to reference it. This is
-why `stored.metrics` always comes back as a list of `MetricRef` references.
+why `stored.spec.metrics` always comes back as a list of `MetricRef` references.
 </Note>
 
 ### Retrieve, list, and delete
 
 ```python
 # Retrieve one task by name (its current content)
 task = tasks.retrieve("capital-of-france")
-print(task.revision, task.tags)  # e.g. 1 {'latest': 1}
+print(task.spec.kind, task.revision, task.tags)  # e.g. evaluator 1 {'latest': 1}
 
 # List tasks in the workspace (paginated)
 page = tasks.list(page=1, page_size=100, sort="-created_at")
 for item in page.data:
-    print(item.name, item.intent)
+    print(item.name, item.spec.kind)
 
 # Delete a task (this also removes all of its revisions)
 tasks.delete("capital-of-france")
@@ -131,9 +166,12 @@ no existence check.
 
 ```python
 revised_task = TaskInput(
-    intent="Answer the user's geography question with the capital city.",
-    inputs=TaskInputs(instruction="Name the capital city of France."),
-    metrics=[MetricRef("default/answer-exact-match")],
+    spec=EvaluatorTaskDefinition(
+        kind="evaluator",
+        intent="Answer the user's geography question with the capital city.",
+        inputs=TaskInputs(instruction="Name the capital city of France."),
+        metrics=[MetricRef("default/answer-exact-match")],
+    ),
     metadata=[MetadataItem(key="suite", value="geography")],
 )
 
@@ -169,7 +207,7 @@ original = tasks.retrieve("capital-of-france", revision=digest)  # revision 1, a
 current = tasks.retrieve("capital-of-france")                    # revision 2, the current content
 
 assert original.revision == 1 and current.revision == 2
-assert original.inputs.instruction != current.inputs.instruction
+assert original.spec.inputs.instruction != current.spec.inputs.instruction
 ```
 
 ### Tag a revision
@@ -361,9 +399,10 @@ A fragment that no longer resolves fails the evaluation rather than falling back
 revision.
 
 <Note>
-Stored tasks carry no grader-only `reference` (held-out ground truth): that field lives only on inline
-`AgentEvalTaskInput`. Taskset-driven tasks therefore run with an empty `reference`, so use a taskset
-when your metrics score the agent's output directly rather than against per-task held-out data.
+A member's grader-only `reference` (held-out ground truth) is loaded from the pinned revision along
+with the rest of its content, so a taskset-driven run grades against the ground truth that revision
+fixed. Because `reference` is covered by the revision digest, changing it publishes a new revision —
+a pin fixes the grading, not just the prompt.
 </Note>
 
 ## Async usage

packages/harbor_nemo/README.md

@@ -0,0 +1,93 @@
+# harbor-nemo
+
+A [Harbor](https://github.com/harbor-framework/harbor) registry backend that publishes to and
+runs from **NeMo Platform** instead of the public Harbor Hub, with no changes to Harbor.
+
+```bash
+pip install -e packages/harbor_nemo
+
+export HARBOR_REGISTRY_BACKEND=nemo
+export NMP_BASE_URL=http://localhost:8080
+
+harbor publish ./my-task
+harbor download nvidia/my-task -o ./out
+harbor run -t nvidia/my-task --agent nop
+```
+
+Installing the package registers `nemo` under the `harbor.registry_backends` entry point.
+That is the whole integration: the stock CLI resolves the backend by name at call time.
+
+## How Harbor concepts map onto NeMo
+
+| Harbor | NeMo |
+|---|---|
+| task package `org/name` | task entity `org.name` (`kind="harbor"`), one workspace |
+| task version | a published *revision* of that entity |
+| task archive (`dist.tar.gz`) | a file in the `harbor-packages` fileset |
+| content hash | `spec.archive_digest` |
+| dataset `org/name` | taskset entity `org.name` |
+| dataset-level files | a JSON blob in taskset `metadata` (see *Known gaps*) |
+| tags (`latest`, …) | revision tags |
+
+**The org is folded into the entity name.** A NeMo workspace is a tenancy boundary with its
+own lifecycle and authorization; a Harbor org is a cheap, self-serve namespace that
+`harbor publish` creates on demand. Mapping org to workspace would make publishing a tenancy
+operation. The cost is that the org prefix is a convention, not an enforced boundary.
+
+## Configuration
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `NMP_BASE_URL` | `http://localhost:8080` | platform to publish to / read from |
+| `HARBOR_NEMO_WORKSPACE` / `NMP_WORKSPACE` | `default` | workspace holding tasks and tasksets |
+| `HARBOR_NEMO_FILESET` | `harbor-packages` | fileset holding package archives |
+| `NMP_TOKEN` / `NMP_API_KEY` | — | bearer token, when the platform has auth enabled |
+| `HARBOR_NEMO_TIMEOUT_SEC` | `120` | HTTP timeout |
+
+Set `HARBOR_REGISTRY_WEBSITE_URL` too: `harbor publish` prints a hub URL from a Harbor-side
+constant, so without it the CLI advertises `hub.harborframework.com` for NeMo packages.
+
+## Two digests, and why it matters
+
+NeMo addresses a revision by a digest of the revision's *content* (canonical JSON of the
+stored spec). Harbor addresses a version by a digest of the task *directory's files*. They are
+different hashes of different things, and both are live:
+
+- `ResolvedTaskVersion.content_hash` carries **Harbor's**, because Harbor's download cache is
+  keyed on it.
+- A `sha256:` reference reaching `resolve_version` is always **Harbor's**, and is *not* a
+  valid NeMo revision selector — the platform returns 404 for it. Resolving one is a scan
+  over revisions comparing `spec.archive_digest`, not a direct fetch.
+- A **revision ordinal** is not a valid selector either: the platform reads any non-digest
+  fragment as a *tag name*, so `/revisions/2` looks for a tag called `"2"`. Ordinals are
+  translated to that revision's content hash first.
+- NeMo digests are **bare hex**, deliberately, so a `#` fragment stays free of `:` — which the
+  entity-ref charset does not admit and the route's path pattern rejects with a 422. Harbor's
+  `sha256:` prefix is stripped before any digest is used as a selector.
+
+Publishing a dataset translates between the two spaces: a Harbor manifest pins members by
+archive digest, a taskset pins by revision digest, so each member costs one lookup. This is
+not optional — the taskset service re-resolves bare member refs at write time, so an
+unpinned member would silently pin whatever was `latest` at publish, not what the manifest
+named.
+
+## Known gaps
+
+- **Dataset-level files ride in taskset `metadata`** as a JSON string, because a taskset has
+  no file-reference field. A taskset-level file reference would replace this.
+- **No yank support.** `ResolvedTaskVersion.yanked_at` is always `None`; NeMo has no
+  equivalent.
+- **`record_download` is a deliberate no-op.** NeMo has no counter primitive, so implementing
+  it would mean a read-modify-write on the hottest entity per package for best-effort
+  telemetry.
+- **`harbor version list|show|tag` is Supabase-pinned** in Harbor itself and will show Hub
+  data regardless of `HARBOR_REGISTRY_BACKEND`.
+
+## Requirements
+
+Needs a NeMo Platform with the `entities`, `files`, and `evaluator` services, and the
+`kind="harbor"` task definition from nemo-platform PR #1071.
+
+```bash
+uv run nemo services run --services entities,files,evaluator --port 8080
+```

packages/harbor_nemo/pyproject.toml

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Deliberately NOT a member of the root `[tool.uv.workspace]`. This package depends on
+# `harbor`, which the platform keeps as a marker-gated optional extra rather than a default
+# dependency; listing it as a workspace member would pull harbor into every bare
+# `uv sync --all-packages`. Install it explicitly instead:
+#
+#     uv pip install -e packages/harbor_nemo
+[project]
+name = "harbor-nemo"
+version = "0.1.0"
+description = "NeMo Platform registry backend for Harbor: publish and run Harbor packages against NeMo."
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+
+dependencies = [
+  "harbor>=0.20.0",
+  "httpx>=0.27",
+  "pydantic>=2.7",
+]
+
+# This is what makes the stock `harbor` CLI find the backend: with the package installed,
+# `HARBOR_REGISTRY_BACKEND=nemo` resolves through here. The value is a zero-argument callable
+# returning a BaseRegistryBackend, loaded lazily so importing harbor stays cheap.
+[project.entry-points."harbor.registry_backends"]
+nemo = "harbor_nemo:load_backend"
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-asyncio>=0.23", "respx>=0.21"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/harbor_nemo"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

packages/harbor_nemo/src/harbor_nemo/__init__.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""A NeMo Platform registry backend for Harbor.
+
+Installing this package registers ``nemo`` under the ``harbor.registry_backends`` entry
+point, so the stock Harbor CLI publishes to and runs from NeMo with no change to Harbor::
+
+    export HARBOR_REGISTRY_BACKEND=nemo
+    harbor publish ./my-task
+    harbor run -d nvidia/my-dataset
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from harbor_nemo.backend import NemoRegistryBackend
+
+__all__ = ["load_backend"]
+
+
+def load_backend() -> "NemoRegistryBackend":
+    """Entry point target: build the backend.
+
+    Imports inside the function rather than at module scope because Harbor resolves entry
+    points lazily and only for a backend that was actually selected. A module-level import
+    would pull httpx and every Harbor publisher model into any process that merely *lists*
+    installed backends — including one using the default Supabase backend.
+    """
+    from harbor_nemo.backend import NemoRegistryBackend
+
+    return NemoRegistryBackend()