feat(harness-bench): bind any registered harness by name (ACP + community plugins)

tests/harness_bench/driver.py

@@ -113,6 +113,14 @@ class ProvisioningError(RuntimeError):
     "provider auth command",
     "empty token",
     "Failed to resolve external API key auth",
+    # Own-auth harness whose vendor CLI is not installed / not logged in (e.g.
+    # an ACP harness like rovo: "Ensure `acli` is installed and you are logged
+    # in"). The vendor process exits before a turn can run — an environment/
+    # login gap, not a capability the harness lacks, so it must SKIP not drift.
+    "are logged in",
+    "AcpProcessExited",
+    "ACP subprocess",
+    "ACP session",
 )
 
 
@@ -160,6 +168,14 @@ def infra_failure_reason(result: TurnResult) -> str | None:
             "gateway/provider token could not be provisioned for this transport "
             "(environment/auth gap, not a capability the harness lacks)"
         )
+    if any(
+        marker in text
+        for marker in ("are logged in", "AcpProcessExited", "ACP subprocess", "ACP session")
+    ):
+        return (
+            "vendor CLI not installed or not logged in (own-auth harness); "
+            "the agent process exited before a turn could run"
+        )
     if "unexpected status" in text:
         return "gateway returned an unexpected status (environment/auth issue)"
     return "environment/connectivity error reaching the gateway"

tests/harness_bench/full_server.py

@@ -166,7 +166,11 @@ def _build_bench_agent_config(
 ) -> dict[str, Any]:
     """The agent spec for a bench harness: the harness + the read-only builtin,
     plus (when *deny*) a baked tool_call-phase deny on that builtin."""
-    name = f"bench-{profile.harness}" + ("-deny" if deny else "")
+    # The agent name must match [a-zA-Z0-9_-]+, but a harness id can contain a
+    # colon (acp:<slug>). Sanitize it for the name only; config.harness keeps
+    # the real id so the runner resolves the right ACP agent at spawn.
+    safe_harness = profile.harness.replace(":", "-")
+    name = f"bench-{safe_harness}" + ("-deny" if deny else "")
     config: dict[str, Any] = {
         "spec_version": 1,
         "name": name,

tests/harness_bench/manifest.py

@@ -36,7 +36,14 @@
 
 from omnigent.harness_aliases import is_native_harness
 from omnigent.harness_capabilities import AuthModel, HarnessCapabilities, IntegrationMode
-from omnigent.harness_plugins import harness_capabilities, model_env_keys
+from omnigent.harness_plugins import (
+    harness_aliases,
+    harness_capabilities,
+    harness_install_keys,
+    harness_modules,
+    install_specs,
+    model_env_keys,
+)
 from tests.e2e._harness_probes import HARNESS_PROBES, HarnessProbe
 from tests.harness_bench.profile import BenchProfile
 from tests.harness_bench.verdict import Verdict
@@ -232,4 +239,118 @@ def _native_tui_harnesses() -> list[str]:
     OFFICIAL_PROFILES[_h] = _native_profile(_h)
 
 
+# ── registry fallback: build a profile for ANY registered harness ─
+#
+# resolve_profile uses this so a harness passed by name (--harness acp,
+# --harness rovo) is runnable even though it is not an official profile and
+# ships no BenchProfile of its own. It covers the harnesses the auto-derivation
+# above misses: ACP / CLI-subprocess harnesses (in the capability model but not
+# NATIVE_TUI, e.g. the in-repo `acp`), and community-plugin harnesses that
+# register via entry point but declare no capabilities entry (e.g. `rovo-cli`
+# from omnigent-rovo, discovered through harness_modules()).
+
+# integration_mode -> bench transport family. SDK / CLI / ACP subprocess
+# harnesses all run through the SDK-wrap drivers (registered as an omnigent
+# agent, driven over the session HTTP surface); only NATIVE_TUI needs the
+# native driver. A harness with no capabilities entry defaults to the SDK
+# family (the common case for a plain subprocess plugin like rovo).
+_INTEGRATION_MODE_TRANSPORT: dict[IntegrationMode, str] = {
+    IntegrationMode.SDK_IN_PROCESS: "sdk-inproc",
+    IntegrationMode.CLI_SUBPROCESS: "sdk-inproc",
+    IntegrationMode.ACP_SUBPROCESS: "sdk-inproc",
+    IntegrationMode.NATIVE_TUI: "native-tui",
+}
+
+
+def _registry_cli_binary(canonical: str) -> str | None:
+    """The vendor binary to skip-gate on, from the harness's install spec.
+
+    e.g. rovo-cli -> ``acli`` (Atlassian CLI). ``None`` when the harness has no
+    install spec (e.g. the generic ``acp`` harness, whose command is supplied
+    at run time via ``HARNESS_ACP_COMMAND``), in which case there is no cheap
+    pre-flight gate and an unrunnable harness skips on the turn instead.
+    """
+    install_key = harness_install_keys().get(canonical)
+    spec = install_specs().get(install_key) if install_key else None
+    return getattr(spec, "binary", None)
+
+
+def _registry_profile(name: str) -> BenchProfile | None:
+    """Build a :class:`BenchProfile` for a harness known to the omnigent registry.
+
+    Resolves aliases (``rovo`` -> ``rovo-cli``) and requires the canonical name
+    to be a registered harness (``harness_modules()``); returns ``None`` for an
+    unknown name so :func:`resolve_profile` can fall through to its error. The
+    transport family comes from the capability model's ``integration_mode``
+    (defaulting to the SDK family when a plugin declares no capabilities), so
+    the harness runs on the existing drivers with no bench edit.
+    """
+    canonical = harness_aliases().get(name, name)
+    # ``acp:<slug>`` is a first-class harness id: the base ``acp`` harness is
+    # registered, and the slug selects a user-configured ACP agent at spawn
+    # (resolved from the ~/.omnigent config ``acp:`` block, see
+    # onboarding/acp_auth.py). Look up caps/module by the base ``acp`` but keep
+    # the full id as the profile harness so ``config.harness=acp:<slug>`` reaches
+    # the runner. Lets ``--harness acp:qwen`` bind to a specific ACP agent.
+    if canonical.startswith("acp:"):
+        if not canonical[len("acp:") :]:
+            return None  # empty slug ("acp:") — use bare "acp" instead
+        registry_key = "acp"
+    else:
+        registry_key = canonical
+    if registry_key not in harness_modules():
+        return None
+
+    caps = harness_capabilities().get(registry_key)
+    mode = caps.integration_mode if caps is not None else None
+    if mode is None:
+        # No capabilities entry (a plain subprocess plugin like rovo): the bench
+        # has no modeled transport, so assume the SDK-wrap family — the only
+        # thing a registered-but-unmodeled harness can plausibly run on.
+        transport = "sdk-inproc"
+    elif mode in _INTEGRATION_MODE_TRANSPORT:
+        transport = _INTEGRATION_MODE_TRANSPORT[mode]
+    else:
+        # A MODELED mode the bench has no driver for (e.g. NATIVE_SERVER /
+        # opencode-native). Refuse rather than silently degrade to the SDK
+        # family: return None so resolve_profile raises a clean KeyError. A
+        # bare "default to sdk-inproc" here would bind a native-server harness
+        # to the wrong driver and drop its skip-gate.
+        return None
+
+    if transport == "native-tui":
+        # A native-tui harness the auto-derivation would already cover; reuse
+        # its builder so the two paths agree.
+        return _native_profile(canonical)
+
+    # env_prefix / marker sanitize non-word chars (an acp:<slug> id has a colon)
+    # to a valid env-var stem, e.g. acp:qwen -> HARNESS_ACP_QWEN_.
+    stem = canonical.upper().replace("-", "_").replace(":", "_")
+    env_prefix = "HARNESS_" + stem + "_"
+    marker = stem + "_OK"
+    # A model is always required: the omnigent executor spec mandates one
+    # (spec/omnigent.py: "executor.type='omnigent' requires a model"), so an
+    # empty model fails agent registration ("llm.model must be present"). For an
+    # own-auth harness (e.g. ACP/rovo) the value is inert — the runner drops any
+    # databricks-* gateway model for it (ACP: workflow.py::_build_acp_spawn_env)
+    # and the agent authenticates + picks its own model — but it must still be a
+    # valid non-empty id to register. So stamp the databricks default in all
+    # cases: real for a gateway harness, an accepted-but-ignored placeholder for
+    # an own-auth one.
+    return BenchProfile(
+        harness=canonical,
+        model=_NATIVE_DEFAULT_MODEL,
+        env_prefix=env_prefix,
+        marker=marker,
+        # install-spec + declared caps are keyed by the base harness, not the
+        # acp:<slug> id, so look them up by registry_key.
+        cli_binary=_registry_cli_binary(registry_key),
+        transport=transport,
+        owner="",
+        auth=_auth_prose(caps),
+        implementation=_implementation_prose(caps),
+        declared=_declared_from_capabilities(registry_key),
+    )
+
+
 __all__ = ["OFFICIAL_PROFILES"]

tests/harness_bench/profile.py

@@ -74,25 +74,29 @@ def declared_for(self, probe_name: str) -> Verdict:
 def resolve_profile(name: str) -> BenchProfile:
     """Resolve a harness name to a :class:`BenchProfile`.
 
-    Resolution chain (option B in the design doc):
+    Resolution chain:
 
     1. An official harness in :mod:`tests.harness_bench.manifest`.
     2. A community harness that ships a profile: *name* is a dotted path
        to either a ``BenchProfile`` instance or a zero-arg
        ``bench_profile()`` factory (e.g.
        ``mypkg.myharness:bench_profile`` or ``mypkg.myharness.PROFILE``).
+    3. Any harness registered in the omnigent registry (in-repo or an
+       entry-point plugin), resolved by name / alias — the profile is
+       derived from the capability model (see
+       :func:`tests.harness_bench.manifest._registry_profile`). This is what
+       lets ``--harness acp`` or ``--harness rovo`` run with no bench edit.
 
     This keeps the official list a convenience index, not a gate: any
-    harness exposing a profile is probeable with ``--harness <path>`` and
-    no bench edits. When per-harness self-registration lands, step 1 swaps
-    from a static dict to dynamic discovery with no change here.
+    registered harness is probeable by name, and any out-of-tree one that
+    ships a ``BenchProfile`` is probeable by reference.
 
-    :param name: Official harness name or a dotted path / ``module:attr``.
+    :param name: Harness name / alias, or a dotted path / ``module:attr``.
     :returns: The resolved profile.
-    :raises KeyError: If *name* is neither official nor an importable
+    :raises KeyError: If *name* is not a registered harness nor an importable
         profile reference.
     """
-    from tests.harness_bench.manifest import OFFICIAL_PROFILES
+    from tests.harness_bench.manifest import OFFICIAL_PROFILES, _registry_profile
 
     if name in OFFICIAL_PROFILES:
         return OFFICIAL_PROFILES[name]
@@ -101,10 +105,14 @@ def resolve_profile(name: str) -> BenchProfile:
     if resolved is not None:
         return resolved
 
+    from_registry = _registry_profile(name)
+    if from_registry is not None:
+        return from_registry
+
     raise KeyError(
-        f"unknown harness {name!r}: not an official harness "
-        f"({', '.join(sorted(OFFICIAL_PROFILES))}) and not an importable "
-        f"BenchProfile reference (try 'module:attr' or 'module.ATTR')"
+        f"unknown harness {name!r}: not a registered omnigent harness, not an "
+        f"official profile ({', '.join(sorted(OFFICIAL_PROFILES))}), and not an "
+        f"importable BenchProfile reference (try 'module:attr' or 'module.ATTR')"
     )
 
 

tests/harness_bench/test_bench.py

@@ -105,6 +105,122 @@ def test_resolve_official_and_community_and_unknown() -> None:
         resolve_profile("no-such-harness")
 
 
+def test_resolve_registered_harness_by_name() -> None:
+    """A registered harness with no official profile is resolvable by name.
+
+    This is the "plugs in with no bench edit" path: an in-repo ACP/CLI-subprocess
+    harness (not auto-derived, since that is native-tui-only) resolves via the
+    registry fallback, deriving a profile from the capability model. ACP is an
+    ACP_SUBPROCESS harness, so it lands on the SDK-wrap driver family
+    (transport "sdk-inproc"), not native-tui.
+    """
+    from omnigent.harness_plugins import harness_modules
+
+    if "acp" not in harness_modules():
+        pytest.skip("acp harness not registered in this build")
+    profile = resolve_profile("acp")
+    assert profile.harness == "acp"
+    assert profile.transport == "sdk-inproc"
+
+    # acp:<slug> is a first-class id: base `acp` is registered and the slug
+    # selects a configured ACP agent at spawn. It binds, keeping the full id as
+    # the harness (so config.harness=acp:qwen reaches the runner) and a valid
+    # env-prefix stem. An empty slug ("acp:") is refused.
+    slug = resolve_profile("acp:qwen")
+    assert slug.harness == "acp:qwen"
+    assert slug.transport == "sdk-inproc"
+    assert slug.env_prefix == "HARNESS_ACP_QWEN_"
+    with pytest.raises(KeyError):
+        resolve_profile("acp:")
+
+
+def test_resolve_entry_point_plugin_and_alias() -> None:
+    """An entry-point community plugin resolves by name AND by alias.
+
+    ``omnigent-rovo`` registers ``rovo-cli`` (alias ``rovo``) via the
+    ``omnigent.community.harness`` entry point and declares no capabilities
+    entry — only a harness module + install spec. The registry fallback still
+    binds it (keying off harness_modules, defaulting to the SDK family) and
+    skip-gates on its install-spec binary. Gated on the plugin being installed
+    so a build without it still passes.
+    """
+    from omnigent.harness_plugins import harness_aliases
+
+    if harness_aliases().get("rovo") != "rovo-cli":
+        pytest.skip("omnigent-rovo plugin not installed")
+    by_alias = resolve_profile("rovo")
+    by_name = resolve_profile("rovo-cli")
+    assert by_alias.harness == "rovo-cli" == by_name.harness
+    assert by_alias.transport == "sdk-inproc"
+    assert by_alias.cli_binary == "acli"  # skip-gates on the Atlassian CLI
+    # A model is always stamped (agent registration requires a non-empty one);
+    # it is inert for an own-auth harness, which uses its own.
+    assert by_alias.model
+
+
+def test_registry_profile_happy_path_no_plugin(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The registry fallback's positive path, independent of any optional plugin.
+
+    Fakes a registered CLI-subprocess harness (+ alias + install-spec binary) so
+    the name/alias resolution, the integration_mode -> sdk-inproc mapping, and
+    the install-spec skip-gate are exercised even in a build without
+    omnigent-rovo. Guards the coverage the plugin tests skip-gate away.
+    """
+    from types import SimpleNamespace
+
+    import tests.harness_bench.manifest as man
+    from omnigent.harness_capabilities import AuthModel, IntegrationMode
+
+    class _Spec:
+        binary = "fakebin"
+
+    # A lightweight caps stand-in: _registry_profile + the prose/declared helpers
+    # only read integration_mode / auth / streaming / interrupt.
+    caps = SimpleNamespace(
+        integration_mode=IntegrationMode.CLI_SUBPROCESS,
+        auth=AuthModel.OWN_AUTH,
+        streaming=True,
+        interrupt=True,
+    )
+    monkeypatch.setattr(man, "harness_modules", lambda: {"fake-cli": "pkg.fake"})
+    monkeypatch.setattr(man, "harness_aliases", lambda: {"fake": "fake-cli"})
+    monkeypatch.setattr(man, "harness_capabilities", lambda: {"fake-cli": caps})
+    monkeypatch.setattr(man, "harness_install_keys", lambda: {"fake-cli": "fake"})
+    monkeypatch.setattr(man, "install_specs", lambda: {"fake": _Spec()})
+
+    for name in ("fake-cli", "fake"):
+        p = man._registry_profile(name)
+        assert p is not None and p.harness == "fake-cli"
+        assert p.transport == "sdk-inproc"
+        assert p.cli_binary == "fakebin"
+        assert p.model  # always non-empty (agent registration requires a model)
+
+
+def test_registry_refuses_native_server_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A MODELED mode the bench has no driver for is refused, not mis-bound.
+
+    NATIVE_SERVER (e.g. opencode-native) has no bench driver. The fallback must
+    return None (-> resolve_profile KeyError) rather than silently degrade to
+    the sdk-inproc default, which would bind a vendor-server harness to the SDK
+    drivers and drop its skip-gate.
+    """
+    from types import SimpleNamespace
+
+    import tests.harness_bench.manifest as man
+    from omnigent.harness_capabilities import AuthModel, IntegrationMode
+
+    caps = SimpleNamespace(
+        integration_mode=IntegrationMode.NATIVE_SERVER,
+        auth=AuthModel.OWN_AUTH,
+        streaming=False,
+        interrupt=False,
+    )
+    monkeypatch.setattr(man, "harness_modules", lambda: {"srv": "pkg.srv"})
+    monkeypatch.setattr(man, "harness_aliases", dict)
+    monkeypatch.setattr(man, "harness_capabilities", lambda: {"srv": caps})
+    assert man._registry_profile("srv") is None
+
+
 def test_infra_failure_reason_classifies_auth_and_ignores_capability_gaps() -> None:
     from tests.harness_bench.driver import TurnResult, infra_failure_reason