From e3e01c1bd773861a8d6d221ab5921e841f8cb137 Mon Sep 17 00:00:00 2001 From: Pat Sukprasert Date: Thu, 9 Jul 2026 13:18:11 +0800 Subject: [PATCH 1/5] feat(harness-bench): bind any registered harness passed by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench could only probe an official profile (the 4 SDK harnesses + auto-derived native-tui) or a dotted module:attr BenchProfile reference. A harness registered in the omnigent registry but neither official nor native-tui -- the in-repo generic ACP harness (`acp`, ACP_SUBPROCESS), or an entry-point community plugin (`rovo`/`rovo-cli` from omnigent-rovo) -- KeyError'd on resolve_profile, so `--harness acp` / `--harness rovo` could not run. Add a registry fallback to resolve_profile: after the official + reference checks, derive a BenchProfile for any harness in the omnigent registry (_registry_profile in manifest.py). It resolves aliases (rovo -> rovo-cli), keys off harness_modules() so it covers plugins that declare no capabilities entry, maps integration_mode -> transport family (SDK/CLI/ACP subprocess -> sdk-inproc family = the existing drivers; NATIVE_TUI -> native-tui), and skip-gates on the harness's install-spec binary when present (rovo -> acli). No new transport driver: an ACP harness registers as an omnigent agent (config.harness=acp:) and runs on the existing SDK-wrap drivers. Both harnesses are OWN_AUTH, so they run only where their vendor binary is installed + authed, and skip cleanly otherwise (verified live: rovo skips on missing `acli`). tool_calling/policy_deny stay `·` for ACP (agent runs its own tools / gates via session/request_permission) -- the same documented gap as native. Tests: resolve_profile binds acp (sdk-inproc) and rovo/rovo-cli (alias, acli gate); unknown still KeyErrors; plugin cases skip if omnigent-rovo absent. Offline suite 71 passed / 18 skipped, ruff clean. --- tests/harness_bench/manifest.py | 87 ++++++++++++++++++++++++++++++- tests/harness_bench/profile.py | 28 ++++++---- tests/harness_bench/test_bench.py | 39 ++++++++++++++ 3 files changed, 143 insertions(+), 11 deletions(-) diff --git a/tests/harness_bench/manifest.py b/tests/harness_bench/manifest.py index e318e9a4a0d..bf5458e2195 100644 --- a/tests/harness_bench/manifest.py +++ b/tests/harness_bench/manifest.py @@ -36,7 +36,14 @@ from omnigent.harness_aliases import is_native_harness from omnigent.harness_capabilities import AuthModel, HarnessCapabilities, IntegrationMode -from omnigent.harness_plugins import harness_capabilities, model_env_keys +from omnigent.harness_plugins import ( + harness_aliases, + harness_capabilities, + harness_install_keys, + harness_modules, + install_specs, + model_env_keys, +) from tests.e2e._harness_probes import HARNESS_PROBES, HarnessProbe from tests.harness_bench.profile import BenchProfile from tests.harness_bench.verdict import Verdict @@ -232,4 +239,82 @@ def _native_tui_harnesses() -> list[str]: OFFICIAL_PROFILES[_h] = _native_profile(_h) +# ── registry fallback: build a profile for ANY registered harness ─ +# +# resolve_profile uses this so a harness passed by name (--harness acp, +# --harness rovo) is runnable even though it is not an official profile and +# ships no BenchProfile of its own. It covers the harnesses the auto-derivation +# above misses: ACP / CLI-subprocess harnesses (in the capability model but not +# NATIVE_TUI, e.g. the in-repo `acp`), and community-plugin harnesses that +# register via entry point but declare no capabilities entry (e.g. `rovo-cli` +# from omnigent-rovo, discovered through harness_modules()). + +# integration_mode -> bench transport family. SDK / CLI / ACP subprocess +# harnesses all run through the SDK-wrap drivers (registered as an omnigent +# agent, driven over the session HTTP surface); only NATIVE_TUI needs the +# native driver. A harness with no capabilities entry defaults to the SDK +# family (the common case for a plain subprocess plugin like rovo). +_INTEGRATION_MODE_TRANSPORT: dict[IntegrationMode, str] = { + IntegrationMode.SDK_IN_PROCESS: "sdk-inproc", + IntegrationMode.CLI_SUBPROCESS: "sdk-inproc", + IntegrationMode.ACP_SUBPROCESS: "sdk-inproc", + IntegrationMode.NATIVE_TUI: "native-tui", +} + + +def _registry_cli_binary(canonical: str) -> str | None: + """The vendor binary to skip-gate on, from the harness's install spec. + + e.g. rovo-cli -> ``acli`` (Atlassian CLI). ``None`` when the harness has no + install spec (e.g. the generic ``acp`` harness, whose command is supplied + at run time via ``HARNESS_ACP_COMMAND``), in which case there is no cheap + pre-flight gate and an unrunnable harness skips on the turn instead. + """ + install_key = harness_install_keys().get(canonical) + spec = install_specs().get(install_key) if install_key else None + return getattr(spec, "binary", None) + + +def _registry_profile(name: str) -> BenchProfile | None: + """Build a :class:`BenchProfile` for a harness known to the omnigent registry. + + Resolves aliases (``rovo`` -> ``rovo-cli``) and requires the canonical name + to be a registered harness (``harness_modules()``); returns ``None`` for an + unknown name so :func:`resolve_profile` can fall through to its error. The + transport family comes from the capability model's ``integration_mode`` + (defaulting to the SDK family when a plugin declares no capabilities), so + the harness runs on the existing drivers with no bench edit. + """ + canonical = harness_aliases().get(name, name) + if canonical not in harness_modules(): + return None + + caps = harness_capabilities().get(canonical) + mode = caps.integration_mode if caps is not None else None + transport = _INTEGRATION_MODE_TRANSPORT.get(mode, "sdk-inproc") if mode else "sdk-inproc" + + if transport == "native-tui": + # A native-tui harness the auto-derivation would already cover; reuse + # its builder so the two paths agree. + return _native_profile(canonical) + + env_prefix = "HARNESS_" + canonical.upper().replace("-", "_") + "_" + marker = canonical.upper().replace("-", "_") + "_OK" + return BenchProfile( + harness=canonical, + # An own-auth subprocess harness owns its model (the gateway model id is + # dropped for it), so this is a placeholder the harness ignores; a + # gateway-routed one would take a real databricks-* id. + model=_NATIVE_DEFAULT_MODEL, + env_prefix=env_prefix, + marker=marker, + cli_binary=_registry_cli_binary(canonical), + transport=transport, + owner="", + auth=_auth_prose(caps), + implementation=_implementation_prose(caps), + declared=_declared_from_capabilities(canonical), + ) + + __all__ = ["OFFICIAL_PROFILES"] diff --git a/tests/harness_bench/profile.py b/tests/harness_bench/profile.py index 4652f8f5563..08f62f0404e 100644 --- a/tests/harness_bench/profile.py +++ b/tests/harness_bench/profile.py @@ -74,25 +74,29 @@ def declared_for(self, probe_name: str) -> Verdict: def resolve_profile(name: str) -> BenchProfile: """Resolve a harness name to a :class:`BenchProfile`. - Resolution chain (option B in the design doc): + Resolution chain: 1. An official harness in :mod:`tests.harness_bench.manifest`. 2. A community harness that ships a profile: *name* is a dotted path to either a ``BenchProfile`` instance or a zero-arg ``bench_profile()`` factory (e.g. ``mypkg.myharness:bench_profile`` or ``mypkg.myharness.PROFILE``). + 3. Any harness registered in the omnigent registry (in-repo or an + entry-point plugin), resolved by name / alias — the profile is + derived from the capability model (see + :func:`tests.harness_bench.manifest._registry_profile`). This is what + lets ``--harness acp`` or ``--harness rovo`` run with no bench edit. This keeps the official list a convenience index, not a gate: any - harness exposing a profile is probeable with ``--harness `` and - no bench edits. When per-harness self-registration lands, step 1 swaps - from a static dict to dynamic discovery with no change here. + registered harness is probeable by name, and any out-of-tree one that + ships a ``BenchProfile`` is probeable by reference. - :param name: Official harness name or a dotted path / ``module:attr``. + :param name: Harness name / alias, or a dotted path / ``module:attr``. :returns: The resolved profile. - :raises KeyError: If *name* is neither official nor an importable + :raises KeyError: If *name* is not a registered harness nor an importable profile reference. """ - from tests.harness_bench.manifest import OFFICIAL_PROFILES + from tests.harness_bench.manifest import OFFICIAL_PROFILES, _registry_profile if name in OFFICIAL_PROFILES: return OFFICIAL_PROFILES[name] @@ -101,10 +105,14 @@ def resolve_profile(name: str) -> BenchProfile: if resolved is not None: return resolved + from_registry = _registry_profile(name) + if from_registry is not None: + return from_registry + raise KeyError( - f"unknown harness {name!r}: not an official harness " - f"({', '.join(sorted(OFFICIAL_PROFILES))}) and not an importable " - f"BenchProfile reference (try 'module:attr' or 'module.ATTR')" + f"unknown harness {name!r}: not a registered omnigent harness, not an " + f"official profile ({', '.join(sorted(OFFICIAL_PROFILES))}), and not an " + f"importable BenchProfile reference (try 'module:attr' or 'module.ATTR')" ) diff --git a/tests/harness_bench/test_bench.py b/tests/harness_bench/test_bench.py index d1ca43f6732..97377cd2485 100644 --- a/tests/harness_bench/test_bench.py +++ b/tests/harness_bench/test_bench.py @@ -105,6 +105,45 @@ def test_resolve_official_and_community_and_unknown() -> None: resolve_profile("no-such-harness") +def test_resolve_registered_harness_by_name() -> None: + """A registered harness with no official profile is resolvable by name. + + This is the "plugs in with no bench edit" path: an in-repo ACP/CLI-subprocess + harness (not auto-derived, since that is native-tui-only) resolves via the + registry fallback, deriving a profile from the capability model. ACP is an + ACP_SUBPROCESS harness, so it lands on the SDK-wrap driver family + (transport "sdk-inproc"), not native-tui. + """ + from omnigent.harness_plugins import harness_modules + + if "acp" not in harness_modules(): + pytest.skip("acp harness not registered in this build") + profile = resolve_profile("acp") + assert profile.harness == "acp" + assert profile.transport == "sdk-inproc" + + +def test_resolve_entry_point_plugin_and_alias() -> None: + """An entry-point community plugin resolves by name AND by alias. + + ``omnigent-rovo`` registers ``rovo-cli`` (alias ``rovo``) via the + ``omnigent.community.harness`` entry point and declares no capabilities + entry — only a harness module + install spec. The registry fallback still + binds it (keying off harness_modules, defaulting to the SDK family) and + skip-gates on its install-spec binary. Gated on the plugin being installed + so a build without it still passes. + """ + from omnigent.harness_plugins import harness_aliases + + if harness_aliases().get("rovo") != "rovo-cli": + pytest.skip("omnigent-rovo plugin not installed") + by_alias = resolve_profile("rovo") + by_name = resolve_profile("rovo-cli") + assert by_alias.harness == "rovo-cli" == by_name.harness + assert by_alias.transport == "sdk-inproc" + assert by_alias.cli_binary == "acli" # skip-gates on the Atlassian CLI + + def test_infra_failure_reason_classifies_auth_and_ignores_capability_gaps() -> None: from tests.harness_bench.driver import TurnResult, infra_failure_reason From acef57fcef57612eb6007a2b0946a3164cfc8f3a Mon Sep 17 00:00:00 2001 From: Pat Sukprasert Date: Thu, 9 Jul 2026 13:42:43 +0800 Subject: [PATCH 2/5] =?UTF-8?q?fix(harness-bench):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20NATIVE=5FSERVER=20refusal,=20own-auth=20model,=20AC?= =?UTF-8?q?P-login=20SKIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes from PR review + a live rovo run: 1. (blocking, Polly) A MODELED integration_mode the bench has no driver for (NATIVE_SERVER, e.g. opencode-native) was silently degrading to the sdk-inproc default via `.get(mode, "sdk-inproc")` — binding a vendor-server harness to the wrong driver and dropping its skip-gate. _registry_profile now distinguishes: no caps (unmodeled plugin) -> assume SDK family; a modeled mode NOT in the transport map -> return None so resolve_profile KeyErrors (honest "unrunnable" rather than a wrong profile). resolve_profile("opencode -native") KeyErrors again. 2. A live rovo run (acli absent) reported `!!✓>✗` DRIFT: the ACP-session / vendor-login failure ("Ensure `acli` is installed and you are logged in", "AcpProcessExited", "ACP subprocess/session") wasn't an infra marker, so it read as a real UNSUPPORTED against the SUPPORTED declaration. Added those markers + a reason so an own-auth harness with no vendor login SKIPs (env gap), never drifts. 3. Registry profiles stamped a databricks-* placeholder model even for own-auth harnesses (rovo/acp), which is misleading — the runner drops the gateway model for them. Now: gateway-credential harness -> the databricks default; own-auth or capless -> empty model (the harness owns it). Tests: NATIVE_SERVER refusal; a plugin-independent happy-path (fake registered CLI harness via monkeypatch) so the fallback's positive path isn't skip-gated away in CI; rovo model=="" assertion. Offline suite 73 passed / 18 skipped. --- tests/harness_bench/driver.py | 16 ++++++++ tests/harness_bench/manifest.py | 28 ++++++++++--- tests/harness_bench/test_bench.py | 65 +++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/tests/harness_bench/driver.py b/tests/harness_bench/driver.py index b572f96072a..5fde73601ac 100644 --- a/tests/harness_bench/driver.py +++ b/tests/harness_bench/driver.py @@ -113,6 +113,14 @@ class ProvisioningError(RuntimeError): "provider auth command", "empty token", "Failed to resolve external API key auth", + # Own-auth harness whose vendor CLI is not installed / not logged in (e.g. + # an ACP harness like rovo: "Ensure `acli` is installed and you are logged + # in"). The vendor process exits before a turn can run — an environment/ + # login gap, not a capability the harness lacks, so it must SKIP not drift. + "are logged in", + "AcpProcessExited", + "ACP subprocess", + "ACP session", ) @@ -160,6 +168,14 @@ def infra_failure_reason(result: TurnResult) -> str | None: "gateway/provider token could not be provisioned for this transport " "(environment/auth gap, not a capability the harness lacks)" ) + if any( + marker in text + for marker in ("are logged in", "AcpProcessExited", "ACP subprocess", "ACP session") + ): + return ( + "vendor CLI not installed or not logged in (own-auth harness); " + "the agent process exited before a turn could run" + ) if "unexpected status" in text: return "gateway returned an unexpected status (environment/auth issue)" return "environment/connectivity error reaching the gateway" diff --git a/tests/harness_bench/manifest.py b/tests/harness_bench/manifest.py index bf5458e2195..907275f6c96 100644 --- a/tests/harness_bench/manifest.py +++ b/tests/harness_bench/manifest.py @@ -291,7 +291,20 @@ def _registry_profile(name: str) -> BenchProfile | None: caps = harness_capabilities().get(canonical) mode = caps.integration_mode if caps is not None else None - transport = _INTEGRATION_MODE_TRANSPORT.get(mode, "sdk-inproc") if mode else "sdk-inproc" + if mode is None: + # No capabilities entry (a plain subprocess plugin like rovo): the bench + # has no modeled transport, so assume the SDK-wrap family — the only + # thing a registered-but-unmodeled harness can plausibly run on. + transport = "sdk-inproc" + elif mode in _INTEGRATION_MODE_TRANSPORT: + transport = _INTEGRATION_MODE_TRANSPORT[mode] + else: + # A MODELED mode the bench has no driver for (e.g. NATIVE_SERVER / + # opencode-native). Refuse rather than silently degrade to the SDK + # family: return None so resolve_profile raises a clean KeyError. A + # bare "default to sdk-inproc" here would bind a native-server harness + # to the wrong driver and drop its skip-gate. + return None if transport == "native-tui": # A native-tui harness the auto-derivation would already cover; reuse @@ -300,12 +313,17 @@ def _registry_profile(name: str) -> BenchProfile | None: env_prefix = "HARNESS_" + canonical.upper().replace("-", "_") + "_" marker = canonical.upper().replace("-", "_") + "_OK" + # Only a gateway-credential harness routes a databricks-* model; stamp the + # default for it. An own-auth harness (e.g. ACP/rovo) owns its model — the + # runner drops any databricks-* gateway id for it (ACP: + # workflow.py::_build_acp_spawn_env) — and a capless plugin's model is its + # own business, so leave the model empty in both cases rather than stamp a + # misleading gateway id. + gateway_auth = caps is not None and caps.auth is AuthModel.OMNIGENT_CREDENTIAL + model = _NATIVE_DEFAULT_MODEL if gateway_auth else "" return BenchProfile( harness=canonical, - # An own-auth subprocess harness owns its model (the gateway model id is - # dropped for it), so this is a placeholder the harness ignores; a - # gateway-routed one would take a real databricks-* id. - model=_NATIVE_DEFAULT_MODEL, + model=model, env_prefix=env_prefix, marker=marker, cli_binary=_registry_cli_binary(canonical), diff --git a/tests/harness_bench/test_bench.py b/tests/harness_bench/test_bench.py index 97377cd2485..fe6dc5e1f52 100644 --- a/tests/harness_bench/test_bench.py +++ b/tests/harness_bench/test_bench.py @@ -142,6 +142,71 @@ def test_resolve_entry_point_plugin_and_alias() -> None: assert by_alias.harness == "rovo-cli" == by_name.harness assert by_alias.transport == "sdk-inproc" assert by_alias.cli_binary == "acli" # skip-gates on the Atlassian CLI + # Own-auth harness owns its model; the bench must not stamp a gateway id. + assert by_alias.model == "" + + +def test_registry_profile_happy_path_no_plugin(monkeypatch: pytest.MonkeyPatch) -> None: + """The registry fallback's positive path, independent of any optional plugin. + + Fakes a registered CLI-subprocess harness (+ alias + install-spec binary) so + the name/alias resolution, the integration_mode -> sdk-inproc mapping, and + the install-spec skip-gate are exercised even in a build without + omnigent-rovo. Guards the coverage the plugin tests skip-gate away. + """ + from types import SimpleNamespace + + import tests.harness_bench.manifest as man + from omnigent.harness_capabilities import AuthModel, IntegrationMode + + class _Spec: + binary = "fakebin" + + # A lightweight caps stand-in: _registry_profile + the prose/declared helpers + # only read integration_mode / auth / streaming / interrupt. + caps = SimpleNamespace( + integration_mode=IntegrationMode.CLI_SUBPROCESS, + auth=AuthModel.OWN_AUTH, + streaming=True, + interrupt=True, + ) + monkeypatch.setattr(man, "harness_modules", lambda: {"fake-cli": "pkg.fake"}) + monkeypatch.setattr(man, "harness_aliases", lambda: {"fake": "fake-cli"}) + monkeypatch.setattr(man, "harness_capabilities", lambda: {"fake-cli": caps}) + monkeypatch.setattr(man, "harness_install_keys", lambda: {"fake-cli": "fake"}) + monkeypatch.setattr(man, "install_specs", lambda: {"fake": _Spec()}) + + for name in ("fake-cli", "fake"): + p = man._registry_profile(name) + assert p is not None and p.harness == "fake-cli" + assert p.transport == "sdk-inproc" + assert p.cli_binary == "fakebin" + assert p.model == "" # own-auth -> no gateway model + + +def test_registry_refuses_native_server_mode(monkeypatch: pytest.MonkeyPatch) -> None: + """A MODELED mode the bench has no driver for is refused, not mis-bound. + + NATIVE_SERVER (e.g. opencode-native) has no bench driver. The fallback must + return None (-> resolve_profile KeyError) rather than silently degrade to + the sdk-inproc default, which would bind a vendor-server harness to the SDK + drivers and drop its skip-gate. + """ + from types import SimpleNamespace + + import tests.harness_bench.manifest as man + from omnigent.harness_capabilities import AuthModel, IntegrationMode + + caps = SimpleNamespace( + integration_mode=IntegrationMode.NATIVE_SERVER, + auth=AuthModel.OWN_AUTH, + streaming=False, + interrupt=False, + ) + monkeypatch.setattr(man, "harness_modules", lambda: {"srv": "pkg.srv"}) + monkeypatch.setattr(man, "harness_aliases", dict) + monkeypatch.setattr(man, "harness_capabilities", lambda: {"srv": caps}) + assert man._registry_profile("srv") is None def test_infra_failure_reason_classifies_auth_and_ignores_capability_gaps() -> None: From 4e6ffd0e6ded117b962c6f03da62cb45d4df05d2 Mon Sep 17 00:00:00 2001 From: Pat Sukprasert Date: Thu, 9 Jul 2026 13:55:00 +0800 Subject: [PATCH 3/5] fix(harness-bench): registry profiles need a valid model to register My previous "empty model for own-auth" change broke agent registration: the omnigent executor spec mandates a model (spec/omnigent.py: "executor.type= 'omnigent' requires a model"), so model="" -> 400 "llm.model must be present when llm block is present" on register_agent. Seen live: rovo got past auth + skip-gate into provisioning, then failed registration. A model is always required for registration, so stamp the databricks default in all cases. For an own-auth harness it is inert: the generic ACP harness drops databricks-* models (workflow.py::_build_acp_spawn_env), and rovo has no spawn-env builder + reads HARNESS_ROVO_MODEL directly from env (which the runner never sets for it), so rovo gets no model and lets Rovo Dev pick its own default at session/new. The placeholder satisfies registration and never reaches acli. Tests updated to assert a non-empty model (registration invariant) rather than empty. --- tests/harness_bench/manifest.py | 19 ++++++++++--------- tests/harness_bench/test_bench.py | 7 ++++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/harness_bench/manifest.py b/tests/harness_bench/manifest.py index 907275f6c96..5801d1f7b33 100644 --- a/tests/harness_bench/manifest.py +++ b/tests/harness_bench/manifest.py @@ -313,17 +313,18 @@ def _registry_profile(name: str) -> BenchProfile | None: env_prefix = "HARNESS_" + canonical.upper().replace("-", "_") + "_" marker = canonical.upper().replace("-", "_") + "_OK" - # Only a gateway-credential harness routes a databricks-* model; stamp the - # default for it. An own-auth harness (e.g. ACP/rovo) owns its model — the - # runner drops any databricks-* gateway id for it (ACP: - # workflow.py::_build_acp_spawn_env) — and a capless plugin's model is its - # own business, so leave the model empty in both cases rather than stamp a - # misleading gateway id. - gateway_auth = caps is not None and caps.auth is AuthModel.OMNIGENT_CREDENTIAL - model = _NATIVE_DEFAULT_MODEL if gateway_auth else "" + # A model is always required: the omnigent executor spec mandates one + # (spec/omnigent.py: "executor.type='omnigent' requires a model"), so an + # empty model fails agent registration ("llm.model must be present"). For an + # own-auth harness (e.g. ACP/rovo) the value is inert — the runner drops any + # databricks-* gateway model for it (ACP: workflow.py::_build_acp_spawn_env) + # and the agent authenticates + picks its own model — but it must still be a + # valid non-empty id to register. So stamp the databricks default in all + # cases: real for a gateway harness, an accepted-but-ignored placeholder for + # an own-auth one. return BenchProfile( harness=canonical, - model=model, + model=_NATIVE_DEFAULT_MODEL, env_prefix=env_prefix, marker=marker, cli_binary=_registry_cli_binary(canonical), diff --git a/tests/harness_bench/test_bench.py b/tests/harness_bench/test_bench.py index fe6dc5e1f52..19e4bf57d1e 100644 --- a/tests/harness_bench/test_bench.py +++ b/tests/harness_bench/test_bench.py @@ -142,8 +142,9 @@ def test_resolve_entry_point_plugin_and_alias() -> None: assert by_alias.harness == "rovo-cli" == by_name.harness assert by_alias.transport == "sdk-inproc" assert by_alias.cli_binary == "acli" # skip-gates on the Atlassian CLI - # Own-auth harness owns its model; the bench must not stamp a gateway id. - assert by_alias.model == "" + # A model is always stamped (agent registration requires a non-empty one); + # it is inert for an own-auth harness, which uses its own. + assert by_alias.model def test_registry_profile_happy_path_no_plugin(monkeypatch: pytest.MonkeyPatch) -> None: @@ -181,7 +182,7 @@ class _Spec: assert p is not None and p.harness == "fake-cli" assert p.transport == "sdk-inproc" assert p.cli_binary == "fakebin" - assert p.model == "" # own-auth -> no gateway model + assert p.model # always non-empty (agent registration requires a model) def test_registry_refuses_native_server_mode(monkeypatch: pytest.MonkeyPatch) -> None: From 79941e3e5176825ea51814274154d6630083072f Mon Sep 17 00:00:00 2001 From: Pat Sukprasert Date: Thu, 9 Jul 2026 14:35:52 +0800 Subject: [PATCH 4/5] feat(harness-bench): bind acp: ids to a specific ACP agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `acp:` is a first-class omnigent harness id — the base `acp` harness is registered and the slug selects a user-configured ACP agent at spawn (resolved from the ~/.omnigent `acp:` block). The registry fallback now recognizes it: look up caps/module/install-spec by the base `acp`, but keep the full `acp:` as the profile harness so `config.harness=acp:` reaches the runner, and sanitize the colon in the env-prefix/marker stem (acp:qwen -> HARNESS_ACP_QWEN_). An empty slug ("acp:") is refused. Lets `--harness acp:qwen` bind to a specific ACP agent for a live turn (qwen is installed + authed), vs the bare `acp` which needs HARNESS_ACP_COMMAND. Test added. Offline suite 73 passed / 18 skipped. --- tests/harness_bench/manifest.py | 29 +++++++++++++++++++++++------ tests/harness_bench/test_bench.py | 11 +++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/harness_bench/manifest.py b/tests/harness_bench/manifest.py index 5801d1f7b33..b19ccb42fb7 100644 --- a/tests/harness_bench/manifest.py +++ b/tests/harness_bench/manifest.py @@ -286,10 +286,22 @@ def _registry_profile(name: str) -> BenchProfile | None: the harness runs on the existing drivers with no bench edit. """ canonical = harness_aliases().get(name, name) - if canonical not in harness_modules(): + # ``acp:`` is a first-class harness id: the base ``acp`` harness is + # registered, and the slug selects a user-configured ACP agent at spawn + # (resolved from the ~/.omnigent config ``acp:`` block, see + # onboarding/acp_auth.py). Look up caps/module by the base ``acp`` but keep + # the full id as the profile harness so ``config.harness=acp:`` reaches + # the runner. Lets ``--harness acp:qwen`` bind to a specific ACP agent. + if canonical.startswith("acp:"): + if not canonical[len("acp:") :]: + return None # empty slug ("acp:") — use bare "acp" instead + registry_key = "acp" + else: + registry_key = canonical + if registry_key not in harness_modules(): return None - caps = harness_capabilities().get(canonical) + caps = harness_capabilities().get(registry_key) mode = caps.integration_mode if caps is not None else None if mode is None: # No capabilities entry (a plain subprocess plugin like rovo): the bench @@ -311,8 +323,11 @@ def _registry_profile(name: str) -> BenchProfile | None: # its builder so the two paths agree. return _native_profile(canonical) - env_prefix = "HARNESS_" + canonical.upper().replace("-", "_") + "_" - marker = canonical.upper().replace("-", "_") + "_OK" + # env_prefix / marker sanitize non-word chars (an acp: id has a colon) + # to a valid env-var stem, e.g. acp:qwen -> HARNESS_ACP_QWEN_. + stem = canonical.upper().replace("-", "_").replace(":", "_") + env_prefix = "HARNESS_" + stem + "_" + marker = stem + "_OK" # A model is always required: the omnigent executor spec mandates one # (spec/omnigent.py: "executor.type='omnigent' requires a model"), so an # empty model fails agent registration ("llm.model must be present"). For an @@ -327,12 +342,14 @@ def _registry_profile(name: str) -> BenchProfile | None: model=_NATIVE_DEFAULT_MODEL, env_prefix=env_prefix, marker=marker, - cli_binary=_registry_cli_binary(canonical), + # install-spec + declared caps are keyed by the base harness, not the + # acp: id, so look them up by registry_key. + cli_binary=_registry_cli_binary(registry_key), transport=transport, owner="", auth=_auth_prose(caps), implementation=_implementation_prose(caps), - declared=_declared_from_capabilities(canonical), + declared=_declared_from_capabilities(registry_key), ) diff --git a/tests/harness_bench/test_bench.py b/tests/harness_bench/test_bench.py index 19e4bf57d1e..b33e76cc683 100644 --- a/tests/harness_bench/test_bench.py +++ b/tests/harness_bench/test_bench.py @@ -122,6 +122,17 @@ def test_resolve_registered_harness_by_name() -> None: assert profile.harness == "acp" assert profile.transport == "sdk-inproc" + # acp: is a first-class id: base `acp` is registered and the slug + # selects a configured ACP agent at spawn. It binds, keeping the full id as + # the harness (so config.harness=acp:qwen reaches the runner) and a valid + # env-prefix stem. An empty slug ("acp:") is refused. + slug = resolve_profile("acp:qwen") + assert slug.harness == "acp:qwen" + assert slug.transport == "sdk-inproc" + assert slug.env_prefix == "HARNESS_ACP_QWEN_" + with pytest.raises(KeyError): + resolve_profile("acp:") + def test_resolve_entry_point_plugin_and_alias() -> None: """An entry-point community plugin resolves by name AND by alias. From c7c129d0cbe8935693dc258de4a0ccd04d11ffb6 Mon Sep 17 00:00:00 2001 From: Pat Sukprasert Date: Thu, 9 Jul 2026 14:51:45 +0800 Subject: [PATCH 5/5] fix(harness-bench): sanitize colon in bench agent name for acp: The bench built its agent name as bench-, but an acp: harness id has a colon, which the agent-name validator rejects ([a-zA-Z0-9_-]+). So a --harness acp:qwen run would 400 at registration. Replace ":" with "-" in the NAME only (bench-acp-qwen); config.harness keeps the real acp: id so the runner still resolves the right ACP agent at spawn. --- tests/harness_bench/full_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/harness_bench/full_server.py b/tests/harness_bench/full_server.py index 8c7347aeba7..f89877a43ef 100644 --- a/tests/harness_bench/full_server.py +++ b/tests/harness_bench/full_server.py @@ -166,7 +166,11 @@ def _build_bench_agent_config( ) -> dict[str, Any]: """The agent spec for a bench harness: the harness + the read-only builtin, plus (when *deny*) a baked tool_call-phase deny on that builtin.""" - name = f"bench-{profile.harness}" + ("-deny" if deny else "") + # The agent name must match [a-zA-Z0-9_-]+, but a harness id can contain a + # colon (acp:). Sanitize it for the name only; config.harness keeps + # the real id so the runner resolves the right ACP agent at spawn. + safe_harness = profile.harness.replace(":", "-") + name = f"bench-{safe_harness}" + ("-deny" if deny else "") config: dict[str, Any] = { "spec_version": 1, "name": name,