diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py index db0cd22a77..521b94d780 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/nooa_model_client.py @@ -9,10 +9,11 @@ public plugin contract. """ -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any from nemo_platform import AsyncNeMoPlatform from nemo_platform.config import get_context @@ -35,6 +36,20 @@ class ConfiguredModelRefs: fast: str +@dataclass(frozen=True) +class CompletionClientOptions: + """Optional kwargs applied when building each run-scoped CompletionClient. + + ``reasoning_effort`` is the OpenAI-shaped shortcut. When ``None``, the field + is omitted so the provider default applies. ``completion_params`` are spread + into ``CompletionClient`` for backend-specific knobs; an explicit + ``reasoning_effort`` wins over the same key inside ``completion_params``. + """ + + reasoning_effort: str | None = None + completion_params: Mapping[str, Any] = field(default_factory=dict) + + @dataclass(frozen=True) class ConfiguredModelClients: """Resolved Nooa clients for default and low-latency agent work.""" @@ -84,10 +99,21 @@ def _parse_model_ref(model_ref: str) -> tuple[str, str]: return workspace, name +def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]: + """Merge completion options: params first, then explicit reasoning_effort.""" + if options is None: + return {} + kwargs: dict[str, Any] = dict(options.completion_params) + if options.reasoning_effort is not None: + kwargs["reasoning_effort"] = options.reasoning_effort + return kwargs + + def _completion_client( client: AsyncNeMoPlatform, model_entity: ModelEntity, served_model_name: str, + options: CompletionClientOptions | None = None, ) -> CompletionClient: """Build a Nooa client that routes through the Model Entity's Platform URL.""" api_base = client.models.get_model_entity_route_openai_url(model_entity) @@ -96,6 +122,7 @@ def _completion_client( # response bytes. Nooa needs decoded JSON/SSE, so make that requirement # explicit at this adapter boundary for every configured agent client. extra_headers[_ACCEPT_ENCODING_HEADER] = _IDENTITY_ENCODING + option_kwargs = _client_option_kwargs(options) # Backend format is the Platform-facing wire contract, not the upstream # provider identity. The LiteLLM prefix selects the adapter for that shape. if model_entity.backend_format == _OPENAI_FORMAT: @@ -114,6 +141,7 @@ def _completion_client( # LiteLLM versions otherwise bridge GPT-5.4+ tool calls with any # reasoning_effort value (including "none") to /responses. _skip_responses_api_bridge=True, + **option_kwargs, ) elif model_entity.backend_format == _ANTHROPIC_FORMAT: api_base = api_base.removesuffix("/v1") @@ -131,6 +159,7 @@ def _completion_client( base_model=litellm_model, extra_headers=extra_headers, drop_params=True, + **option_kwargs, ) @@ -159,6 +188,7 @@ async def _served_model_name( async def resolve_model_clients( client: AsyncNeMoPlatform, refs: ConfiguredModelRefs | None = None, + options: CompletionClientOptions | None = None, ) -> ConfiguredModelClients: """Resolve configured Model Entities and construct each distinct client once.""" selected = refs or configured_model_refs() @@ -171,7 +201,7 @@ async def resolve_model_clients( workspace, name = _parse_model_ref(model_ref) entity = await client.models.retrieve(name, workspace=workspace) served_model_name = await _served_model_name(client, entity, provider_cache) - resolved[model_ref] = _completion_client(client, entity, served_model_name) + resolved[model_ref] = _completion_client(client, entity, served_model_name, options) except Exception as resolution_error: for model_client in resolved.values(): try: diff --git a/packages/nemo_platform_plugin/tests/test_nooa_model_client.py b/packages/nemo_platform_plugin/tests/test_nooa_model_client.py index 8614efb4f4..3c2012882c 100644 --- a/packages/nemo_platform_plugin/tests/test_nooa_model_client.py +++ b/packages/nemo_platform_plugin/tests/test_nooa_model_client.py @@ -7,6 +7,7 @@ import pytest from nemo_platform_plugin import nooa_model_client from nemo_platform_plugin.nooa_model_client import ( + CompletionClientOptions, ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients, @@ -248,3 +249,15 @@ async def test_resolve_model_clients_closes_constructed_client_after_failure(mon ) constructed.aclose.assert_awaited_once() + + +def test_client_option_kwargs_merge_and_omit() -> None: + assert nooa_model_client._client_option_kwargs(None) == {} + assert nooa_model_client._client_option_kwargs(CompletionClientOptions()) == {} + merged = nooa_model_client._client_option_kwargs( + CompletionClientOptions( + reasoning_effort="medium", + completion_params={"reasoning_effort": "minimal", "temperature": 0.2}, + ) + ) + assert merged == {"reasoning_effort": "medium", "temperature": 0.2} diff --git a/plugins/nemo-eval-author/README.md b/plugins/nemo-eval-author/README.md index 62b2ab7005..4fd64fe7a4 100644 --- a/plugins/nemo-eval-author/README.md +++ b/plugins/nemo-eval-author/README.md @@ -56,6 +56,13 @@ For non-interactive and isolated environments, `NEMO_DEFAULT_MODEL` and `NEMO_FAST_MODEL` can override the stored selections. Values must still use `workspace/model-name` and refer to Model Entities on the target Platform. +Completion options (not model IDs) live on `EvalAuthorConfig`. +`reasoning_effort` defaults to `"medium"` and **must stay at `medium` or +higher** for consistent metric authoring; weaker settings (or omitting the +field) often produce flat authored metrics. `completion_params` can pass +backend-specific kwargs. See +[`eval_author/README.md`](src/nemo_eval_author_plugin/eval_author/README.md#evalauthorconfig-model-and-completion-options). + A `nemo agents eval-author` CLI is registered under `nemo.cli.agents` and mounted by the agents plugin. Verb scaffolding is in place (`discover`, `audit`, `propose`, `run`, `doctor`); bodies are still diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md index a98af388b0..e610f08dfd 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md @@ -38,7 +38,41 @@ The preset includes the run inputs needed by `run_eval_author(...)`: - `task_template`: local or `fileset://` evaluator task template URI for production traces. - `experiment_dir`: local Eval Author working directory. - `workspace`, `base_url`, `mode`, and `evaluator_type`: platform and evaluator routing. -- `eval_author.max_summary_tokens` and `eval_author.max_traces`: agent tuning parameters. +- `eval_author.*`: agent tuning — see below. + +### `EvalAuthorConfig` (model and completion options) + +**Use `reasoning_effort` of `medium` or higher** (`high`, and any stronger +provider value your model accepts). Below `medium` — including `minimal`, +`low`, `none`, or omitting the field so the provider picks a weak default — +Eval Author often authors flat, non-discriminating metrics that look fine on a +broken baseline. The default is `"medium"` for that reason. Prefer raising +effort over lowering it when Author quality is inconsistent. + +| Field | Default | Meaning | +| --- | --- | --- | +| `max_summary_tokens` | `80000` | Token budget for the fast-model summarizer. | +| `max_traces` | `10` | Insight `trace_refs` to analyze in depth. | +| `max_validation_repair_attempts` | `5` | Repair attempts after Insight verifier validation fails. | +| `reasoning_effort` | `"medium"` | OpenAI-shaped effort for Author clients. Keep at `medium` or higher for consistent metric authoring; do not set `null` / `minimal` / `low` unless you are deliberately testing failure modes. | +| `completion_params` | `{}` | Extra kwargs forwarded to `CompletionClient` (non-OpenAI backends or other OpenAI knobs). An explicit `reasoning_effort` wins over the same key here. | + +Standalone `run_eval_author(...)` builds clients with these options. In +Experimentalist Insight mode, the runner nested-resolves Author-scoped clients +from the run config's `eval_author` block (same defaults), then restores the +outer Experimentalist default/fast pair for the optimization loop. + +Example override in Experimentalist `--config` YAML: + +```yaml +eval_author: + max_traces: 5 + reasoning_effort: medium # required floor for consistent Author metrics; use high if needed + # completion_params: + # thinking: + # type: enabled + # budget_tokens: 2048 +``` ## Materialized Insight Suite diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml index 559b5e62c7..3e53670449 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml @@ -23,3 +23,6 @@ experiment_dir: tmp/eval_author eval_author: max_summary_tokens: 80000 max_traces: 10 + # medium or higher required for consistent Author metrics + reasoning_effort: medium + # completion_params: {} diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py index 67dba19207..d9c1174436 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py @@ -3,7 +3,7 @@ """Small boundary models for Eval Author.""" -from typing import Self +from typing import Any, Self from nemo_experimentalist_plugin.entities import Dataset from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -65,6 +65,23 @@ class EvalAuthorConfig(BaseModel): le=10, description="Max Eval Author repair attempts after mandatory Insight verifier validation fails.", ) + reasoning_effort: str | None = Field( + default="medium", + description=( + "OpenAI-shaped reasoning_effort passed to CompletionClient. " + "Default medium. Keep at medium or higher for consistent metric " + "authoring; weaker values (or None, which omits the field) often " + "produce flat authored metrics." + ), + ) + completion_params: dict[str, Any] = Field( + default_factory=dict, + description=( + "Extra kwargs forwarded to CompletionClient (for non-OpenAI backends or " + "additional OpenAI knobs). An explicit reasoning_effort wins over the " + "same key here." + ), + ) class EvalAuthorResult(BaseModel): diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py index 08303119ad..ee50238352 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py @@ -19,6 +19,7 @@ from nemo_insights_plugin.entities import Insight from nemo_platform import AsyncNeMoPlatform from nemo_platform_plugin.nooa_model_client import ( + CompletionClientOptions, ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients, @@ -82,7 +83,14 @@ async def run_eval_author( client = make_client(base_url) model_clients: ConfiguredModelClients | None = None try: - model_clients = await resolve_model_clients(client, selected_model_refs) + model_clients = await resolve_model_clients( + client, + selected_model_refs, + CompletionClientOptions( + reasoning_effort=config.reasoning_effort, + completion_params=config.completion_params, + ), + ) backend = make_experimentalist_backend( client=client, experiments_output=str(experiment_dir), diff --git a/plugins/nemo-eval-author/tests/test_eval_author_run.py b/plugins/nemo-eval-author/tests/test_eval_author_run.py index f0d86c96d5..e299010f19 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_run.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_run.py @@ -45,12 +45,15 @@ def model_clients(monkeypatch: pytest.MonkeyPatch) -> ClosingModelClients: default="workspace-a/default-model", fast="workspace-a/fast-model", ) + resolve_calls: list[tuple[object, ...]] = [] - async def resolve(*_: object) -> ClosingModelClients: + async def resolve(*args: object) -> ClosingModelClients: + resolve_calls.append(args) return clients monkeypatch.setattr(eval_author_run, "configured_model_refs", lambda: refs) monkeypatch.setattr(eval_author_run, "resolve_model_clients", resolve) + clients.resolve_calls = resolve_calls # type: ignore[attr-defined] return clients @@ -223,6 +226,8 @@ async def test_run_eval_author_resolves_inputs_and_returns_datasets( ) assert client.closed assert model_clients.closed + options = model_clients.resolve_calls[0][2] # type: ignore[attr-defined] + assert options.reasoning_effort == "medium" def test_public_apis_accept_train_validation_and_generated_task_inputs() -> None: diff --git a/plugins/nemo-experimentalist/README.md b/plugins/nemo-experimentalist/README.md index 5041c9e206..476b4c8ccc 100644 --- a/plugins/nemo-experimentalist/README.md +++ b/plugins/nemo-experimentalist/README.md @@ -78,6 +78,9 @@ single leader, so complementary strengths stay alive across rounds. analyze traces or host an Insight API. - [NeMo Eval Author](../nemo-eval-author/README.md) builds the Insight-specific evaluation suite, invoked automatically in Insight mode. + Authoring uses `eval_author` run settings; keep `reasoning_effort` at + `medium` or higher so authored metrics stay discriminating. See the Eval + Author README for the full config surface. - **Harbor** runs the task containers that score every candidate. - **NeMo Experiments** mirrors each run and its candidates as an experiment group, so the lineage is visible in Studio. Structure only — rewards and @@ -171,7 +174,7 @@ win. "Required" below means required *when the profile does not supply it*. | `--train-dataset` | Local Harbor dataset or registry ref — the split candidates are proposed against. | Yes. | | `--validation-dataset` | The held-out split that selects the winner. | Yes. | | `--task-template` | Directory holding one Harbor task template (`task.toml` with placeholders); Eval Author fills a copy per failing trace. | Insight-driven mode only. | -| `--config` | Run configuration: round and candidate limits plus `source`, `storage`, `goal_config`, `coder`, `analyzer`, `proposer`, `evaluator`, `eval_author`. Rejects a `models:` key. | No — defaults apply. | +| `--config` | Run configuration: round and candidate limits plus `source`, `storage`, `goal_config`, `coder`, `analyzer`, `proposer`, `evaluator`, `eval_author` (Insight-mode Author tuning; keep `reasoning_effort` at `medium` or higher). Rejects a `models:` key. | No — defaults apply. | | `--workspace` | NeMo workspace for traces and run metadata. | No — profile, else `default`. | | `--base-url` | URL of the running platform. | No — `NMP_BASE_URL`, else `http://localhost:8080`. | | `--experiment-dir` | Where `eval-and-optimize/` is written. Also `-o`, `--output`, `--experiments-output`. | No — see [Output](#output). | @@ -285,7 +288,9 @@ Objectives are what candidates are Pareto-ranked on, with minimized metrics sign-inverted; regression metrics are deliberately kept out of that ranking. In an Insight-driven run, Eval Author's authored Insight metrics take over as the objective and your configured targets move to `regression_metrics`, so the -Insight gets fixed without giving up what the run already cared about. +Insight gets fixed without giving up what the run already cared about. Keep +`eval_author.reasoning_effort` at `medium` or higher (the default is `medium`); +weaker effort often yields flat authored metrics that do not track the repair. The agent under test is configured separately, and none of the variables above reach it. What arrives in the evaluation container is whatever each Harbor task diff --git a/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml b/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml index 0461d875b5..e2cf56cf89 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml @@ -29,3 +29,4 @@ optimizer: eval_author: max_traces: 10 max_validation_repair_attempts: 5 + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml b/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml index ddb518059d..0482e3aa05 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml @@ -27,3 +27,4 @@ optimizer: eval_author: max_traces: 3 max_validation_repair_attempts: 2 + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml index 5f4e3d577c..01b6d96e8c 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml @@ -23,3 +23,4 @@ optimizer: eval_author: max_traces: 10 max_validation_repair_attempts: 5 + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml index 942e051505..3ebfae6378 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml @@ -24,3 +24,4 @@ optimizer: eval_author: max_traces: 3 max_validation_repair_attempts: 2 + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml index b55362393f..5a750a9915 100644 --- a/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml @@ -94,3 +94,5 @@ outcome_evaluator_config: # shapes recorded for every smoke-agent group. eval_author: max_traces: 5 + # medium or higher — required for consistent Author metrics + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml index 4b6c043500..397ca476a4 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml @@ -19,3 +19,4 @@ outcome_evaluator_config: eval_author: max_traces: 3 max_validation_repair_attempts: 2 + reasoning_effort: medium diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py index 886246955f..c081307b29 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py @@ -214,7 +214,14 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: default_factory=dict, description="Config for the selected 'evaluation' component; its own model validates it.", ) - eval_author: EvalAuthorConfig = Field(default_factory=EvalAuthorConfig) + eval_author: EvalAuthorConfig = Field( + default_factory=EvalAuthorConfig, + description=( + "Insight-mode Eval Author settings. Defaults match EvalAuthorConfig, including " + "reasoning_effort='medium'. Keep reasoning_effort at medium or higher for " + "consistent discriminating metrics; weaker values often author flat metrics." + ), + ) @model_validator(mode="after") def validate_metric_contract(self) -> Self: diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/runner.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/runner.py index ccb68b22a0..71ca88af2e 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/runner.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/runner.py @@ -239,17 +239,41 @@ async def _prepare_inputs(self) -> PreparedInputs: # Lazy: a run without an Insight never authors an eval suite, so it must not # fail to import when the Eval Author package is absent. from nemo_eval_author_plugin.eval_author.agent import EvalAuthor # noqa: PLC0415 + from nemo_platform_plugin.nooa_model_client import ( # noqa: PLC0415 + CompletionClientOptions, + activate_model_clients, + get_configured_model_refs, + resolve_model_clients, + ) - authored = await EvalAuthor( - experiment_dir=self._root, config=self._config.eval_author, reporter=self._reporter - ).run( - insight=insight, - agent_path=agent_dir, - task_template=dataset_factory.build_task_template(self._config.outcome_evaluator, template_ref), - train_dataset=datasets["train"], - validation_dataset=datasets["validation"], - client=self._backend.client, + # Author gets its own clients so eval_author.reasoning_effort / + # completion_params apply without changing the outer Experimentalist pair. + author_clients = await resolve_model_clients( + self._backend.client, + get_configured_model_refs(), + CompletionClientOptions( + reasoning_effort=self._config.eval_author.reasoning_effort, + completion_params=self._config.eval_author.completion_params, + ), ) + try: + with activate_model_clients(author_clients): + authored = await EvalAuthor( + experiment_dir=self._root, + config=self._config.eval_author, + reporter=self._reporter, + ).run( + insight=insight, + agent_path=agent_dir, + task_template=dataset_factory.build_task_template( + self._config.outcome_evaluator, template_ref + ), + train_dataset=datasets["train"], + validation_dataset=datasets["validation"], + client=self._backend.client, + ) + finally: + await author_clients.aclose() datasets["train"] = authored.train_dataset datasets["validation"] = authored.validation_dataset diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index 0cf2c1c9b1..10536921b3 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -260,6 +260,8 @@ to select the winner. | `builder_config.max_fix_attempts` | `1` | `2` (default) | Maximum repair iterations when a candidate fails its integration check. | | `outcome_evaluator_config.n_attempts` | `1` | `1`; increase only when task results are noisy | Repeats each evaluation trial. | | `eval_author.max_traces` | `3` | `10` | Representative Insight traces deeply analyzed in Insight-driven mode. | +| `eval_author.reasoning_effort` | `"medium"` | `"medium"` or `"high"` | OpenAI-shaped effort for Author clients. **Must be `medium` or higher** for consistent discriminating metrics; do not use `null` / `minimal` / `low` except when testing failure modes. | +| `eval_author.completion_params` | `{}` | `{}` | Extra `CompletionClient` kwargs for non-OpenAI backends or other knobs. | A small explicit smoke configuration looks like this: @@ -281,6 +283,8 @@ outcome_evaluator_config: n_attempts: 1 eval_author: max_traces: 3 + # medium or higher — required for consistent Author metrics + reasoning_effort: medium ``` ### Create a low-cost smoke dataset diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_runner.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_runner.py index 6ee92cd68c..79492f1eb5 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_runner.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_runner.py @@ -38,6 +38,23 @@ def _write_tree(root: Path, content: str) -> None: (tests / "test.sh").write_text(content, encoding="utf-8") +def _stub_author_model_clients(monkeypatch: pytest.MonkeyPatch) -> None: + """Mode 1 resolves Author-scoped clients; keep staging tests offline.""" + + class _Clients: + async def aclose(self) -> None: + return None + + async def resolve(*_a: object, **_k: object) -> _Clients: + return _Clients() + + monkeypatch.setattr("nemo_platform_plugin.nooa_model_client.resolve_model_clients", resolve) + monkeypatch.setattr( + "nemo_platform_plugin.nooa_model_client.get_configured_model_refs", + lambda: SimpleNamespace(default="default/m", fast="default/m"), + ) + + @pytest.mark.asyncio async def test_insight_run_stages_inputs_and_stops_at_eval_author_handoff( monkeypatch: pytest.MonkeyPatch, @@ -95,6 +112,7 @@ async def run( ) monkeypatch.setattr(runner_module, "DatasetFactory", RecordingDatasetFactory) monkeypatch.setattr("nemo_eval_author_plugin.eval_author.agent.EvalAuthor", MutatingEvalAuthor) + _stub_author_model_clients(monkeypatch) agent_dir = tmp_path / "agent" agent_dir.mkdir() @@ -180,6 +198,7 @@ async def run(self, *, train_dataset, validation_dataset, **kwargs: object) -> A "distribute_insight_suite_tasks", lambda suite, train, validation: distributed.update(suite=suite, train=train, validation=validation), ) + _stub_author_model_clients(monkeypatch) agent_dir = tmp_path / "agent" agent_dir.mkdir() diff --git a/plugins/nemo-experimentalist/tests/test_eval_author_config.py b/plugins/nemo-experimentalist/tests/test_eval_author_config.py index 997cf65ae0..da20c32a35 100644 --- a/plugins/nemo-experimentalist/tests/test_eval_author_config.py +++ b/plugins/nemo-experimentalist/tests/test_eval_author_config.py @@ -13,6 +13,7 @@ def test_evolutionary_optimizer_uses_top_level_eval_author_config() -> None: assert type(config) is EvalAuthorConfig assert config.max_validation_repair_attempts == 5 + assert config.reasoning_effort == "medium" def test_evolutionary_optimizer_tolerates_unknown_eval_author_config() -> None: