NVIDIA-NeMo · aleckhoury · Aug 17, 2026 · Aug 17, 2026 · Aug 17, 2026 · Aug 17, 2026
@@ -9,10 +9,11 @@
 public plugin contract.
 """
 
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Any
 
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform.config import get_context
@@ -35,6 +36,20 @@ class ConfiguredModelRefs:
     fast: str
 
 
+@dataclass(frozen=True)
+class CompletionClientOptions:
+    """Optional kwargs applied when building each run-scoped CompletionClient.
+
+    ``reasoning_effort`` is the OpenAI-shaped shortcut. When ``None``, the field
+    is omitted so the provider default applies. ``completion_params`` are spread
+    into ``CompletionClient`` for backend-specific knobs; an explicit
+    ``reasoning_effort`` wins over the same key inside ``completion_params``.
+    """
+
+    reasoning_effort: str | None = None
+    completion_params: Mapping[str, Any] = field(default_factory=dict)
+
+
 @dataclass(frozen=True)
 class ConfiguredModelClients:
     """Resolved Nooa clients for default and low-latency agent work."""
@@ -84,10 +99,21 @@ def _parse_model_ref(model_ref: str) -> tuple[str, str]:
     return workspace, name
 
 
+def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]:
+    """Merge completion options: params first, then explicit reasoning_effort."""
+    if options is None:
+        return {}
+    kwargs: dict[str, Any] = dict(options.completion_params)
+    if options.reasoning_effort is not None:
+        kwargs["reasoning_effort"] = options.reasoning_effort
+    return kwargs
-def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]:
-    """Merge completion options: params first, then explicit reasoning_effort."""
-    if options is None:
-        return {}
-    kwargs: dict[str, Any] = dict(options.completion_params)
-    if options.reasoning_effort is not None:
-        kwargs["reasoning_effort"] = options.reasoning_effort
-    return kwargs
+def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]:
+    """Merge completion options: params first, then explicit reasoning_effort."""
+    if options is None:
+        return {}
+    kwargs: dict[str, Any] = dict(options.completion_params)
+    if options.reasoning_effort is None:
+        kwargs.pop("reasoning_effort", None)
+    else:
+        kwargs["reasoning_effort"] = options.reasoning_effort
+    return kwargs
-def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]:
-    """Merge completion options: params first, then explicit reasoning_effort."""
-    if options is None:
-        return {}
-    kwargs: dict[str, Any] = dict(options.completion_params)
-    if options.reasoning_effort is not None:
-        kwargs["reasoning_effort"] = options.reasoning_effort
-    return kwargs
+def _client_option_kwargs(options: CompletionClientOptions | None) -> dict[str, Any]:
+    """Merge completion options: params first, then explicit reasoning_effort."""
+    if options is None:
+        return {}
+    kwargs: dict[str, Any] = dict(options.completion_params)
+    if options.reasoning_effort is None:
+        kwargs.pop("reasoning_effort", None)
+    else:
+        kwargs["reasoning_effort"] = options.reasoning_effort
+    return kwargs
+
+
 def _completion_client(
     client: AsyncNeMoPlatform,
     model_entity: ModelEntity,
     served_model_name: str,
+    options: CompletionClientOptions | None = None,
 ) -> CompletionClient:
     """Build a Nooa client that routes through the Model Entity's Platform URL."""
     api_base = client.models.get_model_entity_route_openai_url(model_entity)
@@ -96,6 +122,7 @@ def _completion_client(
     # response bytes. Nooa needs decoded JSON/SSE, so make that requirement
     # explicit at this adapter boundary for every configured agent client.
     extra_headers[_ACCEPT_ENCODING_HEADER] = _IDENTITY_ENCODING
+    option_kwargs = _client_option_kwargs(options)
     # Backend format is the Platform-facing wire contract, not the upstream
     # provider identity. The LiteLLM prefix selects the adapter for that shape.
     if model_entity.backend_format == _OPENAI_FORMAT:
@@ -114,6 +141,7 @@ def _completion_client(
             # LiteLLM versions otherwise bridge GPT-5.4+ tool calls with any
             # reasoning_effort value (including "none") to /responses.
             _skip_responses_api_bridge=True,
+            **option_kwargs,
         )
     elif model_entity.backend_format == _ANTHROPIC_FORMAT:
         api_base = api_base.removesuffix("/v1")
@@ -131,6 +159,7 @@ def _completion_client(
         base_model=litellm_model,
         extra_headers=extra_headers,
         drop_params=True,
+        **option_kwargs,
     )
 
 
@@ -159,6 +188,7 @@ async def _served_model_name(
 async def resolve_model_clients(
     client: AsyncNeMoPlatform,
     refs: ConfiguredModelRefs | None = None,
+    options: CompletionClientOptions | None = None,
 ) -> ConfiguredModelClients:
     """Resolve configured Model Entities and construct each distinct client once."""
     selected = refs or configured_model_refs()
@@ -171,7 +201,7 @@ async def resolve_model_clients(
             workspace, name = _parse_model_ref(model_ref)
             entity = await client.models.retrieve(name, workspace=workspace)
             served_model_name = await _served_model_name(client, entity, provider_cache)
-            resolved[model_ref] = _completion_client(client, entity, served_model_name)
+            resolved[model_ref] = _completion_client(client, entity, served_model_name, options)
     except Exception as resolution_error:
         for model_client in resolved.values():
             try:

@@ -7,6 +7,7 @@
 import pytest
 from nemo_platform_plugin import nooa_model_client
 from nemo_platform_plugin.nooa_model_client import (
+    CompletionClientOptions,
     ConfiguredModelClients,
     ConfiguredModelRefs,
     activate_model_clients,
@@ -248,3 +249,15 @@ async def test_resolve_model_clients_closes_constructed_client_after_failure(mon
         )
 
     constructed.aclose.assert_awaited_once()
+
+
+def test_client_option_kwargs_merge_and_omit() -> None:
+    assert nooa_model_client._client_option_kwargs(None) == {}
+    assert nooa_model_client._client_option_kwargs(CompletionClientOptions()) == {}
+    merged = nooa_model_client._client_option_kwargs(
+        CompletionClientOptions(
+            reasoning_effort="medium",
+            completion_params={"reasoning_effort": "minimal", "temperature": 0.2},
+        )
+    )
+    assert merged == {"reasoning_effort": "medium", "temperature": 0.2}
@@ -56,6 +56,13 @@ For non-interactive and isolated environments, `NEMO_DEFAULT_MODEL` and
 `NEMO_FAST_MODEL` can override the stored selections. Values must still use
 `workspace/model-name` and refer to Model Entities on the target Platform.
 
+Completion options (not model IDs) live on `EvalAuthorConfig`.
+`reasoning_effort` defaults to `"medium"` and **must stay at `medium` or
+higher** for consistent metric authoring; weaker settings (or omitting the
+field) often produce flat authored metrics. `completion_params` can pass
+backend-specific kwargs. See
+[`eval_author/README.md`](src/nemo_eval_author_plugin/eval_author/README.md#evalauthorconfig-model-and-completion-options).
+
 A `nemo agents eval-author` CLI is registered under `nemo.cli.agents` and
 mounted by the agents plugin. Verb scaffolding is in place
 (`discover`, `audit`, `propose`, `run`, `doctor`); bodies are still

@@ -38,7 +38,41 @@ The preset includes the run inputs needed by `run_eval_author(...)`:
 - `task_template`: local or `fileset://` evaluator task template URI for production traces.
 - `experiment_dir`: local Eval Author working directory.
 - `workspace`, `base_url`, `mode`, and `evaluator_type`: platform and evaluator routing.
-- `eval_author.max_summary_tokens` and `eval_author.max_traces`: agent tuning parameters.
+- `eval_author.*`: agent tuning — see below.
+
+### `EvalAuthorConfig` (model and completion options)
+
+**Use `reasoning_effort` of `medium` or higher** (`high`, and any stronger
+provider value your model accepts). Below `medium` — including `minimal`,
+`low`, `none`, or omitting the field so the provider picks a weak default —
+Eval Author often authors flat, non-discriminating metrics that look fine on a
+broken baseline. The default is `"medium"` for that reason. Prefer raising
+effort over lowering it when Author quality is inconsistent.
+
+| Field | Default | Meaning |
+| --- | --- | --- |
+| `max_summary_tokens` | `80000` | Token budget for the fast-model summarizer. |
+| `max_traces` | `10` | Insight `trace_refs` to analyze in depth. |
+| `max_validation_repair_attempts` | `5` | Repair attempts after Insight verifier validation fails. |
+| `reasoning_effort` | `"medium"` | OpenAI-shaped effort for Author clients. Keep at `medium` or higher for consistent metric authoring; do not set `null` / `minimal` / `low` unless you are deliberately testing failure modes. |
+| `completion_params` | `{}` | Extra kwargs forwarded to `CompletionClient` (non-OpenAI backends or other OpenAI knobs). An explicit `reasoning_effort` wins over the same key here. |
+
+Standalone `run_eval_author(...)` builds clients with these options. In
+Experimentalist Insight mode, the runner nested-resolves Author-scoped clients
+from the run config's `eval_author` block (same defaults), then restores the
+outer Experimentalist default/fast pair for the optimization loop.
+
+Example override in Experimentalist `--config` YAML:
+
+```yaml
+eval_author:
+  max_traces: 5
+  reasoning_effort: medium   # required floor for consistent Author metrics; use high if needed
+  # completion_params:
+  #   thinking:
+  #     type: enabled
+  #     budget_tokens: 2048
+```
 
 ## Materialized Insight Suite
 

@@ -23,3 +23,6 @@ experiment_dir: tmp/eval_author
 eval_author:
   max_summary_tokens: 80000
   max_traces: 10
+  # medium or higher required for consistent Author metrics
+  reasoning_effort: medium
+  # completion_params: {}
@@ -3,7 +3,7 @@
 
 """Small boundary models for Eval Author."""
 
-from typing import Self
+from typing import Any, Self
 
 from nemo_experimentalist_plugin.entities import Dataset
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
@@ -65,6 +65,23 @@ class EvalAuthorConfig(BaseModel):
         le=10,
         description="Max Eval Author repair attempts after mandatory Insight verifier validation fails.",
     )
+    reasoning_effort: str | None = Field(
+        default="medium",
+        description=(
+            "OpenAI-shaped reasoning_effort passed to CompletionClient. "
+            "Default medium. Keep at medium or higher for consistent metric "
+            "authoring; weaker values (or None, which omits the field) often "
+            "produce flat authored metrics."
+        ),
+    )
+    completion_params: dict[str, Any] = Field(
+        default_factory=dict,
+        description=(
+            "Extra kwargs forwarded to CompletionClient (for non-OpenAI backends or "
+            "additional OpenAI knobs). An explicit reasoning_effort wins over the "
+            "same key here."
+        ),
+    )
 
 
 class EvalAuthorResult(BaseModel):

@@ -19,6 +19,7 @@
 from nemo_insights_plugin.entities import Insight
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.nooa_model_client import (
+    CompletionClientOptions,
     ConfiguredModelClients,
     ConfiguredModelRefs,
     activate_model_clients,
@@ -82,7 +83,14 @@ async def run_eval_author(
     client = make_client(base_url)
     model_clients: ConfiguredModelClients | None = None
     try:
-        model_clients = await resolve_model_clients(client, selected_model_refs)
+        model_clients = await resolve_model_clients(
+            client,
+            selected_model_refs,
+            CompletionClientOptions(
+                reasoning_effort=config.reasoning_effort,
+                completion_params=config.completion_params,
+            ),
+        )
         backend = make_experimentalist_backend(
             client=client,
             experiments_output=str(experiment_dir),

@@ -45,12 +45,15 @@ def model_clients(monkeypatch: pytest.MonkeyPatch) -> ClosingModelClients:
         default="workspace-a/default-model",
         fast="workspace-a/fast-model",
     )
+    resolve_calls: list[tuple[object, ...]] = []
 
-    async def resolve(*_: object) -> ClosingModelClients:
+    async def resolve(*args: object) -> ClosingModelClients:
+        resolve_calls.append(args)
         return clients
 
     monkeypatch.setattr(eval_author_run, "configured_model_refs", lambda: refs)
     monkeypatch.setattr(eval_author_run, "resolve_model_clients", resolve)
+    clients.resolve_calls = resolve_calls  # type: ignore[attr-defined]
     return clients
 
 
@@ -223,6 +226,8 @@ async def test_run_eval_author_resolves_inputs_and_returns_datasets(
     )
     assert client.closed
     assert model_clients.closed
+    options = model_clients.resolve_calls[0][2]  # type: ignore[attr-defined]
+    assert options.reasoning_effort == "medium"
 
 
 def test_public_apis_accept_train_validation_and_generated_task_inputs() -> None:

@@ -78,6 +78,9 @@ single leader, so complementary strengths stay alive across rounds.
   analyze traces or host an Insight API.
 - [NeMo Eval Author](../nemo-eval-author/README.md) builds the
   Insight-specific evaluation suite, invoked automatically in Insight mode.
+  Authoring uses `eval_author` run settings; keep `reasoning_effort` at
+  `medium` or higher so authored metrics stay discriminating. See the Eval
+  Author README for the full config surface.
 - **Harbor** runs the task containers that score every candidate.
 - **NeMo Experiments** mirrors each run and its candidates as an experiment
   group, so the lineage is visible in Studio. Structure only — rewards and
@@ -171,7 +174,7 @@ win. "Required" below means required *when the profile does not supply it*.
 | `--train-dataset` | Local Harbor dataset or registry ref — the split candidates are proposed against. | Yes. |
 | `--validation-dataset` | The held-out split that selects the winner. | Yes. |
 | `--task-template` | Directory holding one Harbor task template (`task.toml` with placeholders); Eval Author fills a copy per failing trace. | Insight-driven mode only. |
-| `--config` | Run configuration: round and candidate limits plus `source`, `storage`, `goal_config`, `coder`, `analyzer`, `proposer`, `evaluator`, `eval_author`. Rejects a `models:` key. | No — defaults apply. |
+| `--config` | Run configuration: round and candidate limits plus `source`, `storage`, `goal_config`, `coder`, `analyzer`, `proposer`, `evaluator`, `eval_author` (Insight-mode Author tuning; keep `reasoning_effort` at `medium` or higher). Rejects a `models:` key. | No — defaults apply. |
 | `--workspace` | NeMo workspace for traces and run metadata. | No — profile, else `default`. |
 | `--base-url` | URL of the running platform. | No — `NMP_BASE_URL`, else `http://localhost:8080`. |
 | `--experiment-dir` | Where `eval-and-optimize/` is written. Also `-o`, `--output`, `--experiments-output`. | No — see [Output](#output). |
@@ -285,7 +288,9 @@ Objectives are what candidates are Pareto-ranked on, with minimized metrics
 sign-inverted; regression metrics are deliberately kept out of that ranking. In
 an Insight-driven run, Eval Author's authored Insight metrics take over as the
 objective and your configured targets move to `regression_metrics`, so the
-Insight gets fixed without giving up what the run already cared about.
+Insight gets fixed without giving up what the run already cared about. Keep
+`eval_author.reasoning_effort` at `medium` or higher (the default is `medium`);
+weaker effort often yields flat authored metrics that do not track the repair.
 
 The agent under test is configured separately, and none of the variables above
 reach it. What arrives in the evaluation container is whatever each Harbor task

@@ -29,3 +29,4 @@ optimizer:
   eval_author:
     max_traces: 10
     max_validation_repair_attempts: 5
+    reasoning_effort: medium
@@ -27,3 +27,4 @@ optimizer:
   eval_author:
     max_traces: 3
     max_validation_repair_attempts: 2
+    reasoning_effort: medium
@@ -23,3 +23,4 @@ optimizer:
   eval_author:
     max_traces: 10
     max_validation_repair_attempts: 5
+    reasoning_effort: medium
@@ -24,3 +24,4 @@ optimizer:
   eval_author:
     max_traces: 3
     max_validation_repair_attempts: 2
+    reasoning_effort: medium
@@ -94,3 +94,5 @@ outcome_evaluator_config:
 # shapes recorded for every smoke-agent group.
 eval_author:
   max_traces: 5
+  # medium or higher — required for consistent Author metrics
+  reasoning_effort: medium
@@ -19,3 +19,4 @@ outcome_evaluator_config:
 eval_author:
   max_traces: 3
   max_validation_repair_attempts: 2
+  reasoning_effort: medium
@@ -214,7 +214,14 @@ def reject_legacy_curator_config(cls, data: Any) -> Any:
         default_factory=dict,
         description="Config for the selected 'evaluation' component; its own model validates it.",
     )
-    eval_author: EvalAuthorConfig = Field(default_factory=EvalAuthorConfig)
+    eval_author: EvalAuthorConfig = Field(
+        default_factory=EvalAuthorConfig,
+        description=(
+            "Insight-mode Eval Author settings. Defaults match EvalAuthorConfig, including "
+            "reasoning_effort='medium'. Keep reasoning_effort at medium or higher for "
+            "consistent discriminating metrics; weaker values often author flat metrics."
+        ),
+    )
 
     @model_validator(mode="after")
     def validate_metric_contract(self) -> Self:

@@ -239,17 +239,41 @@ async def _prepare_inputs(self) -> PreparedInputs:
             # Lazy: a run without an Insight never authors an eval suite, so it must not
             # fail to import when the Eval Author package is absent.
             from nemo_eval_author_plugin.eval_author.agent import EvalAuthor  # noqa: PLC0415
+            from nemo_platform_plugin.nooa_model_client import (  # noqa: PLC0415
+                CompletionClientOptions,
+                activate_model_clients,
+                get_configured_model_refs,
+                resolve_model_clients,
+            )
 
-            authored = await EvalAuthor(
-                experiment_dir=self._root, config=self._config.eval_author, reporter=self._reporter
-            ).run(
-                insight=insight,
-                agent_path=agent_dir,
-                task_template=dataset_factory.build_task_template(self._config.outcome_evaluator, template_ref),
-                train_dataset=datasets["train"],
-                validation_dataset=datasets["validation"],
-                client=self._backend.client,
+            # Author gets its own clients so eval_author.reasoning_effort /
+            # completion_params apply without changing the outer Experimentalist pair.
+            author_clients = await resolve_model_clients(
+                self._backend.client,
+                get_configured_model_refs(),
+                CompletionClientOptions(
+                    reasoning_effort=self._config.eval_author.reasoning_effort,
+                    completion_params=self._config.eval_author.completion_params,
+                ),
             )
+            try:
+                with activate_model_clients(author_clients):
+                    authored = await EvalAuthor(
+                        experiment_dir=self._root,
+                        config=self._config.eval_author,
+                        reporter=self._reporter,
+                    ).run(
+                        insight=insight,
+                        agent_path=agent_dir,
+                        task_template=dataset_factory.build_task_template(
+                            self._config.outcome_evaluator, template_ref
+                        ),
+                        train_dataset=datasets["train"],
+                        validation_dataset=datasets["validation"],
+                        client=self._backend.client,
+                    )
+            finally:
+                await author_clients.aclose()
             datasets["train"] = authored.train_dataset
             datasets["validation"] = authored.validation_dataset