Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ class CodeEditBuilderConfig(BaseModel):
default=2,
description="Max LLM repair iterations inside integration_check before giving up on a candidate.",
)
max_architecture_doc_iterations: int = Field(
default=100,
gt=0,
description="Max CodeAct iterations allowed to write architecture.md. Raise it for agents with many source files.",
)
model_catalog_path: Path | None = Field(
default=None,
description="Optional YAML model catalog path overriding the packaged assets/models.yaml.",
Expand Down Expand Up @@ -578,6 +583,11 @@ async def run(self, dataset: Dataset) -> None:
"""


def _architecture_doc_codeact(max_iterations: int) -> CodeActConfig:
"""Bound one architecture-doc generation, which reads source through slow shell work."""
return CodeActConfig(max_iterations=max_iterations, cell_timeout=3600.0)


# Keeps its own Evaluator rather than using ctx.evaluate: its smoke checks run against an
# artifact that is deliberately not yet a Candidate, and ctx.evaluate exists to associate a
# result with one. An internal check has nothing to associate and must not be recorded
Expand Down Expand Up @@ -1041,11 +1051,32 @@ async def run_smoke_eval(
options=smoke_options,
)

async def create_architecture_doc(
self, workdir: Path, source_path: str | None = None, entrypoint: str | None = None
) -> None:
"""Update (or, only if absent, create) architecture.md for the given agent.

Args:
workdir: The agent directory to document.
source_path: Directory holding the agent source, relative to the agent directory.
entrypoint: File the evaluation harness invokes, relative to the agent directory.

"""
# A @strategy config is fixed when the class is defined, but how many iterations
# documenting an agent takes scales with the source this builder was handed.
codeact = _architecture_doc_codeact(self._config.max_architecture_doc_iterations)
await self._create_architecture_doc(
workdir,
source_path=source_path,
entrypoint=entrypoint,
_strategy=CodeActStrategy(config=codeact), # ty: ignore[unknown-argument]
)

@strategy(
CodeActStrategy(config=CodeActConfig(max_iterations=50, cell_timeout=3600.0)),
CodeActStrategy(config=_architecture_doc_codeact(CodeEditBuilderConfig().max_architecture_doc_iterations)),
llm=lambda self: self._architecture_model,
)
async def create_architecture_doc(
async def _create_architecture_doc(
self, workdir: Path, source_path: str | None = None, entrypoint: str | None = None
) -> None:
"""Update (or, only if absent, create) architecture.md for the given agent.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,13 @@ eval_author:
max_traces: 3
```

`builder_config.max_architecture_doc_iterations` is how many iterations the
builder can use to write `architecture.md`, and it defaults to `100`. It is
separate from the evaluation budget above, but each additional iteration adds
model usage and run time to that step. Increase it when a run stops with
`Generation failed after 100 iterations (max_iterations=100)`. An agent that has
many source files needs more iterations.

### Create a low-cost smoke dataset

When the full dataset is expensive, create small **copied** train and
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,30 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import json
from pathlib import Path
from typing import cast

import pytest
from nemo_experimentalist_plugin.experimentalist.components.coder import CodeEditBuilder, CodeEditBuilderConfig
from nemo_experimentalist_plugin.experimentalist.components.holdout_utils import BLOCKED_MESSAGE
from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools
from nemo_platform_plugin.nooa_model_client import ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients
from nooa.agentdoc import pformat
from nooa.errors import GenerationError
from nooa.tools import ShellResult
from nooa.unifiedllm import CompletionClient, FakeLLMClient
from nooa.unifiedllm import CompletionClient, FakeLLMClient, LLMResponse, ToolCall


def _exec_response(code: str) -> LLMResponse:
"""A scripted LLM turn that drives CodeAct's ``execute_python`` tool with ``code``."""
return LLMResponse(
raw_response=None,
content="",
finish_reason="tool_calls",
assistant_message={"role": "assistant", "content": ""},
tool_calls=[ToolCall(id="call_exec", name="execute_python", arguments=json.dumps({"code": code}))],
)


async def test_guarded_shell_tools_runs_allowed_commands(tmp_path):
Expand Down Expand Up @@ -65,6 +79,20 @@ def test_coder_uses_default_model_for_architecture_docs(tmp_path: Path) -> None:
assert coder._architecture_model is default


async def test_architecture_doc_stops_at_the_configured_iteration_limit(tmp_path: Path) -> None:
"""The configured limit, not the one fixed on the @strategy decorator, must bound the run."""
assert CodeEditBuilderConfig().max_architecture_doc_iterations == 100, "the documented default"

# Each scripted turn runs a no-op cell instead of returning a result, so the only way
# out is exhausting the limit. A turn past the limit would report a larger count.
never_finishes = FakeLLMClient(scripted_responses=[_exec_response("x = 1") for _ in range(4)])
builder = CodeEditBuilder(workspace=tmp_path, config=CodeEditBuilderConfig(max_architecture_doc_iterations=3))
builder._architecture_model = never_finishes

with pytest.raises(GenerationError, match=r"after 3 iterations \(max_iterations=3\)"):
await builder.create_architecture_doc(tmp_path)


async def test_coder_lists_agent_mutation_models_from_catalog(tmp_path: Path) -> None:
catalog = tmp_path / "models.yaml"
catalog.write_text(
Expand Down