Skip to content
Merged
17 changes: 12 additions & 5 deletions services/studio/src/nmp/studio/coding_agent_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,16 +165,23 @@ def _decode_answer_pair_value(value: str) -> str:
return decoded if isinstance(decoded, str) else value


def answer_selection_pairs(text: str) -> list[tuple[str, str]]:
"""Return the question and answer pairs persisted by AskUserQuestion."""
pairs: list[tuple[str, str]] = []
for match in _ANSWER_PAIR_RE.finditer(text):
question = _decode_answer_pair_value(match.group(1)).strip()
answer = _decode_answer_pair_value(match.group(2)).strip()
if question and answer:
pairs.append((question, answer))
return pairs


def record_answer_selections(
artifacts: ChatArtifactsResponse,
text: str,
question_labels: dict[str, str] | None = None,
) -> None:
for match in _ANSWER_PAIR_RE.finditer(text):
question = _decode_answer_pair_value(match.group(1)).strip()
answer = _decode_answer_pair_value(match.group(2)).strip()
if not question or not answer:
continue
for question, answer in answer_selection_pairs(text):
label = question_labels.get(question) if question_labels else None
_set_selection_artifact(artifacts, label or _selection_label(question), answer)

Expand Down
10 changes: 6 additions & 4 deletions services/studio/src/nmp/studio/coding_agent_mcp_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,12 +226,14 @@
),
(
"For clarification, multiple-choice, yes/no, or freeform questions that do NOT map to one of "
"the select_* tools, use Claude Code's AskUserQuestion tool rather than writing a "
"questionnaire in markdown."
"the select_* tools, use Claude Code's AskUserQuestion tool rather than a questionnaire "
"in markdown."
),
(
"Only fall back to plain chat questions when no suitable UI tool is available, the user "
"already provided the value, or the UI tool returns skipped or error."
"Only fall back to plain chat questions when no suitable UI tool exists, the user already "
"provided the value, or the user explicitly skips the UI tool. A timeout, disconnect, or "
"other UI-tool error is not permission to continue or repeat the question in plain text; "
"leave the input unresolved and tell the user the interactive request must be retried."
),
(
"Set UI tool titles, descriptions, display labels, and output_key values to match the "
Expand Down
275 changes: 220 additions & 55 deletions services/studio/src/nmp/studio/coding_agents.py

Large diffs are not rendered by default.

161 changes: 161 additions & 0 deletions services/studio/tests/unit/test_coding_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ def test_build_claude_argv_uses_new_session_then_resume_flag():
assert argv[:3] == ["claude", "-p", "hello"]
assert "--output-format" in argv
assert "stream-json" in argv
mcp_config = json.loads(argv[argv.index("--mcp-config") + 1])
assert mcp_config["mcpServers"][coding_agents.CLAUDE_MCP_SERVER_NAME] == {
"type": "http",
"url": "http://test/mcp",
"timeout": coding_agents.CLAUDE_MCP_TOOL_TIMEOUT_MS,
}
assert "--allowedTools" in argv
allowed_tools = argv[argv.index("--allowedTools") + 1].split(",")
assert f"mcp__{coding_agents.CLAUDE_MCP_SERVER_NAME}__select_agent" in allowed_tools
Expand All @@ -127,6 +133,7 @@ def test_build_claude_argv_uses_new_session_then_resume_flag():
assert f"mcp__{coding_agents.CLAUDE_MCP_SERVER_NAME}__select_model" in allowed_tools
assert f"mcp__{coding_agents.CLAUDE_MCP_SERVER_NAME}__job_progress" in allowed_tools
assert f"mcp__{coding_agents.CLAUDE_MCP_SERVER_NAME}__studio_link" in allowed_tools
assert "--disallowedTools" not in argv
assert "--append-system-prompt" in argv
assert argv[argv.index("--append-system-prompt") + 1] == coding_agents.STUDIO_CODING_AGENT_CONTEXT
assert "--permission-prompt-tool" in argv
Expand Down Expand Up @@ -408,6 +415,7 @@ def test_list_and_get_history_sessions(
},
],
},
{"kind": "user", "text": "Which agent should be used?\nbeach-finder"},
{
"kind": "assistant",
"parts": [
Expand Down Expand Up @@ -478,6 +486,50 @@ def test_list_claude_skills_returns_claude_install_metadata(
assert response.json() == [_expected_inference_skill_response(installed=True)]


@pytest.mark.parametrize(
("tool_name", "tool_input", "result", "expected"),
[
(
"mcp__nemo_studio__select_agent",
{},
'{"status":"submitted","agent":"beach-finder"}',
"Selected agent: beach-finder",
),
(
"mcp__nemo_studio__select_model",
{"display_label": "Fallback model", "output_key": "fallback_model"},
'{"status":"submitted","fallback_model":"nemotron"}',
"Fallback model: nemotron",
),
(
"mcp__nemo_studio__select_dataset_file",
{},
'{"status":"submitted","dataset_fileset":"eval-data","dataset_path":"input.jsonl"}',
"Selected dataset: eval-data/input.jsonl",
),
(
"mcp__nemo_studio__select_eval_config",
{},
'{"status":"submitted","needs_eval_config":true}',
"I don't have an evaluation config yet",
),
],
)
def test_history_interaction_text_restores_studio_picker_submissions(
tool_name: str,
tool_input: dict[str, Any],
result: str,
expected: str,
):
assert (
coding_agents._history_interaction_text(
coding_agents.HistoryToolUse(name=tool_name, input=tool_input),
result,
)
== expected
)


def test_load_claude_skills_falls_back_on_duplicate_skill_error(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
Expand Down Expand Up @@ -632,6 +684,34 @@ def test_build_studio_system_prompt_preserves_empty_enabled_destinations():
assert destinations_line == "Enabled Studio link destinations for this Studio instance: ."


def test_build_studio_system_prompt_includes_message_summary_contract():
prompt = coding_agents._build_studio_system_prompt(
"default",
"https://studio.test",
"/workspaces/default/dashboard/code-agent",
{},
)

assert "Required message-summary behavior:" in prompt
assert coding_agents.STUDIO_MESSAGE_SUMMARY_START in prompt
assert coding_agents.STUDIO_MESSAGE_SUMMARY_END in prompt
assert "worked_for: <elapsed time if you know it, otherwise unknown>" in prompt
assert "summary: <concise Markdown" in prompt
assert "details_label: worked for <same elapsed time or unknown>" in prompt
assert "behind a 'worked for <time>' accordion" in prompt
assert "Never end a message with only a plain-text question" in prompt
assert "call the matching select_* tool before completing the message" in prompt
assert "use Claude Code's AskUserQuestion tool" in prompt
assert "For AskUserQuestion, provide input shaped as" in prompt
assert "A timeout, disconnect, or other interactive-tool error is not permission to continue" in prompt
assert "summary's final sentence MUST state the exact unresolved selection or action" in prompt
assert "Never show only the investigation result" in prompt
assert "use a numbered or bulleted list" in prompt
assert "repeat those links at the bottom of the summary" in prompt
assert "Put repeated links on separate lines without a heading" in prompt
assert "Do not omit the summary block because the message is short." in prompt


def test_studio_link_destinations_cover_registered_workspace_routes():
repo_root = Path(__file__).resolve().parents[4]
routes_index = (repo_root / "web/packages/studio/src/routes/index.tsx").read_text()
Expand Down Expand Up @@ -1282,6 +1362,81 @@ async def test_request_agent_input_rejects_reserved_response_keys():
}


async def test_permission_request_waits_until_user_resolves_it():
session_id = str(uuid.uuid4())
coding_agents._session_streams[session_id] = asyncio.Queue()

request_task = asyncio.create_task(
coding_agents._request_permission(
session_id,
{"tool_name": "AskUserQuestion", "input": {"question": "Continue?"}},
)
)
_, payload = await coding_agents._session_streams[session_id].get()
request_id = json.loads(payload)["request_id"]

await asyncio.sleep(0)
assert not request_task.done()

await coding_agents.resolve_permission(
session_id,
request_id,
coding_agents.PermissionDecision(approved=True),
)

assert await request_task == {"behavior": "allow", "updatedInput": {"question": "Continue?"}}


async def test_agent_input_request_cleans_up_when_wait_is_cancelled():
session_id = str(uuid.uuid4())
coding_agents._session_streams[session_id] = asyncio.Queue()

request_task = asyncio.create_task(coding_agents._request_agent_input(session_id, "agent", {}))
_, payload = await coding_agents._session_streams[session_id].get()
request_id = json.loads(payload)["request_id"]

assert request_id in coding_agents._pending_agent_inputs
request_task.cancel()
with pytest.raises(asyncio.CancelledError):
await request_task
assert request_id not in coding_agents._pending_agent_inputs


async def test_blocking_mcp_tool_response_streams_keepalives_until_user_responds():
session_id = str(uuid.uuid4())
coding_agents._session_streams[session_id] = asyncio.Queue()
result = asyncio.get_running_loop().create_future()

response = await coding_agents._blocking_mcp_tool_response(session_id, 7, result)

assert response.media_type == "text/event-stream"
assert response.headers["cache-control"] == "no-cache, no-transform"
assert response.headers["x-accel-buffering"] == "no"

iterator = response.body_iterator
assert await anext(iterator) == ": keepalive\n\n"

result.set_result({"status": "answered", "response": "A detailed answer"})
final_event = await anext(iterator)
assert final_event.startswith("event: message\ndata: ")
payload = json.loads(final_event.removeprefix("event: message\ndata: ").removesuffix("\n\n"))
assert payload == {
"jsonrpc": "2.0",
"id": 7,
"result": {
"content": [
{
"type": "text",
"text": json.dumps({"status": "answered", "response": "A detailed answer"}),
}
]
},
}

with pytest.raises(StopAsyncIteration):
await anext(iterator)


def test_platform_route_stream_uses_public_mcp_callback(monkeypatch: pytest.MonkeyPatch):
service = StudioService()
app = FastAPI()
Expand Down Expand Up @@ -1423,6 +1578,8 @@ def test_public_mcp_route_is_mounted_before_static_fallback():
f"/studio/api/coding-agents/mcp/{session_id}",
json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"},
)
get_response = client.get(f"/studio/api/coding-agents/mcp/{session_id}")
delete_response = client.delete(f"/studio/api/coding-agents/mcp/{session_id}")

assert response.status_code == 200
assert [tool["name"] for tool in response.json()["result"]["tools"]] == [
Expand All @@ -1434,6 +1591,10 @@ def test_public_mcp_route_is_mounted_before_static_fallback():
"job_progress",
"studio_link",
]
assert get_response.status_code == 405
assert get_response.headers["allow"] == "POST"
assert delete_response.status_code == 405
assert delete_response.headers["allow"] == "POST"


def test_coding_agent_routes_are_available_by_default():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,34 @@ describe('ClaudeCodeHistoryPanel', () => {
);

expect(screen.getByText('Jobs')).toBeInTheDocument();
expect(screen.queryByText('Workspace')).not.toBeInTheDocument();
expect(screen.getByRole('link', { name: /agent-eval-1/ })).toHaveAttribute(
'href',
'/workspaces/default/agents/evaluations/agent-eval-1'
);
});

it('does not treat workspace metadata as a visible chat artifact', () => {
render(
<ClaudeCodeHistoryPanel
activeSessionId="session-1"
artifacts={{
workspace: 'default',
selections: [],
files: [],
links: [],
jobs: [],
tools: [],
}}
onNewChat={vi.fn()}
onSelectSession={vi.fn()}
/>
);

expect(screen.queryByText('Workspace')).not.toBeInTheDocument();
expect(screen.getByText('No artifacts yet')).toBeInTheDocument();
});

it('lists Claude Code skills in the skills tab', async () => {
const user = userEvent.setup();
render(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
CLAUDE_CODE_JOB_PROGRESS_TOOL_NAME,
} from '@studio/routes/agents/ClaudeCodeChatRoute/jobProgressConsts';
import {
CLAUDE_CODE_COLLAPSED_STUDIO_DETAILS_TOOL_NAME,
CLAUDE_CODE_COLLAPSED_THINKING_TOOL_NAME,
CLAUDE_CODE_SUBTLE_TOOL_GROUP_NAME,
} from '@studio/routes/agents/ClaudeCodeChatRoute/toolParts';
Expand Down Expand Up @@ -111,6 +112,63 @@ const expectFileChangeBlockFullWidth = () => {
};

describe('ClaudeCodeToolCallPart', () => {
it('renders Studio summary details behind a worked-for disclosure', async () => {
const user = userEvent.setup();

render(
<ClaudeCodeToolCallPart
addResult={vi.fn()}
args={{
label: 'worked for 42s',
parts: [
{
type: 'text',
text: [
'## Optimization report',
'',
'**Current config:** `meta-llama-3-1-70b-instruct`',
'',
'- Preserved first suggestion',
'- Preserved second suggestion',
].join('\n'),
},
{
type: 'tool-call',
args: { command: 'pwd' },
argsText: '{"command":"pwd"}',
toolCallId: 'toolu_bash',
toolName: 'Bash',
},
],
}}
argsText=""
resume={vi.fn()}
status={{ type: 'complete' }}
toolCallId="claude-code-collapsed-studio-details"
toolName={CLAUDE_CODE_COLLAPSED_STUDIO_DETAILS_TOOL_NAME}
type="tool-call"
/>
);

const disclosure = screen.getByTestId('claude-code-collapsed-studio-details');
expect(disclosure).toHaveTextContent('worked for 42s');
expect(disclosure).not.toHaveAttribute('open');
expect(screen.getByTestId('claude-code-collapsed-studio-details-content')).toHaveTextContent(
'Optimization report'
);
expect(screen.getByTestId('claude-code-collapsed-studio-details-content')).toHaveTextContent(
'Ran pwd'
);

await user.click(screen.getByText('worked for 42s'));

expect(disclosure).toHaveAttribute('open');
expect(screen.getByRole('heading', { level: 2, name: 'Optimization report' })).toBeVisible();
expect(screen.getByText('Current config:')).toHaveProperty('tagName', 'STRONG');
expect(screen.getByText('meta-llama-3-1-70b-instruct')).toHaveProperty('tagName', 'CODE');
expect(screen.getAllByRole('listitem')[0]).toHaveTextContent('Preserved first suggestion');
});

it('renders collapsed thinking as an expandable subtle disclosure', async () => {
const user = userEvent.setup();

Expand Down Expand Up @@ -144,6 +202,27 @@ describe('ClaudeCodeToolCallPart', () => {
);
});

it('replaces a persisted unknown work time with a neutral label', () => {
render(
<ClaudeCodeToolCallPart
addResult={vi.fn()}
args={{
label: 'worked for unknown',
parts: [{ type: 'text', text: 'Completed work.' }],
}}
argsText=""
resume={vi.fn()}
status={{ type: 'complete' }}
toolCallId="claude-code-collapsed-studio-details"
toolName={CLAUDE_CODE_COLLAPSED_STUDIO_DETAILS_TOOL_NAME}
type="tool-call"
/>
);

expect(screen.getByText('Work details')).toBeVisible();
expect(screen.queryByText('worked for unknown')).not.toBeInTheDocument();
});

it.each(subtleToolCases)(
'renders $toolName as subtle text',
({ args, expectedText, toolName }) => {
Expand Down
Loading