Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 90 additions & 8 deletions libs/code/deepagents_code/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -10144,6 +10144,26 @@ def _goal_application_matches_proposal(
and application.request_id == proposal.request_id
)

def _has_clearable_goal_rubric_state(self) -> bool:
"""Return whether `/rubric clear` would drop any correlated local state.

Pending proposals and accepted-but-queued applications can be live while
`_active_rubric` / `_next_rubric` are still empty, so the empty-check for
clear must look beyond those two user-visible fields.
"""
return bool(
self._active_rubric
or self._next_rubric
or self._active_goal
or self._goal_status_note
or self._pending_goal_completion_note
or self._pending_goal_objective
or self._pending_goal_rubric
or self._queued_goal_application is not None
or self._last_consumed_next_rubric is not None
or self._last_consumed_next_previous_rubric is not None
)

def _clear_all_goal_rubric_state(self) -> None:
"""Clear every goal and rubric field (sticky, one-shot, goal, pending).

Expand Down Expand Up @@ -11768,7 +11788,7 @@ async def _handle_rubric_command(self, command: str) -> None:
if subcommand == "set":
await self._mount_message(UserMessage(command))
if not arg:
await self._mount_message(AppMessage("Usage: /rubric set <criteria>"))
await self._mount_message(AppMessage(self._rubric_set_usage_text()))
return
async with self._goal_state_mutation_boundary():
previous_state = self._goal_state_update()
Expand All @@ -11787,7 +11807,7 @@ async def _handle_rubric_command(self, command: str) -> None:
if subcommand == "next":
await self._mount_message(UserMessage(command))
if not arg:
await self._mount_message(AppMessage("Usage: /rubric next <criteria>"))
await self._mount_message(AppMessage(self._rubric_next_usage_text()))
return
self._next_rubric = arg
self._sync_status_rubric()
Expand All @@ -11797,13 +11817,18 @@ async def _handle_rubric_command(self, command: str) -> None:
if subcommand == "file":
await self._mount_message(UserMessage(command))
if not arg:
await self._mount_message(AppMessage("Usage: /rubric file <path>"))
await self._mount_message(AppMessage(self._rubric_file_usage_text()))
return
await self._set_rubric_from_file(arg)
return

if subcommand == "clear":
await self._mount_message(UserMessage(command))
if not self._has_clearable_goal_rubric_state():
await self._mount_message(
AppMessage("No rubric set. Nothing to clear.")
)
return
async with self._goal_state_mutation_boundary():
previous_state = self._goal_state_update()
self._clear_all_goal_rubric_state()
Expand Down Expand Up @@ -11842,7 +11867,46 @@ def _rubric_usage_text() -> str:
" /rubric model [provider:model|clear]\n"
" /rubric max-iterations <N|clear>\n\n"
"Use /rubric next for a one-turn quality gate. Use /rubric set "
"when you want explicit acceptance criteria to persist across turns."
"when you want explicit acceptance criteria to persist across turns.\n"
"Example: /rubric set tests pass; keep the diff minimal"
)

@staticmethod
def _rubric_set_usage_text() -> str:
"""Return usage for bare `/rubric set`."""
return (
"Usage: /rubric set <criteria>\n\n"
"Keep these acceptance criteria across turns.\n"
"Graded after each agent response until you clear or replace them.\n"
"Example: /rubric set tests pass; keep the diff minimal"
)

@staticmethod
def _rubric_next_usage_text() -> str:
"""Return usage for bare `/rubric next`."""
return (
"Usage: /rubric next <criteria>\n\n"
"Use these acceptance criteria for the next agent turn only.\n"
"Example: /rubric next tests pass; no unrelated changes"
)

@staticmethod
def _rubric_file_usage_text() -> str:
"""Return usage for bare `/rubric file`."""
return (
"Usage: /rubric file <path>\n\n"
"Load acceptance criteria from a file (same as /rubric set).\n"
"Example: /rubric file ./rubric.md"
)

@staticmethod
def _rubric_empty_state_text() -> str:
"""Return the empty-state message for `/rubric show`."""
return (
"No rubric set.\n\n"
"Set one with `/rubric set <criteria>`, or load a file:\n"
" /rubric set tests pass; keep the diff minimal\n"
" /rubric file ./rubric.md"
)

async def _show_rubric_usage(self) -> None:
Expand All @@ -11856,13 +11920,15 @@ async def _show_rubric_usage(self) -> None:
):
state: list[str] = []
if self._active_rubric:
state.append("Sticky rubric is set.")
state.append("Rubric is set.")
if self._next_rubric:
state.append("Next-turn rubric is set.")
if self._rubric_model:
state.append(f"Rubric grader model: {self._rubric_model}")
if self._rubric_max_iterations is not None:
state.append(f"Rubric max iterations: {self._rubric_max_iterations}")
if self._active_rubric or self._next_rubric:
state.append("Use /rubric show to view.")
parts.append(
"Current state:\n" + "\n".join(f" - {line}" for line in state)
)
Expand All @@ -11875,8 +11941,24 @@ async def _show_rubric_state(self) -> None:
lines.append(f"Rubric:\n{self._active_rubric}")
if self._next_rubric:
lines.append(f"Next-turn rubric:\n{self._next_rubric}")
if not lines and not self._rubric_model and self._rubric_max_iterations is None:
await self._mount_message(AppMessage("No rubric set."))
if not lines:
# Grader settings can exist without criteria; still teach how to set
# a rubric when nothing is grading yet.
if self._rubric_model or self._rubric_max_iterations is not None:
grader_model, grader_iterations = self._grader_display_values()
await self._mount_message(
AppMessage(
"\n\n".join(
[
self._rubric_empty_state_text(),
f"Rubric grader model: {grader_model}",
f"Rubric max iterations: {grader_iterations}",
]
)
)
)
return
await self._mount_message(AppMessage(self._rubric_empty_state_text()))
return
grader_model, grader_iterations = self._grader_display_values()
lines.extend(
Expand All @@ -11895,7 +11977,7 @@ async def _set_rubric_from_file(self, path_arg: str) -> None:
await self._mount_message(ErrorMessage(f"Could not parse path: {exc}"))
return
if len(parts) != 1:
await self._mount_message(AppMessage("Usage: /rubric file <path>"))
await self._mount_message(AppMessage(self._rubric_file_usage_text()))
return
try:
path, text = await asyncio.to_thread(
Expand Down
118 changes: 117 additions & 1 deletion libs/code/tests/unit_tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -10867,14 +10867,18 @@ async def test_bare_rubric_appends_current_state_when_set(self) -> None:
async with app.run_test() as pilot:
await pilot.pause()
app._active_rubric = "tests pass"
app._next_rubric = "diff is minimal"
app._rubric_model = "anthropic:claude-sonnet-4-6"

await app._handle_command("/rubric")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Current state:" in rendered
assert "Sticky rubric is set." in rendered
assert "Rubric is set." in rendered
assert "Next-turn rubric is set." in rendered
assert "Use /rubric show to view." in rendered
assert "Sticky rubric" not in rendered
assert "Rubric grader model: anthropic:claude-sonnet-4-6" in rendered

async def test_unknown_rubric_subcommand_shows_usage(self) -> None:
Expand Down Expand Up @@ -10918,8 +10922,61 @@ async def test_rubric_show_without_rubric_reports_empty_state(self) -> None:

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "No rubric set." in rendered
assert "/rubric set <criteria>" in rendered
assert "Rubric grader model:" not in rendered

async def test_rubric_set_without_criteria_shows_usage_tip(self) -> None:
"""Bare `/rubric set` should include a short example tip."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
await app._handle_command("/rubric set")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Usage: /rubric set <criteria>" in rendered
assert "Keep these acceptance criteria across turns." in rendered
assert (
"Graded after each agent response until you clear or replace them."
in rendered
)
assert "Example: /rubric set tests pass; keep the diff minimal" in rendered
assert app._active_rubric is None

async def test_rubric_next_without_criteria_shows_usage_tip(self) -> None:
"""Bare `/rubric next` should include a one-turn tip and example."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
await app._handle_command("/rubric next")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Usage: /rubric next <criteria>" in rendered
assert (
"Use these acceptance criteria for the next agent turn only."
in rendered
)
assert "Example: /rubric next tests pass; no unrelated changes" in rendered
assert app._next_rubric is None

async def test_rubric_file_without_path_shows_usage_tip(self) -> None:
"""Bare `/rubric file` should include a path example."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
await app._handle_command("/rubric file")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Usage: /rubric file <path>" in rendered
assert (
"Load acceptance criteria from a file (same as /rubric set)."
in rendered
)
assert "Example: /rubric file ./rubric.md" in rendered
assert app._active_rubric is None

async def test_rubric_set_passes_sticky_rubric_to_turn(self) -> None:
"""`/rubric set` should apply to subsequent TUI agent turns."""
app = DeepAgentsApp(agent=MagicMock())
Expand Down Expand Up @@ -11508,6 +11565,7 @@ async def test_rubric_file_rejects_multiple_path_tokens(self) -> None:

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Usage: /rubric file <path>" in rendered
assert "Example: /rubric file ./rubric.md" in rendered
assert app._active_rubric is None

async def test_rubric_file_reports_missing_file(self, tmp_path: Path) -> None:
Expand Down Expand Up @@ -12184,6 +12242,64 @@ async def test_rubric_clear_resets_status(self) -> None:
assert app._next_rubric is None
assert app._status_bar is not None
assert app._status_bar.rubric_label == ""
rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric cleared." in rendered

async def test_rubric_clear_without_rubric_is_noop(self) -> None:
"""`/rubric clear` with no criteria should not claim a rubric was cleared."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
await app._handle_command("/rubric clear")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "No rubric set. Nothing to clear." in rendered
assert "Rubric cleared." not in rendered
assert app._active_rubric is None
assert app._next_rubric is None

async def test_rubric_clear_drops_pending_goal_when_no_visible_rubric(
self,
) -> None:
"""Pending goal criteria still clear even when sticky/next are empty."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._pending_goal_objective = "ship login"
app._pending_goal_rubric = "- tests pass"
app._pending_goal_kind = "create"

await app._handle_command("/rubric clear")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric cleared." in rendered
assert "No rubric set. Nothing to clear." not in rendered
assert app._pending_goal_objective is None
assert app._pending_goal_rubric is None
assert app._pending_goal_kind is None

async def test_rubric_clear_drops_queued_goal_when_no_visible_rubric(
self,
) -> None:
"""Accepted-but-queued goals still clear even when sticky/next are empty."""
app = DeepAgentsApp(agent=MagicMock())
async with app.run_test() as pilot:
await pilot.pause()
app._queued_goal_application = _GoalApplication(
"ship login",
"- tests pass",
"create",
)

await app._handle_command("/rubric clear")
await pilot.pause()

rendered = "\n".join(str(w._content) for w in app.query(AppMessage))
assert "Rubric cleared." in rendered
assert "No rubric set. Nothing to clear." not in rendered
assert app._queued_goal_application is None

async def test_clear_command_clears_rubric_state(self) -> None:
"""Starting a new thread should not carry hidden rubric behavior."""
Expand Down