Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions agent/agent_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -1198,6 +1198,17 @@ def init_agent(
_agent_section = {}
agent._tool_use_enforcement = _agent_section.get("tool_use_enforcement", "auto")

# Execution-discipline gate. Controls whether OPENAI_MODEL_EXECUTION_GUIDANCE
# is appended on top of TOOL_USE_ENFORCEMENT_GUIDANCE for a given session.
# Mirrors the tool_use_enforcement value semantics:
# "auto" (default) — match against OPENAI_EXECUTION_DISCIPLINE_MODELS
# true / "true" / "always" / "yes" / "on" — always inject
# false / "false" / "never" / "no" / "off" — never inject
# list of substrings — custom model-name match list
# See agent/system_prompt.py for the gate logic and prompt_builder.py
# for the model substring tuple.
agent._execution_discipline = _agent_section.get("execution_discipline", "auto")

# Universal task-completion guidance toggle. Default True. Surfaced
# as a separate flag from tool_use_enforcement because the guidance
# applies to ALL models, not just the model families enforcement
Expand Down
25 changes: 25 additions & 0 deletions agent/prompt_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,31 @@ def _strip_yaml_frontmatter(content: str) -> str:
# Add new patterns here when a model family needs explicit steering.
TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", "qwen", "deepseek")

# Model name substrings that, in addition to the tier-1 tool-use enforcement
# block, also receive the longer OPENAI_MODEL_EXECUTION_GUIDANCE block (tool
# persistence, mandatory tool use, prerequisite checks, verification,
# anti-hallucination).
#
# History: started as a hard-coded {gpt, codex} substring check in
# system_prompt.py. PR #27797 extended it to {grok} after observing that
# xAI Grok / xai-oauth models exhibit the same failure modes — claiming
# completion without tool calls, suggesting workarounds instead of using
# existing tools.
#
# Local-model users running Qwen, DeepSeek, or GLM via oMLX / LM Studio /
# OpenRouter hit the same failure modes (e.g. skipping later phases of a
# multi-phase workflow after declaring an earlier phase "done"). Adding
# these families here gives them the same discipline injection.
#
# Override via config.yaml `agent.execution_discipline`:
# "auto" (default) — match against this tuple
# true — always inject
# false — never inject
# [list] — custom substring list
OPENAI_EXECUTION_DISCIPLINE_MODELS = (
"gpt", "codex", "grok", "qwen", "deepseek", "glm",
)

# Universal "finish the job" guidance — applied to ALL models, not gated
# by model family. Addresses two cross-model failure modes:
# 1. Stopping after a stub: writing a tiny file or running one command
Expand Down
35 changes: 30 additions & 5 deletions agent/system_prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,12 +168,37 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None)
# paths, parallel tool calls, verify-before-edit, etc.)
if "gemini" in _model_lower or "gemma" in _model_lower:
stable_parts.append(GOOGLE_MODEL_OPERATIONAL_GUIDANCE)
# OpenAI GPT/Codex execution discipline (tool persistence,
# OpenAI-style execution discipline (tool persistence,
# prerequisite checks, verification, anti-hallucination).
# Also applied to xAI Grok — same failure modes (claims completion
# without tool calls, suggests workarounds instead of using
# existing tools, replies with plans instead of executing).
if "gpt" in _model_lower or "codex" in _model_lower or "grok" in _model_lower:
# Originally for GPT/Codex; extended to xAI Grok (PR #27797)
# after observing the same failure modes; now also covers
# Qwen / DeepSeek / GLM by default for local-model parity.
# Config-driven gate mirrors agent.tool_use_enforcement
# semantics (auto/true/false/list).
from agent.prompt_builder import OPENAI_EXECUTION_DISCIPLINE_MODELS
_discipline = getattr(agent, "_execution_discipline", "auto")
_discipline_inject = False
if _discipline is True or (
isinstance(_discipline, str)
and _discipline.lower() in {"true", "always", "yes", "on"}
):
_discipline_inject = True
elif _discipline is False or (
isinstance(_discipline, str)
and _discipline.lower() in {"false", "never", "no", "off"}
):
_discipline_inject = False
elif isinstance(_discipline, list):
_discipline_inject = any(
isinstance(p, str) and p.lower() in _model_lower
for p in _discipline
)
else:
# "auto" or any unrecognised value — fall back to defaults
_discipline_inject = any(
p in _model_lower for p in OPENAI_EXECUTION_DISCIPLINE_MODELS
)
if _discipline_inject:
stable_parts.append(OPENAI_MODEL_EXECUTION_GUIDANCE)

has_skills_tools = any(name in agent.valid_tool_names for name in ['skills_list', 'skill_view', 'skill_manage'])
Expand Down
105 changes: 102 additions & 3 deletions tests/run_agent/test_run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1150,8 +1150,21 @@ def test_skills_prompt_derives_available_toolsets_from_loaded_tools(self):
class TestToolUseEnforcementConfig:
"""Tests for the agent.tool_use_enforcement config option."""

def _make_agent(self, model="openai/gpt-4.1", tool_use_enforcement="auto"):
"""Create an agent with tools and a specific enforcement config."""
def _make_agent(
self,
model="openai/gpt-4.1",
tool_use_enforcement="auto",
execution_discipline=None,
):
"""Create an agent with tools and a specific enforcement config.

``execution_discipline`` mirrors ``tool_use_enforcement`` semantics:
``None`` means "do not set the key" (use the production default of
"auto"); ``True``/``False``/string/list values are passed through.
"""
agent_cfg = {"tool_use_enforcement": tool_use_enforcement}
if execution_discipline is not None:
agent_cfg["execution_discipline"] = execution_discipline
with (
patch(
"run_agent.get_tool_definitions",
Expand All @@ -1161,7 +1174,7 @@ def _make_agent(self, model="openai/gpt-4.1", tool_use_enforcement="auto"):
patch("run_agent.OpenAI"),
patch(
"hermes_cli.config.load_config",
return_value={"agent": {"tool_use_enforcement": tool_use_enforcement}},
return_value={"agent": agent_cfg},
),
):
a = AIAgent(
Expand Down Expand Up @@ -1241,6 +1254,92 @@ def test_auto_does_not_inject_execution_guidance_for_claude(self):
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE not in prompt

# ── Execution discipline extension: Qwen / DeepSeek / GLM families ──
# The previous behaviour hard-coded substring matching against
# {gpt, codex, grok}. Local-model users running Qwen / DeepSeek / GLM
# hit the same failure modes (claiming completion without tool calls,
# skipping prerequisite phases) but received no discipline injection.
# See feat/qwen-execution-discipline.

def test_auto_injects_execution_guidance_for_qwen(self):
"""Qwen models hit the same 'claims done without tool calls' pattern.
Verified empirically on Qwen3.5-35B via oMLX during a 7-phase
COBOL→Java translation where Phase 5/6 were skipped after Phase 4.
"""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="qwen/qwen-plus", tool_use_enforcement="auto"
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_auto_injects_execution_guidance_for_qwen_bare_local_name(self):
"""Local oMLX / LM Studio users often use bare model names like
'qwen3-coder-next' or 'MLX-Qwen3.5-35B-A3B-...' without provider slash.
Substring match must still catch these."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="MLX-Qwen3.5-35B-A3B-Claude-4.6-Opus-Reasoning-Distilled-8bit",
tool_use_enforcement="auto",
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_auto_injects_execution_guidance_for_deepseek(self):
"""DeepSeek reasoning models exhibit empty-response and over-confident
completion patterns documented in v0.13.0 release notes."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="deepseek/deepseek-r1", tool_use_enforcement="auto"
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_auto_injects_execution_guidance_for_glm(self):
"""GLM family also benefits from the discipline block (same
precedent as Grok extension in #27797)."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="glm-4.6", tool_use_enforcement="auto"
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_execution_discipline_explicit_off_for_qwen(self):
"""When execution_discipline=false, do not inject even for matched models."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="qwen/qwen-plus",
tool_use_enforcement="auto",
execution_discipline=False,
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE not in prompt

def test_execution_discipline_explicit_on_for_claude(self):
"""When execution_discipline=true, inject for any model (escape
hatch for users whose local model has a renamed identifier that
doesn't substring-match the default list)."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="anthropic/claude-sonnet-4",
tool_use_enforcement=True, # required for any injection
execution_discipline=True,
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_execution_discipline_custom_list(self):
"""Custom list lets users add new local-model substrings."""
from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE
agent = self._make_agent(
model="my-custom-7b",
tool_use_enforcement=True,
execution_discipline=["my-custom"],
)
prompt = agent._build_system_prompt()
assert OPENAI_MODEL_EXECUTION_GUIDANCE in prompt

def test_true_forces_for_all_models(self):
from agent.prompt_builder import TOOL_USE_ENFORCEMENT_GUIDANCE
agent = self._make_agent(model="anthropic/claude-sonnet-4", tool_use_enforcement=True)
Expand Down
30 changes: 28 additions & 2 deletions website/docs/user-guide/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1121,7 +1121,7 @@ agent:

| Value | Behavior |
|-------|----------|
| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `gemini`, `gemma`, `grok`. Disabled for all others (Claude, DeepSeek, Qwen, etc.). |
| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `gemini`, `gemma`, `grok`, `glm`, `qwen`, `deepseek`. Disabled for Claude and other families. |
| `true` | Always enabled, regardless of model. Useful if you notice your current model describing actions instead of performing them. |
| `false` | Always disabled, regardless of model. |
| `["gpt", "codex", "qwen", "llama"]` | Enabled only when the model name contains one of the listed substrings (case-insensitive). |
Expand All @@ -1132,7 +1132,7 @@ When enabled, three layers of guidance may be added to the system prompt:

1. **General tool-use enforcement** (all matched models) — instructs the model to make tool calls immediately instead of describing intentions, keep working until the task is complete, and never end a turn with a promise of future action.

2. **OpenAI execution discipline** (GPT and Codex models only) — additional guidance addressing GPT-specific failure modes: abandoning work on partial results, skipping prerequisite lookups, hallucinating instead of using tools, and declaring "done" without verification.
2. **OpenAI execution discipline** (controlled separately by `agent.execution_discipline`; default-on for `gpt`, `codex`, `grok`, `qwen`, `deepseek`, `glm`) — addresses cross-model failure modes: abandoning work on partial results, skipping prerequisite lookups, hallucinating instead of using tools, and declaring "done" without verification.

3. **Google operational guidance** (Gemini and Gemma models only) — conciseness, absolute paths, parallel tool calls, and verify-before-edit patterns.

Expand All @@ -1147,6 +1147,32 @@ agent:
tool_use_enforcement: ["gpt", "codex", "gemini", "grok", "my-custom-model"]
```

## Execution Discipline (Layered on Tool-Use Enforcement)

`agent.execution_discipline` toggles the longer OpenAI-style execution-discipline block. It only fires when tool-use enforcement is already active for the same session (so it inherits the `tool_use_enforcement` gate first, then applies its own model-family check on top).

```yaml
agent:
execution_discipline: "auto" # "auto" | true | false | ["model-substring", ...]
```

| Value | Behavior |
|-------|----------|
| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `grok`, `qwen`, `deepseek`, `glm`. Disabled for Claude, Gemini, Gemma, and others. |
| `true` | Always inject when tool-use enforcement is active (useful for locally renamed models whose identifier doesn't substring-match the default list). |
| `false` | Never inject. |
| `["qwen", "my-local-model"]` | Inject only for models whose name contains one of the listed substrings (case-insensitive). |

### What it injects

A block called **Execution discipline** with five sections: `<tool_persistence>` (keep calling tools until done + verified), `<mandatory_tool_use>` (never compute from memory when a tool is available), `<act_dont_ask>` (clarifying questions only when they change which tool you'd call), `<prerequisite_checks>` (resolve dependencies before action), `<verification>` (sanity-check correctness/grounding/format/safety before finalising), and `<missing_context>` (look up missing info, never hallucinate it).

### When to override the auto behavior

- **Locally-renamed models** (e.g. an oMLX or LM Studio model loaded under `my-custom-7b`) — the substring match won't catch them. Set `execution_discipline: true` or add the name to the list.
- **You're confident your model doesn't need it** — set `execution_discipline: false` to save the prompt tokens.
- **You want to apply it to Claude or Gemini selectively** — set `execution_discipline: true` (combined with `tool_use_enforcement: true`).

## TTS Configuration

```yaml
Expand Down