diff --git a/HERMES_RESEARCH.md b/HERMES_RESEARCH.md new file mode 100644 index 000000000000..f922c4ef3532 --- /dev/null +++ b/HERMES_RESEARCH.md @@ -0,0 +1,202 @@ +# Hermes AutoResearch + +## What This Is + +Hermes AutoResearch is the **Karpathy inner loop** for autonomous experimentation inside Hermes. Given a research topic, it runs a baseline experiment, proposes improvements via LLM, executes them through `delegate_task`, keeps improvements and discards regressions, and records structured learnings. + +The architecture is **desacoplada**: long-running research loops execute as independent OS processes with durable checkpoints, so the parent agent does not burn iteration budget or die to timeouts. + +## Quick Start + +### Running a Research Job (Detached) + +```bash +# Create a job spec JSON +python -c ' +import json +spec = { + "job_id": "my-research", + "job_dir": "/home/user/.hermes/research-jobs/my-research", + "model": "kimi-for-coding", + "provider": "kimi-coding", + "topic": "Analyze WebAssembly adoption in 2025", + "deliverable": "Ranked list of relevant papers with abstracts", + "metric_key": "completeness_score", + "metric_direction": "maximize", + "task_type": "research", + "max_iterations": 3, +} +json.dump(spec, open("/home/user/.hermes/research-jobs/my-research/job.json", "w")) +' + +# Launch detached runner +source venv/bin/activate +HERMES_YOLO_MODE=1 python -m agent.research.job_runner \ + /home/user/.hermes/research-jobs/my-research/job.json +``` + +### From Python (Synchronous) + +```python +from agent.research.supervisor import ResearchSupervisor, TaskSpec +from pathlib import Path + +spec = TaskSpec( + topic="Analyze WebAssembly adoption in 2025", + deliverable="Ranked list of relevant papers with abstracts", + metric_key="completeness_score", + metric_direction="maximize", + task_type="research", +) + +supervisor = ResearchSupervisor(parent_agent=agent, workspace=Path("research-workspace")) +history = supervisor.run( + spec, + initial_attempt="", + run_id="run-001", + max_iterations=3, + llm=agent.llm_client, +) +``` + +## Architecture + +``` +Parent Agent / CLI + │ + ▼ +┌─────────────────────────┐ +│ research/job_runner.py│ ← Detached OS process +│ (entrypoint) │ +└─────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ ResearchSupervisor │ ← Karpathy loop orchestrator +│ • TaskSpec │ +│ • run() │ +│ • _observe() │ +│ • _checkpoint() │ +└─────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ ┌─────────────────────────┐ +│ delegate_task │────▶│ Worker Subagent │ +│ (per iteration) │ │ • Reads task_brief.md │ +└─────────────────────────┘ │ • Writes attempt.md │ + │ • Writes results.json │ + │ • Reports metric │ + └─────────────────────────┘ +``` + +## Project Structure + +``` +agent/ +├── research/job_runner.py # Detached entrypoint: builds AIAgent, calls run_research +├── research/supervisor.py # ResearchSupervisor + TaskSpec + task briefs +├── research/runner.py # ExperimentRunner + ExperimentHistory +├── research/metrics.py # UniversalMetricParser +└── subdirectory_hints.py # Progressive context discovery (cached) + +tools/ +├── research_tool.py # run_research() public API +└── research_job_tool.py # start_research_job, research_job_status, collect_research_job + +~/.hermes/research-jobs/ # Job specs + checkpoints + logs +~/.hermes/research-workspace/ # Round artifacts (attempt.md, results.json, learnings.jsonl) +``` + +## The Karpathy Loop + +``` +Step 1: BASELINE — Worker receives task brief + attempt file, produces deliverable +Step 2: METRIC — UniversalMetricParser reads results.json / stdout +Step 3: JUDGE — LLM judge scores deliverable (if evaluation_mode="llm_judge") +Step 4: OBSERVE — Structured learning appended to learnings.jsonl +Step 5: CHECKPOINT — history.json + checkpoint.json written to disk +Step 6: OPTIMIZE — LLM proposes revised attempt based on history +Step 7: KEEP/DISCARD — If metric improved: keep, else discard; iterate +``` + +## Task Types + +| Type | Default Toolsets | Deliverable | Attempt File | +|------|-----------------|-------------|--------------| +| `code` | terminal, file | Python code | attempt.py | +| `search` | web, terminal, file | Search results | attempt.md | +| `research` | web, terminal, file | Synthesis | attempt.md | +| `generic` | terminal, file | Any text | attempt.md | + +## Worker Contract + +The worker receives: +- `task_brief.md` — Full instructions including think block, rules, tools available +- `attempt.py` or `attempt.md` — Current attempt to refine +- Environment variable `HERMES_YOLO_MODE=1` to skip command approval + +The worker must produce: +- `results.json` with `{"": }` +- Final line: `METRIC: = STATUS: improved|regressed|neutral NOTES: ` + +## Checkpoints and Recovery + +After every round, the supervisor writes: + +``` +~/.hermes/research-jobs// +├── checkpoint.json # {round, total_rounds, best_metric, updated_at} +├── history.json # Full results array + best reference +├── runner.log # Runner + supervisor logs +└── state.json # {status, pid, started_at} +``` + +External monitors can read `checkpoint.json` without polling the process. + +## Decision Guide + +| Situation | Action | +|-----------|--------| +| Long-running research (>5 min) | Use `research/job_runner` detached | +| Quick experiment (<2 min) | Call `run_research()` directly | +| Need baseline only | Set `llm=None` in supervisor | +| Worker times out | `DelegateSandboxResult.timed_out=True`; loop continues | +| 3 consecutive non-improving | Runner stops early (or 1 if high baseline) | +| Want to inspect history | Read `history.json` from checkpoint dir | + +## Performance Optimizations + +| Optimization | File | Impact | +|-------------|------|--------| +| **Lock file** | `research/job_runner.py` | Prevents duplicate restarts (~16 min saved) | +| **Provider cache** | `auxiliary_client.py` | Caches `resolve_provider_client` (~14 calls → 1) | +| **Subdirectory hints cache** | `subdirectory_hints.py` | Caches hint loads per directory | +| **Aggressive early stop** | `research/supervisor.py` | Baseline ≥0.9 → stop after 1 non-improving iter | +| **LLM judge every iter** | `research/supervisor.py` | Objective scoring on all loops | + +## Anti-Patterns + +- **DO NOT** run `research/job_runner` in foreground without `timeout >= 300` +- **DO NOT** poll the process with `ps` / `tail` — read `checkpoint.json` instead +- **DO NOT** launch the same job twice — the lock file prevents this +- **DO NOT** delete `.runner.lock` manually — use `kill` on the process + +## Metric Reporting (Worker Contract) + +Workers must print metrics in one of these formats: + +``` +# Hermes format (preferred) +METRIC: accuracy=0.923 STATUS: improved NOTES: Adam lr=0.001 beat SGD baseline + +# Standard key: value format +accuracy: 0.923 +loss: 0.112 +``` + +The `UniversalMetricParser` also reads `results.json` (structured) or `results.csv` if present in the round directory. + +## Skills + +Hermes AutoResearch skills are in `skills/autoresearch/` and are loaded automatically. +Domain-specific skills (ML, chemistry, biology) are in `skills/autoresearch/domain/`. diff --git a/RESEARCH_AGENTS.md b/RESEARCH_AGENTS.md new file mode 100644 index 000000000000..6e3374c5013c --- /dev/null +++ b/RESEARCH_AGENTS.md @@ -0,0 +1,105 @@ +# Hermes AutoResearch — Worker Agent Contract + +## Overview + +You are a **Hermes AutoResearch worker**. You receive a goal string and a working directory from the supervisor. Your job is to run the experiment described in `task_brief.md` and report a metric in the required format. + +You are NOT responsible for the loop logic (keep/discard, iteration, LLM code improvement). That is handled by the supervisor via `ResearchSupervisor`. + +## Inputs + +| Input | Source | Description | +|-------|--------|-------------| +| Working directory | `delegate_task` argument | Directory containing `task_brief.md` and `attempt` file | +| Goal string | `delegate_task` argument | Includes metric key and output format | +| `task_brief.md` | Read from working directory | Full instructions, think block, rules, tools available | +| `attempt.py` / `attempt.md` | Read from working directory | Current attempt to refine (iteration > 0) or baseline seed | + +## Your Steps + +1. **Read `task_brief.md`** — understand the experiment goal, deliverable, and metric key +2. **Read the attempt file** — see what the previous iteration produced +3. **Set up experiment files** — write refined code/synthesis to the working directory +4. **Run the experiment** — execute the code, collect results, verify metric +5. **Write `results.json`** with `{"": }` (structured output, preferred) +6. **Print metric line** — required for fallback stdout parsing +7. **Report status** — include STATUS word in output + +## Required Output Format + +Your final output MUST include a metric line in one of these formats: + +``` +# Preferred (Hermes format) +METRIC: = STATUS: improved|regressed|neutral NOTES: + +# Acceptable (standard) +: +``` + +Example: +``` +METRIC: completeness_score=0.95 STATUS: improved NOTES: Covered WebAssembly browser support, non-browser runtimes, and language bindings +``` + +The metric key must match the key specified in the goal string (e.g., `completeness_score`, `accuracy`, `pass_rate`). + +## Tool Format + +When calling tools, use the **JSON format** provided by the system. Do NOT use XML tags like ``. + +## Tools Available + +The task brief declares available tools explicitly. Common sets: + +| Task Type | Tools | +|-----------|-------| +| code | terminal, file, code_execution | +| search | web_search, browser, file, terminal | +| research | web_search, browser, file, terminal | +| generic | terminal, file, code_execution | + +Use these actively — do NOT assume they are unavailable. + +## Stopping Conditions + +Stop and report when ANY of the following occurs: + +- Experiment completes successfully — report final metric +- Time budget exceeded (check `TIME_ESTIMATE` vs elapsed) — report partial results +- Unrecoverable error — report `STATUS: regressed` with error in NOTES +- Code validation fails after 3 auto-repair attempts — report failure + +Do NOT loop indefinitely. The supervisor handles retry logic. + +## Lattice State Transitions + +You do NOT transition Lattice states directly. The supervisor monitors your output and handles: +- `in_progress` → your worker is running +- Lattice comment posted = supervisor read your metric +- `done` = experiment accepted (supervisor action) +- `archived` = experiment discarded (supervisor action) + +If you need to signal an issue to the supervisor, print a line starting with `HERMES_STATUS:`: +``` +HERMES_STATUS: blocked — missing numpy, cannot proceed +HERMES_STATUS: timeout — partial results in results.json +``` + +## Configuration + +No configuration file needed. The supervisor (Hermes) provides: +- LLM provider via environment (already configured) +- Working directory via `delegate_task` call +- Metric key and format via goal string +- `HERMES_YOLO_MODE=1` to skip command approval + +## Anti-Patterns + +Do NOT: +- Use subprocess, os.system, eval, exec, or shell escapes in experiment code +- Make network calls (experiments must be self-contained) +- Invent or fabricate metric values — measure real outcomes +- Run without a time guard (always implement elapsed-time check near 80% of budget) +- Print non-metric lines as `key: value` (they will be parsed as metrics) +- Use XML `` format — use JSON tool format instead diff --git a/RESEARCH_OPERATIONS.md b/RESEARCH_OPERATIONS.md new file mode 100644 index 000000000000..c059c944018a --- /dev/null +++ b/RESEARCH_OPERATIONS.md @@ -0,0 +1,219 @@ +# Hermes AutoResearch — Operations Guide + +> This document captures the operational procedures, anti-patterns, and performance characteristics discovered during the development and validation of the Hermes AutoResearch orchestration layer. + +## Launching a Research Job + +### Method 1: Detached Runner (Recommended for >2 min tasks) + +Create a job spec JSON and launch via `research/job_runner`: + +```json +{ + "job_id": "unique-job-id", + "job_dir": "/home/user/.hermes/research-jobs/unique-job-id", + "model": "kimi-for-coding", + "provider": "kimi-coding", + "topic": "Your research topic here", + "deliverable": "What the worker must produce", + "metric_key": "completeness_score", + "metric_direction": "maximize", + "task_type": "research", + "max_iterations": 3, + "env": {"HERMES_YOLO_MODE": "1"} +} +``` + +Launch: +```bash +cd /path/to/hermes-agent +source venv/bin/activate +HERMES_YOLO_MODE=1 python -m agent.research.job_runner /path/to/job.json +``` + +### Method 2: Background Process (Non-blocking) + +```bash +HERMES_YOLO_MODE=1 python -m agent.research.job_runner /path/to/job.json & +``` + +The runner creates a `.runner.lock` file atomically. If the job is already running, it exits with code 2. + +### Method 3: Direct Python API (Blocking) + +```python +from agent.research.supervisor import ResearchSupervisor, TaskSpec +from pathlib import Path + +spec = TaskSpec( + topic="...", + deliverable="...", + metric_key="completeness_score", + metric_direction="maximize", + task_type="research", +) + +supervisor = ResearchSupervisor(parent_agent=agent) +history = supervisor.run(spec, initial_attempt="", run_id="run-001", max_iterations=3, llm=llm_client) +``` + +## Monitoring Progress + +### Passive Monitoring (Recommended) + +Read checkpoint files without polling the process: + +```bash +# Quick status +cat ~/.hermes/research-jobs//checkpoint.json + +# Full history +cat ~/.hermes/research-jobs//history.json + +# Live log +tail -f ~/.hermes/research-jobs//runner.log +``` + +### File Structure + +``` +~/.hermes/research-jobs// +├── job.json # Original spec +├── .runner.lock # PID lock (prevents duplicate runs) +├── state.json # {status, pid, started_at} +├── checkpoint.json # {round, total_rounds, best_metric} +├── history.json # Full results array + best reference +├── result.json # Final result (appears on completion) +└── runner.log # Runner + supervisor logs +``` + +## Anti-Patterns and Fixes + +| Anti-Pattern | Why It Fails | Fix | +|-------------|-------------|-----| +| Foreground run with default timeout (60s) | MCP init takes 30-60s; runner killed before loop starts | Use `timeout=300` minimum, or background launch | +| Active process polling (`ps`, `find`, `tail` in loop) | Wastes iterations, creates noise | Read `checkpoint.json` or `history.json` passively | +| Deleting logs and retrying identically | Same failure repeats, no learning | Change timeout or use background mode | +| Launching same job twice | Double resource usage, conflicting checkpoints | Lock file prevents this; check `.runner.lock` | +| No `terminal` in default toolsets | Worker cannot execute code even if brief says it can | `_DEFAULT_TOOLSETS["research"] = ["web", "terminal", "file"]` | +| XML `` from worker | kimi-coding generates XML instead of JSON tools | Add anti-XML guard to task brief | +| Worker without `HERMES_YOLO_MODE` | Worker stalls waiting for command approval | Set `HERMES_YOLO_MODE=1` in env or job spec | + +## Performance Baselines + +Measured on kimi-for-coding via kimi-coding provider: + +| Metric | Value | Notes | +|--------|-------|-------| +| MCP init time | ~30-60s | 3 MCP servers (ia-bridge, lattice, obsidian) | +| Init-to-first-checkpoint (simple) | ~30s | smoke-test with minimal topic | +| Init-to-first-checkpoint (complex) | ~300s | Benchmark with multi-step worker | +| Iteration time (research task) | ~290-350s | Includes worker execution + judge | +| Provider resolution (cached) | ~0s | Cache hit after first call | +| Provider resolution (uncached) | ~1-2s | Auth resolution + client build | +| Subdirectory hints (cached) | ~0s | Per-directory cache | +| Subdirectory hints (uncached) | ~50-100ms | Disk read + scan | + +## Early Stop Behavior + +| Baseline | Early Stop Limit | Min Delta | Rationale | +|----------|-----------------|-----------|-----------| +| < 0.9 (maximize) or > 0.1 (minimize) | 3 iterations | 0.0 | Standard exploration | +| ≥ 0.9 (maximize) or ≤ 0.1 (minimize) | 1 iteration | 0.05 | Aggressive stop for high baselines | + +## Recovery Scenarios + +### Scenario: Job appears stuck + +1. Check `checkpoint.json` — has `round` advanced? +2. Check `runner.log` — are there recent `Omitting temperature` lines? +3. If log is stale >5 min, process may be waiting on API +4. Do NOT delete `.runner.lock` — kill the process instead: `kill $(cat .runner.lock)` + +### Scenario: Job crashed + +1. Read `runner.log` for traceback +2. Fix the issue (e.g., missing field in job.json) +3. Delete `.runner.lock` if stale (see below) +4. Relaunch + +### Scenario: Stale `.runner.lock` after crash + +The lock file holds the runner PID. To verify it's actually stale before deleting: + +```bash +PID=$(cat ~/.hermes/research-jobs//.runner.lock) +ps -p "$PID" > /dev/null && echo "STILL RUNNING (PID $PID)" || echo "stale, safe to remove" +rm -f ~/.hermes/research-jobs//.runner.lock # only if stale +``` + +Never blind-delete the lock while the runner is alive — you'll get duplicate +processes writing to the same checkpoint and corrupted state. + +### Scenario: Want to resume from checkpoint + +Current implementation does not support automatic resume from `checkpoint.json`. To resume: +1. Read `history.json` to find the best artifact +2. Create a new job spec with `initial_attempt` set to the best artifact content +3. Launch as new job + +## LLM Judge + +The judge runs on **every iteration** when `evaluation_mode="llm_judge"`. + +- Skipping iterations risks accepting worker-inflated self-reported scores +- Judge latency: ~5-15s per evaluation (one API call) +- Judge prompt is in `_score_with_llm_judge()` — customizable via `evaluation_prompt` in TaskSpec + +## Task Types and Toolsets + +| Type | Default Toolsets | Use When | +|------|-----------------|----------| +| `code` | terminal, file | Writing/running Python code | +| `search` | web, terminal, file | Web research, data collection | +| `research` | web, terminal, file | Synthesis, analysis, reporting | +| `generic` | terminal, file | Any custom task | + +Override with `worker_toolsets` parameter in `supervisor.run()`. + +## Environment Variables + +| Variable | Effect | +|----------|--------| +| `HERMES_YOLO_MODE=1` | Skip command approval (required for workers) | +| `DELEGATION_MAX_CONCURRENT_CHILDREN=3` | Parallel workers (default 3) | + +## Lattice Integration (Optional) + +If you pass `lattice_task_id` when starting a research job, the supervisor will post round-by-round progress comments to that Lattice task. This requires: + +1. Lattice initialized at `~/.hermes/org/.lattice/` (run `lattice init` in `~/.hermes/org` if missing) +2. The target task ID must exist in that Lattice database + +If Lattice is not available, the research job still runs normally — only the progress comments are skipped. Check `runner.log` for "Lattice comment failed" warnings if you expected comments but don't see them. + +### Verifying Lattice availability + +```bash +# Quick check +ls ~/.hermes/org/.lattice/ids.json + +# If missing, initialize: +cd ~/.hermes/org && lattice init +``` + +## Git Workflow for AutoResearch Changes + +All changes to the autoresearch stack are committed to branch `feat/hermes-autoresearch-upstream`: + +```bash +git log --oneline feat/hermes-autoresearch-upstream +``` + +Key commits: +- `cc929c0a` — Add research_job orchestration for long-running loops +- `4d64d568` — Include terminal in research/search default toolsets +- `0da707a3` — Add Tools Available + anti-XML guard to task briefs +- `f08dc63c` — Correct sandbox messaging and add partial recovery +- `8e68e23d` — Performance optimizations (lock, cache, early stop) +- `61c6e994` — Restore LLM judge on every iteration diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 327ef25e4d39..3dd3f3bd3ccf 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -161,6 +161,10 @@ def _extract_url_query_params(url: str): "tencentmaas": "tencent-tokenhub", } +# Cache for resolve_provider_client to avoid repeated auth resolution +_resolve_provider_cache: dict[tuple, tuple] = {} +_resolve_provider_cache_lock = threading.Lock() + def _normalize_aux_provider(provider: Optional[str]) -> str: normalized = (provider or "auto").strip().lower() @@ -1967,6 +1971,66 @@ def resolve_provider_client( Codex/Responses API providers, an adapter handles the translation transparently. + Args: + provider: Provider identifier. One of: + "openrouter", "nous", "openai-codex" (or "codex"), + "zai", "kimi-coding", "minimax", "minimax-cn", + "custom" (OPENAI_BASE_URL + OPENAI_API_KEY), + "auto" (full auto-detection chain). + model: Model slug override. If None, uses the provider's default + auxiliary model. + async_mode: If True, return an async-compatible client. + raw_codex: If True, return a raw OpenAI client for Codex providers + instead of wrapping in CodexAuxiliaryClient. Use this when + the caller needs direct access to responses.stream() (e.g., + the main agent loop). + explicit_base_url: Optional direct OpenAI-compatible endpoint. + explicit_api_key: Optional API key paired with explicit_base_url. + api_mode: API mode override. One of "chat_completions", + "codex_responses", or None (auto-detect). When set to + "codex_responses", the client is wrapped in + CodexAuxiliaryClient to route through the Responses API. + + Returns: + (client, resolved_model) or (None, None) if auth is unavailable. + """ + cache_key = ( + provider, model, async_mode, raw_codex, + explicit_base_url, explicit_api_key, api_mode, + tuple(sorted((main_runtime or {}).items())) if main_runtime else None, + ) + with _resolve_provider_cache_lock: + if cache_key in _resolve_provider_cache: + client, resolved_model = _resolve_provider_cache[cache_key] + logger.debug("resolve_provider_client: cache hit for %s", provider) + return client, resolved_model + + result = _resolve_provider_client_impl( + provider, model, async_mode, raw_codex, + explicit_base_url, explicit_api_key, api_mode, main_runtime, + ) + with _resolve_provider_cache_lock: + _resolve_provider_cache[cache_key] = result + return result + + +def _resolve_provider_client_impl( + provider: str, + model: str = None, + async_mode: bool = False, + raw_codex: bool = False, + explicit_base_url: str = None, + explicit_api_key: str = None, + api_mode: str = None, + main_runtime: Optional[Dict[str, Any]] = None, +) -> Tuple[Optional[Any], Optional[str]]: + """Central router: given a provider name and optional model, return a + configured client with the correct auth, base URL, and API format. + + The returned client always exposes ``.chat.completions.create()`` — for + Codex/Responses API providers, an adapter handles the translation + transparently. + Args: provider: Provider identifier. One of: "openrouter", "nous", "openai-codex" (or "codex"), diff --git a/agent/factory.py b/agent/factory.py new file mode 100644 index 000000000000..4ac840930ea5 --- /dev/null +++ b/agent/factory.py @@ -0,0 +1,70 @@ +"""Centralized AIAgent construction for non-CLI entrypoints. + +Most Hermes entrypoints (CLI, gateway, ACP, TUI gateway, batch_runner) build +``AIAgent`` directly with their own kwargs. The detached research-job runner +historically did the same plus a manual post-init patching block to satisfy +the runtime invariants ``delegate_task`` expects. + +This module consolidates that patching into one well-named factory so: +1. The fragile patch list lives in *one* place — easier to keep in sync as + ``AIAgent`` evolves upstream. +2. Other detached entrypoints (cron, batch jobs, future schedulers) can + reuse the same factory rather than copy-pasting the patch block. + +The longer-term plan (HRM-57 follow-up, requires upstream coordination) +is to absorb the five fragile internal attributes — ``_delegate_depth``, +``terminal_cwd``, ``cwd``, ``_subdirectory_hints``, ``_delegate_spinner`` +— into ``AIAgent.__init__`` itself so this factory becomes a thin profile +mapper. Until then it is the single point of fragility. +""" +from __future__ import annotations + +import os +from typing import Any + + +def build_agent_for_research_job(spec: dict[str, Any]) -> Any: + """Build an AIAgent suitable for running a detached research job. + + Reads model/provider/toolset config from ``spec`` (typically loaded from + ``/job.json``). The parent agent inherits the active profile's + SOUL.md, AGENTS.md, and MEMORY.md unless ``spec`` explicitly opts out + via ``skip_context_files`` or ``skip_memory``. + + Returns: + Live ``AIAgent`` ready to be passed as ``parent_agent`` to + ``run_research`` / ``ResearchSupervisor``. + """ + from run_agent import AIAgent + + agent = AIAgent( + model=spec["model"], + provider=spec.get("provider"), + base_url=spec.get("base_url"), + api_key=spec.get("api_key"), + api_mode=spec.get("api_mode"), + enabled_toolsets=spec.get("toolsets", ["research", "terminal", "file"]), + quiet_mode=True, + platform="cli", + session_id=f"research-job:{spec['job_id']}", + skip_context_files=spec.get("skip_context_files", False), + skip_memory=spec.get("skip_memory", False), + # HRM-57 full: the five formerly-patched runtime invariants are now + # constructor kwargs on AIAgent. terminal_cwd / cwd default to + # env-derived values inside __init__; the rest are zero/None defaults + # appropriate for a top-level detached parent. + delegate_depth=0, + terminal_cwd=os.getcwd(), + cwd=os.getcwd(), + subdirectory_hints=None, + ) + + # Only the progress callback still needs post-init wiring because it is + # not a constructor kwarg today. tool_progress_callback IS in __init__, + # but defaults to None — set a no-op so callers that read it can dispatch + # without checking. provider_* are honored by __init__ from the spec, so + # the previous defensive getattr block is no longer necessary. + if agent.tool_progress_callback is None: + agent.tool_progress_callback = lambda *a, **k: None + + return agent diff --git a/agent/research/__init__.py b/agent/research/__init__.py new file mode 100644 index 000000000000..9b0c782dc631 --- /dev/null +++ b/agent/research/__init__.py @@ -0,0 +1,33 @@ +"""Hermes AutoResearch — Karpathy inner loop + Autogenesis AOOR for Hermes. + +Public API re-exports for convenience. Internal callers should import from +the submodules directly to keep dependency graph explicit. + +Pattern parallel: agent.research is a self-contained orchestration module +in the same shape as ``cron/`` and ``gateway/`` — a directory bundle of +related primitives, not a flat collection of agent.research_*.py files. +""" +from agent.research.supervisor import ResearchSupervisor, TaskSpec +from agent.research.runner import ( + DelegateSandboxResult, + ExperimentHistory, + ExperimentResult, + ExperimentRunner, + HermesExperimentConfig, +) +from agent.research.metrics import UniversalMetricParser +from agent.research.evolution import EvolutionStore, LessonEntry, LessonCategory + +__all__ = [ + "ResearchSupervisor", + "TaskSpec", + "DelegateSandboxResult", + "ExperimentHistory", + "ExperimentResult", + "ExperimentRunner", + "HermesExperimentConfig", + "UniversalMetricParser", + "EvolutionStore", + "LessonEntry", + "LessonCategory", +] diff --git a/agent/research/evolution.py b/agent/research/evolution.py new file mode 100644 index 000000000000..ef113dcd3f57 --- /dev/null +++ b/agent/research/evolution.py @@ -0,0 +1,588 @@ +"""Self-evolution system for the ResearchClaw pipeline. + +Records lessons from each pipeline run (failures, slow stages, quality issues) +and injects them into future runs as prompt overlays. Inspired by Sibyl's +time-weighted evolution mechanism. + +Architecture +------------ +* ``LessonCategory`` — 6 issue categories for classification. +* ``LessonEntry`` — single lesson (stage, category, severity, description, ts). +* ``EvolutionStore`` — JSONL-backed persistent store with append + query. +* ``extract_lessons()`` — auto-extract lessons from ``StageResult`` lists. +* ``build_overlay()`` — generate per-stage prompt overlay text. + +Usage +----- +:: + + from researchclaw.evolution import EvolutionStore, extract_lessons + + store = EvolutionStore(Path("evolution")) + lessons = extract_lessons(results) + store.append_many(lessons) + overlay = store.build_overlay("hypothesis_gen", max_lessons=5) +""" + +from __future__ import annotations + +import json +import logging +import math +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path + +logger = logging.getLogger(__name__) + +# Skills directories to scan — Hermes autoresearch skills +_PROJECT_SKILLS_DIRS: tuple[str, ...] = ( + "skills/autoresearch", +) + + +def _load_project_skills() -> list[str]: + """Load skill content from Hermes autoresearch skills directory.""" + skills: list[str] = [] + root = Path(__file__).resolve().parent.parent + for rel_dir in _PROJECT_SKILLS_DIRS: + skills_dir = root / rel_dir + if not skills_dir.is_dir(): + continue + for skill_sub in sorted(skills_dir.iterdir()): + if not skill_sub.is_dir(): + continue + # Skip the main researchclaw CLI skill — it's not a pipeline overlay + if skill_sub.name == "researchclaw": + continue + skill_file = skill_sub / "SKILL.md" + if skill_file.is_file(): + try: + text = skill_file.read_text(encoding="utf-8").strip() + if text: + skills.append(text) + except OSError: + continue + return skills + + +class LessonCategory(str, Enum): + """Issue classification for extracted lessons.""" + + SYSTEM = "system" # Environment / network / timeout + EXPERIMENT = "experiment" # Code validation, sandbox timeout + WRITING = "writing" # Paper quality issues + ANALYSIS = "analysis" # Weak analysis, missing comparison + LITERATURE = "literature" # Search / verification failures + PIPELINE = "pipeline" # Stage orchestration issues + + +@dataclass +class LessonEntry: + """A single lesson extracted from a pipeline run.""" + + stage_name: str + stage_num: int + category: str + severity: str # "info", "warning", "error" + description: str + timestamp: str # ISO 8601 + run_id: str = "" + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, object]) -> LessonEntry: + return cls( + stage_name=str(data.get("stage_name", "")), + stage_num=int(data.get("stage_num", 0)), + category=str(data.get("category", "pipeline")), + severity=str(data.get("severity", "info")), + description=str(data.get("description", "")), + timestamp=str(data.get("timestamp", "")), + run_id=str(data.get("run_id", "")), + ) + + +# --------------------------------------------------------------------------- +# Lesson classification keywords +# --------------------------------------------------------------------------- + +_CATEGORY_KEYWORDS: dict[str, list[str]] = { + LessonCategory.SYSTEM: [ + "timeout", "connection", "network", "oom", "memory", + "permission", "ssh", "socket", "dns", + ], + LessonCategory.EXPERIMENT: [ + "sandbox", "validation", "import", "syntax", "subprocess", + "experiment", "code", "execution", + ], + LessonCategory.WRITING: [ + "paper", "draft", "outline", "revision", "review", + "template", "latex", + ], + LessonCategory.ANALYSIS: [ + "analysis", "metric", "statistic", "comparison", "baseline", + ], + LessonCategory.LITERATURE: [ + "search", "citation", "verify", "hallucin", "arxiv", + "semantic_scholar", "literature", "collect", + ], +} + + +def _classify_error(stage_name: str, error_text: str) -> str: + """Classify an error into a LessonCategory based on keywords.""" + combined = f"{stage_name} {error_text}".lower() + best_category = LessonCategory.PIPELINE + best_score = 0 + for category, keywords in _CATEGORY_KEYWORDS.items(): + score = sum(1 for kw in keywords if kw in combined) + if score > best_score: + best_score = score + best_category = category + return best_category + + +# --------------------------------------------------------------------------- +# Lesson extraction from pipeline results +# --------------------------------------------------------------------------- + +# Stage name mapping (import-free to avoid circular deps) +_STAGE_NAMES: dict[int, str] = { + 1: "topic_init", 2: "problem_decompose", 3: "search_strategy", + 4: "literature_collect", 5: "literature_screen", 6: "knowledge_extract", + 7: "synthesis", 8: "hypothesis_gen", 9: "experiment_design", + 10: "code_generation", 11: "resource_planning", 12: "experiment_run", + 13: "iterative_refine", 14: "result_analysis", 15: "research_decision", + 16: "paper_outline", 17: "paper_draft", 18: "peer_review", + 19: "paper_revision", 20: "quality_gate", 21: "knowledge_archive", + 22: "export_publish", 23: "citation_verify", +} + + +def extract_lessons( + results: list[object], + run_id: str = "", + run_dir: Path | None = None, +) -> list[LessonEntry]: + """Extract lessons from a list of StageResult objects. + + Detects: + - Failed stages → error lesson + - Blocked stages → pipeline lesson + - Decision pivots/refines → pipeline lesson (with rationale if available) + - Runtime warnings from experiment stderr → code_bug lesson + - Metric anomalies (NaN, identical convergence) → metric_anomaly lesson + """ + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + lessons: list[LessonEntry] = [] + + for result in results: + stage_num = int(getattr(result, "stage", 0)) + stage_name = _STAGE_NAMES.get(stage_num, f"stage_{stage_num}") + status = str(getattr(result, "status", "")) + error = getattr(result, "error", None) + decision = str(getattr(result, "decision", "proceed")) + + # Failed stages + if "failed" in status.lower() and error: + category = _classify_error(stage_name, str(error)) + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=stage_num, + category=category, + severity="error", + description=f"Stage {stage_name} failed: {str(error)[:300]}", + timestamp=now, + run_id=run_id, + )) + + # Blocked stages + if "blocked" in status.lower(): + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=stage_num, + category=LessonCategory.PIPELINE, + severity="warning", + description=f"Stage {stage_name} blocked awaiting approval", + timestamp=now, + run_id=run_id, + )) + + # PIVOT / REFINE decisions — extract rationale if available + if decision in ("pivot", "refine"): + rationale = _extract_decision_rationale(run_dir) if run_dir else "" + desc = f"Research decision was {decision.upper()}" + if rationale: + desc += f": {rationale[:200]}" + else: + desc += " — prior hypotheses/experiments were insufficient" + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=stage_num, + category=LessonCategory.PIPELINE, + severity="warning", + description=desc, + timestamp=now, + run_id=run_id, + )) + + # --- Extract lessons from experiment artifacts --- + if run_dir is not None: + lessons.extend(_extract_runtime_lessons(run_dir, now, run_id)) + + return lessons + + +def _extract_decision_rationale(run_dir: Path) -> str: + """Extract rationale from the most recent decision_structured.json. + + Supports multiple field formats: + - ``rationale`` or ``reason`` key (direct) + - ``raw_text_excerpt`` containing ``## Justification`` section (LLM output) + """ + for stage_dir in sorted(run_dir.glob("stage-15*"), reverse=True): + decision_file = stage_dir / "decision_structured.json" + if decision_file.exists(): + try: + data = json.loads(decision_file.read_text(encoding="utf-8")) + if not isinstance(data, dict): + continue + # Try direct rationale/reason keys first + direct = data.get("rationale", "") or data.get("reason", "") + if direct: + return str(direct) + # Parse raw_text_excerpt for Justification section + raw = data.get("raw_text_excerpt", "") + if raw: + return _parse_justification_from_excerpt(str(raw)) + except (json.JSONDecodeError, OSError): + pass + return "" + + +def _parse_justification_from_excerpt(text: str) -> str: + """Extract the Justification/Rationale section from LLM decision text.""" + import re + + # Match ## Justification, ## Rationale, or similar headings + pattern = re.compile( + r"##\s*(?:Justification|Rationale|Reason)\s*\n(.*?)(?=\n##|\Z)", + re.DOTALL | re.IGNORECASE, + ) + match = pattern.search(text) + if match: + return match.group(1).strip()[:300] + # Fallback: skip the first line (## Decision / **REFINE**) and return the rest + lines = [l.strip() for l in text.splitlines() if l.strip()] + # Skip heading lines starting with ## or ** + content_lines = [ + l for l in lines + if not l.startswith("##") and not (l.startswith("**") and l.endswith("**")) + ] + if content_lines: + return " ".join(content_lines)[:300] + return "" + + +def _extract_runtime_lessons( + run_dir: Path, timestamp: str, run_id: str +) -> list[LessonEntry]: + """Extract fine-grained lessons from experiment run artifacts.""" + import math + + lessons: list[LessonEntry] = [] + + # Check sandbox run results for stderr warnings and NaN + for runs_dir in run_dir.glob("stage-*/runs"): + for run_file in runs_dir.glob("*.json"): + if run_file.name == "results.json": + continue + try: + payload = json.loads(run_file.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + if not isinstance(payload, dict): + continue + + # Check stderr for runtime warnings + stderr = payload.get("stderr", "") + if stderr and any( + kw in stderr for kw in ("Warning", "Error", "divide", "overflow", "invalid value") + ): + lessons.append(LessonEntry( + stage_name="experiment_run", + stage_num=12, + category=LessonCategory.EXPERIMENT, + severity="warning", + description=f"Runtime warning in experiment: {stderr[:200]}", + timestamp=timestamp, + run_id=run_id, + )) + + # Check metrics for NaN/Inf + metrics = payload.get("metrics", {}) + if isinstance(metrics, dict): + for key, val in metrics.items(): + try: + fval = float(val) + if math.isnan(fval) or math.isinf(fval): + lessons.append(LessonEntry( + stage_name="experiment_run", + stage_num=12, + category=LessonCategory.EXPERIMENT, + severity="error", + description=f"Metric '{key}' was {val} — code bug (division by zero or overflow)", + timestamp=timestamp, + run_id=run_id, + )) + except (TypeError, ValueError): + pass + + return lessons + + +# --------------------------------------------------------------------------- +# Time-decay weighting +# --------------------------------------------------------------------------- + +HALF_LIFE_DAYS: float = 30.0 +MAX_AGE_DAYS: float = 90.0 + + +def _time_weight(timestamp_iso: str) -> float: + """Compute exponential decay weight for a lesson based on age. + + Uses 30-day half-life: weight = exp(-age_days * ln(2) / 30). + Returns 0.0 for lessons older than 90 days. + """ + try: + ts = datetime.fromisoformat(timestamp_iso) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + age = datetime.now(timezone.utc) - ts + age_days = age.total_seconds() / 86400.0 + if age_days > MAX_AGE_DAYS: + return 0.0 + return math.exp(-age_days * math.log(2) / HALF_LIFE_DAYS) + except (ValueError, TypeError): + return 0.0 + + +# --------------------------------------------------------------------------- +# Evolution store +# --------------------------------------------------------------------------- + + +class EvolutionStore: + """JSONL-backed store for pipeline lessons.""" + + def __init__(self, store_dir: Path) -> None: + self._dir = store_dir + self._dir.mkdir(parents=True, exist_ok=True) + self._lessons_path = self._dir / "lessons.jsonl" + + @property + def lessons_path(self) -> Path: + return self._lessons_path + + def append(self, lesson: LessonEntry) -> None: + """Append a single lesson to the store.""" + with self._lessons_path.open("a", encoding="utf-8") as f: + f.write(json.dumps(lesson.to_dict(), ensure_ascii=False) + "\n") + + def append_many(self, lessons: list[LessonEntry]) -> None: + """Append multiple lessons atomically.""" + if not lessons: + return + with self._lessons_path.open("a", encoding="utf-8") as f: + for lesson in lessons: + f.write(json.dumps(lesson.to_dict(), ensure_ascii=False) + "\n") + logger.info("Appended %d lessons to evolution store", len(lessons)) + + def load_all(self) -> list[LessonEntry]: + """Load all lessons from disk.""" + if not self._lessons_path.exists(): + return [] + lessons: list[LessonEntry] = [] + for line in self._lessons_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + lessons.append(LessonEntry.from_dict(data)) + except (json.JSONDecodeError, TypeError): + continue + return lessons + + def query_for_stage( + self, stage_name: str, *, max_lessons: int = 5 + ) -> list[LessonEntry]: + """Return the most relevant lessons for a stage, weighted by recency. + + Includes lessons that directly match the stage, plus high-severity + lessons from related stages. + """ + all_lessons = self.load_all() + scored: list[tuple[float, LessonEntry]] = [] + for lesson in all_lessons: + weight = _time_weight(lesson.timestamp) + if weight <= 0.0: + continue + # Boost direct stage matches + if lesson.stage_name == stage_name: + weight *= 2.0 + # Boost errors over warnings/info + if lesson.severity == "error": + weight *= 1.5 + scored.append((weight, lesson)) + scored.sort(key=lambda x: x[0], reverse=True) + return [entry for _, entry in scored[:max_lessons]] + + def build_overlay( + self, + stage_name: str, + *, + max_lessons: int = 5, + skills_dir: str = "", + ) -> str: + """Generate a prompt overlay string for a given stage. + + Combines two sources: + 1. Current-run lessons from ``lessons.jsonl`` (intra-run learning). + 2. Cross-run MetaClaw ``arc-*`` skills from *skills_dir* (inter-run + learning via the MetaClaw skill-generation feedback loop). + + Project-level and user-level skills are handled separately by the + SkillRegistry in ``_helpers._get_skill_registry()``. + + Returns empty string if no relevant lessons or skills exist. + """ + parts: list[str] = [] + + # --- Section 1: intra-run lessons --- + lessons = self.query_for_stage(stage_name, max_lessons=max_lessons) + if lessons: + parts.append("## Lessons from Prior Runs") + for i, lesson in enumerate(lessons, 1): + severity_icon = {"error": "❌", "warning": "⚠️", "info": "ℹ️"}.get( + lesson.severity, "•" + ) + parts.append( + f"{i}. {severity_icon} [{lesson.category}] {lesson.description}" + ) + parts.append( + "\nUse these lessons to avoid repeating past mistakes." + ) + + # --- Section 2: cross-run MetaClaw arc-* skills --- + arc_skills: list[str] = [] + if skills_dir: + from pathlib import Path as _Path + + sd = _Path(skills_dir).expanduser() + if sd.is_dir(): + for skill_dir in sorted(sd.iterdir()): + if skill_dir.is_dir() and skill_dir.name.startswith("arc-"): + skill_file = skill_dir / "SKILL.md" + if skill_file.is_file(): + try: + text = skill_file.read_text(encoding="utf-8").strip() + if text: + arc_skills.append(text) + except OSError: + continue + + if arc_skills: + parts.append("\n## Learned Skills from Prior Runs") + for skill_text in arc_skills[:5]: + parts.append(skill_text) + parts.append( + "\nApply these skills proactively to improve quality." + ) + + return "\n".join(parts) + + def count(self) -> int: + """Return total number of stored lessons.""" + return len(self.load_all()) + + def export_to_memory(self, memory_store: object) -> int: + """Export lessons to a memory store (duck-typed to avoid circular imports). + + The *memory_store* must expose an ``add(content, category, metadata)`` method + (compatible with ``researchclaw.memory.store.MemoryStore``). + + Returns the number of lessons exported. + """ + add_fn = getattr(memory_store, "add", None) + if add_fn is None or not callable(add_fn): + logger.warning("export_to_memory: memory_store has no add() method") + return 0 + lessons = self.load_all() + exported = 0 + for lesson in lessons: + weight = _time_weight(lesson.timestamp) + if weight <= 0.0: + continue + try: + # Map lesson categories to valid MemoryStore categories + _CAT_MAP = { + "system": "experiment", "analysis": "experiment", + "literature": "ideation", "pipeline": "experiment", + "experiment": "experiment", "writing": "writing", + "ideation": "ideation", + } + _mem_cat = _CAT_MAP.get(lesson.category, "experiment") + add_fn( + content=lesson.description, + category=_mem_cat, + metadata={ + "source": "evolution", + "stage": lesson.stage_name, + "severity": lesson.severity, + "run_id": lesson.run_id, + "timestamp": lesson.timestamp, + }, + ) + exported += 1 + except Exception: + logger.debug("Failed to export lesson: %s", lesson.description[:80]) + return exported + + def get_lessons_for_stage_with_memory( + self, + stage_name: str, + memory_store: object, + *, + max_lessons: int = 5, + ) -> str: + """Combine evolution overlay with memory context for a stage. + + *memory_store* must expose a ``recall(query, category, max_results)`` method + returning objects with a ``.content`` attribute. + """ + overlay = self.build_overlay(stage_name, max_lessons=max_lessons) + recall_fn = getattr(memory_store, "recall", None) + if recall_fn is None or not callable(recall_fn): + return overlay + try: + memories = recall_fn( + query=stage_name, + category=None, + max_results=max_lessons, + ) + if memories: + parts = ["\n## Recalled Memories"] + for i, mem in enumerate(memories, 1): + content = getattr(mem, "content", str(mem)) + parts.append(f"{i}. {content}") + memory_text = "\n".join(parts) + return f"{overlay}\n{memory_text}" if overlay else memory_text + except Exception: + logger.debug("Failed to recall memories for stage %s", stage_name) + return overlay diff --git a/agent/research/job_runner.py b/agent/research/job_runner.py new file mode 100644 index 000000000000..39b9e972bdfe --- /dev/null +++ b/agent/research/job_runner.py @@ -0,0 +1,132 @@ +"""agent.research.job_runner — detached process entrypoint for long-running research loops. + +Reads a job spec JSON, builds an AIAgent, calls run_research, and writes +durable checkpoint state after every completed round. + +Usage: + python -m agent.research.job_runner /path/to/job.json +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def _setup_logging(job_dir: Path) -> None: + log_path = job_dir / "runner.log" + handler = logging.FileHandler(log_path, mode="a") + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + root = logging.getLogger() + root.setLevel(logging.DEBUG) + root.addHandler(handler) + + +def _write_state(job_dir: Path, **fields: Any) -> None: + state_path = job_dir / "state.json" + state = json.loads(state_path.read_text()) if state_path.exists() else {} + state.update(fields) + state["updated_at"] = time.time() + state_path.write_text(json.dumps(state, indent=2)) + + +def _build_agent(spec: dict[str, Any]) -> Any: + """Build an AIAgent from the job spec. + + Thin wrapper around ``agent.factory.build_agent_for_research_job`` — + construction + post-init patching live there so other detached + entrypoints can reuse the same logic. See agent/factory.py for the + "keep in sync with AIAgent" caveat. + """ + from agent.factory import build_agent_for_research_job + return build_agent_for_research_job(spec) + + +def main(spec_path: str) -> int: + spec = json.loads(Path(spec_path).read_text()) + job_dir = Path(spec["job_dir"]) + job_dir.mkdir(parents=True, exist_ok=True) + + # --- Lock file to prevent multiple instances of the same job --- + lock_path = job_dir / ".runner.lock" + try: + # Use O_EXCL to atomically create the lock file; fail if it exists + fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) + with os.fdopen(fd, "w") as f: + f.write(str(os.getpid())) + except FileExistsError: + print(f"ERROR: Job {spec['job_id']} is already running (lock file exists). Exiting.", file=sys.stderr) + return 2 + + try: + _setup_logging(job_dir) + logger.info("Job %s starting", spec["job_id"]) + + _write_state( + job_dir, + job_id=spec["job_id"], + status="running", + pid=os.getpid(), + started_at=time.time(), + spec_path=spec_path, + ) + + try: + agent = _build_agent(spec) + except Exception as exc: + logger.exception("Failed to build parent agent") + _write_state(job_dir, status="failed", error=f"parent_agent build failed: {exc}") + return 1 + + from tools.research_tool import run_research + + try: + logger.info("Calling run_research with checkpoint_dir=%s", job_dir) + raw = run_research( + topic=spec["topic"], + deliverable=spec["deliverable"], + metric_key=spec["metric_key"], + metric_direction=spec.get("metric_direction", "maximize"), + task_type=spec.get("task_type", "generic"), + evaluation_mode=spec.get("evaluation_mode", "self_report"), + evaluation_prompt=spec.get("evaluation_prompt", ""), + initial_attempt=spec.get("initial_attempt", ""), + max_iterations=spec.get("max_iterations", 3), + time_budget_sec=spec.get("time_budget_sec", 0), + lattice_task_id=spec.get("lattice_task_id"), + parent_agent=agent, + checkpoint_dir=str(job_dir), + ) + + result = json.loads(raw) + result_path = job_dir / "result.json" + result_path.write_text(json.dumps(result, indent=2)) + + status = "completed" if "error" not in result else "failed" + _write_state(job_dir, status=status, **result) + logger.info("Job %s finished: %s", spec["job_id"], status) + return 0 if status == "completed" else 1 + + except Exception as exc: + logger.exception("Job %s failed", spec["job_id"]) + _write_state(job_dir, status="failed", error=str(exc)) + return 1 + finally: + try: + lock_path.unlink(missing_ok=True) + except Exception: + pass + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python -m agent.research.job_runner ", file=sys.stderr) + sys.exit(1) + sys.exit(main(sys.argv[1])) diff --git a/agent/research/metrics.py b/agent/research/metrics.py new file mode 100644 index 000000000000..035866f5c7b8 --- /dev/null +++ b/agent/research/metrics.py @@ -0,0 +1,293 @@ +"""Universal metric parser — supports JSON, CSV, and stdout regex formats. + +Parse priority: + 1. ``results.json`` — structured JSON output (recommended for all domains) + 2. ``results.csv`` — tabular output + 3. stdout regex — backward-compatible with existing ``metric: value`` format + +This module extends (not replaces) the existing ``sandbox.parse_metrics`` +function. The existing stdout parser remains the fallback. +""" + +from __future__ import annotations + +import csv +import json +import logging +import math +import re +from dataclasses import dataclass, field +from enum import Enum +from io import StringIO +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +class MetricType(str, Enum): + SCALAR = "scalar" + TABLE = "table" + CONVERGENCE = "convergence" + LEARNING_CURVE = "learning_curve" + CONFUSION_MATRIX = "confusion" + STRUCTURED = "structured" + PARETO = "pareto" + + +@dataclass +class ExperimentResults: + """Unified experiment results container. + + Works for all domains — ML scalar metrics, physics convergence data, + economics regression tables, etc. + """ + + # Flat scalar metrics (backward-compatible with existing pipeline) + scalars: dict[str, float] = field(default_factory=dict) + + # Per-condition results (new universal format) + conditions: dict[str, dict[str, Any]] = field(default_factory=dict) + + # Convergence data (for physics/math domains) + convergence: dict[str, list[dict[str, float]]] = field(default_factory=dict) + + # Regression tables (for economics) + regression_table: dict[str, dict[str, Any]] = field(default_factory=dict) + + # Full structured data (raw JSON) + structured: dict[str, Any] = field(default_factory=dict) + + # Metadata + experiment_type: str = "" + domain: str = "" + total_runtime_sec: float = 0.0 + source: str = "" # "json" | "csv" | "stdout" + + def to_flat_metrics(self) -> dict[str, float]: + """Convert to flat metric dict for backward compatibility. + + The existing pipeline expects dict[str, float] from parse_metrics(). + This method flattens all result types into that format. + """ + metrics: dict[str, float] = dict(self.scalars) + + # Flatten conditions + for cond_name, seeds in self.conditions.items(): + if isinstance(seeds, dict): + for seed_or_metric, value in seeds.items(): + if isinstance(value, dict): + for metric_name, metric_val in value.items(): + if isinstance(metric_val, (int, float)) and math.isfinite(metric_val): + metrics[f"{cond_name}/{metric_name}"] = float(metric_val) + elif isinstance(value, (int, float)) and math.isfinite(value): + metrics[f"{cond_name}/{seed_or_metric}"] = float(value) + + # Flatten convergence (take final/best error per method) + for method, points in self.convergence.items(): + if points: + last = points[-1] + for key, val in last.items(): + if key != "h" and isinstance(val, (int, float)) and math.isfinite(val): + metrics[f"{method}/{key}"] = float(val) + + # Flatten regression table + for spec, coeffs in self.regression_table.items(): + if isinstance(coeffs, dict): + for key, val in coeffs.items(): + if isinstance(val, (int, float)) and math.isfinite(val): + metrics[f"{spec}/{key}"] = float(val) + + return metrics + + +class UniversalMetricParser: + """Parse experiment results from multiple output formats. + + Usage:: + + parser = UniversalMetricParser() + results = parser.parse(run_dir) + flat = results.to_flat_metrics() # backward-compatible + """ + + def parse(self, run_dir: Path, stdout: str = "") -> ExperimentResults: + """Parse experiment results from a run directory. + + Tries formats in order: JSON → CSV → stdout regex. + """ + # 1. Try JSON + results_json = run_dir / "results.json" + if results_json.exists(): + try: + result = self._parse_json(results_json) + if result.scalars or result.conditions or result.convergence or result.regression_table: + logger.info("Parsed results from results.json") + return result + except Exception: + logger.warning("Failed to parse results.json", exc_info=True) + + # 2. Try CSV + results_csv = run_dir / "results.csv" + if results_csv.exists(): + try: + result = self._parse_csv(results_csv) + if result.source == "csv": + logger.info("Parsed results from results.csv") + return result + except Exception: + logger.warning("Failed to parse results.csv", exc_info=True) + + # 3. Fallback: stdout regex (existing behavior) + if stdout: + return self._parse_stdout(stdout) + + # Try reading stdout.log from run_dir + stdout_log = run_dir / "stdout.log" + if stdout_log.exists(): + try: + stdout_text = stdout_log.read_text(encoding="utf-8", errors="replace") + return self._parse_stdout(stdout_text) + except Exception: + logger.warning("Failed to read stdout.log", exc_info=True) + + return ExperimentResults(source="none") + + def _parse_json(self, path: Path) -> ExperimentResults: + """Parse structured JSON results.""" + with path.open(encoding="utf-8") as fh: + data = json.load(fh) + + if not isinstance(data, dict): + return ExperimentResults(source="json") + + result = ExperimentResults( + source="json", + experiment_type=data.get("experiment_type", ""), + structured=data, + ) + + # Extract metadata + meta = data.get("metadata", {}) + if isinstance(meta, dict): + result.domain = meta.get("domain", "") + result.total_runtime_sec = float(meta.get("total_runtime_sec", 0)) + + # Extract conditions (comparison experiments) + conditions = data.get("conditions", {}) + if isinstance(conditions, dict): + result.conditions = conditions + # Also extract scalar metrics for backward compatibility + for cond_name, seeds in conditions.items(): + if isinstance(seeds, dict): + for seed_key, metrics in seeds.items(): + if isinstance(metrics, dict): + for metric_name, val in metrics.items(): + if isinstance(val, (int, float)) and math.isfinite(val): + result.scalars[f"{cond_name}/{metric_name}"] = float(val) + result.scalars[metric_name] = float(val) + elif isinstance(metrics, (int, float)) and math.isfinite(metrics): + result.scalars[f"{cond_name}/{seed_key}"] = float(metrics) + + # Extract convergence data + convergence = data.get("convergence", {}) + if isinstance(convergence, dict): + result.convergence = convergence + + # Extract regression table + reg_table = data.get("regression_table", {}) + if isinstance(reg_table, dict): + result.regression_table = reg_table + + # Top-level scalar metrics + for key, val in data.items(): + if key not in ("conditions", "convergence", "regression_table", "metadata", "experiment_type"): + if isinstance(val, (int, float)) and math.isfinite(val): + result.scalars[key] = float(val) + + return result + + def _parse_csv(self, path: Path) -> ExperimentResults: + """Parse CSV results (one row per condition/seed/metric).""" + text = path.read_text(encoding="utf-8", errors="replace") + reader = csv.DictReader(StringIO(text)) + + result = ExperimentResults(source="csv") + rows_processed = 0 + + for row in reader: + rows_processed += 1 + # Expected columns: condition, seed, metric, value + # Or: method, h, error (for convergence) + cond = row.get("condition", row.get("method", "")) + metric = row.get("metric", "") + value_str = row.get("value", row.get("error", "")) + + try: + val = float(value_str) + except (ValueError, TypeError): + continue + + if not math.isfinite(val): + continue + + if metric: + key = f"{cond}/{metric}" if cond else metric + result.scalars[key] = val + elif cond: + # Convergence-style: method, h, error + h_str = row.get("h", "") + try: + h = float(h_str) + except (ValueError, TypeError): + continue + if cond not in result.convergence: + result.convergence[cond] = [] + result.convergence[cond].append({"h": h, "error": val}) + + # Mark as CSV source if we processed any rows (even if no valid data) + if rows_processed == 0: + result.source = "none" + + return result + + def _parse_stdout(self, stdout: str) -> ExperimentResults: + """Parse stdout using regex: 'metric: value' and 'METRIC: key=value' formats.""" + metrics = _parse_metrics_from_stdout(stdout) + return ExperimentResults( + scalars={k: float(v) for k, v in metrics.items() if isinstance(v, (int, float))}, + source="stdout", + ) + + +# Inline metric parser (ported from researchclaw.experiment.sandbox) +_FLOAT_RE = r"[+-]?\d+\.?\d*(?:[eE][+-]?\d+)?" +_METRIC_PATTERN = re.compile( + rf"^(?:\S+=\S+\s+)?(\w[\w.]*)\s*:\s*({_FLOAT_RE})\s*$" +) +_HERMES_METRIC_PATTERN = re.compile( + r"METRIC:\s*(\w[\w.]*)\s*=\s*(" + _FLOAT_RE + r")" +) + + +def _parse_metrics_from_stdout(stdout: str) -> dict[str, float]: + """Extract metric: value pairs from stdout text.""" + metrics: dict[str, float] = {} + for line in stdout.splitlines(): + line = line.strip() + # Hermes format: "METRIC: key=value STATUS: ..." + m = _HERMES_METRIC_PATTERN.search(line) + if m: + try: + metrics[m.group(1)] = float(m.group(2)) + except ValueError: + pass + continue + # Standard format: "metric_name: value" + m = _METRIC_PATTERN.match(line) + if m: + try: + metrics[m.group(1)] = float(m.group(2)) + except ValueError: + pass + return metrics diff --git a/agent/research/runner.py b/agent/research/runner.py new file mode 100644 index 000000000000..0eaa7781f33c --- /dev/null +++ b/agent/research/runner.py @@ -0,0 +1,402 @@ +"""Experiment execution engine — Karpathy edit→run→eval→keep/discard loop for Hermes. + +Ported from researchclaw/experiment/runner.py (aiming-lab/AutoResearchClaw, MIT). +Seams replaced: + - sandbox.run() → delegate_fn (async callable wrapping delegate_task) + - git branch/commit/discard → Lattice task lifecycle via lattice_comment_fn + - ExperimentConfig → HermesExperimentConfig (simple dataclass, no researchclaw deps) +""" + +from __future__ import annotations + +import json +import logging +import re +import time as _time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional, Protocol, cast + +logger = logging.getLogger(__name__) + + +@dataclass +class HermesExperimentConfig: + """Minimal experiment config — replaces researchclaw ExperimentConfig.""" + metric_key: str = "primary_metric" + metric_direction: str = "maximize" # "maximize" or "minimize" + time_budget_sec: int = 0 + max_iterations: int = 5 + keep_threshold: float = 0.0 # min abs delta to consider "kept" + + +@dataclass +class DelegateSandboxResult: + """Adapter wrapping delegate_task JSON result into sandbox-like shape.""" + metrics: dict[str, object] + stdout: str + stderr: str + elapsed_sec: float + timed_out: bool = False + returncode: int = 0 + error: Optional[str] = None + + +@dataclass(frozen=True) +class ExperimentResult: + run_id: str + iteration: int + code: str + metrics: dict[str, object] + primary_metric: float | None + improved: bool + kept: bool + elapsed_sec: float + stdout: str + stderr: str + error: str | None = None + + +@dataclass +class ExperimentHistory: + results: list[ExperimentResult] = field(default_factory=list) + best_result: ExperimentResult | None = None + baseline_metric: float | None = None + + def add(self, result: ExperimentResult) -> None: + self.results.append(result) + if self.baseline_metric is None and result.primary_metric is not None: + self.baseline_metric = result.primary_metric + + def to_dict(self) -> dict[str, object]: + return { + "results": [asdict(result) for result in self.results], + "best_result": asdict(self.best_result) if self.best_result else None, + "baseline_metric": self.baseline_metric, + } + + @classmethod + def from_dict(cls, data: dict[str, object]) -> ExperimentHistory: + results: list[ExperimentResult] = [] + raw_results = data.get("results") + if isinstance(raw_results, list): + for item in cast(list[object], raw_results): + if isinstance(item, dict): + item_map = cast(dict[object, object], item) + normalized_item: dict[str, object] = {} + for key, value in item_map.items(): + normalized_item[str(key)] = value + parsed = _result_from_dict(normalized_item) + if parsed is not None: + results.append(parsed) + best_raw = data.get("best_result") + best_result = ( + _result_from_dict( + { + str(key): value + for key, value in cast(dict[object, object], best_raw).items() + } + ) + if isinstance(best_raw, dict) + else None + ) + baseline_metric_raw = data.get("baseline_metric") + baseline_metric = ( + float(baseline_metric_raw) + if isinstance(baseline_metric_raw, (int, float)) + else None + ) + return cls( + results=results, best_result=best_result, baseline_metric=baseline_metric + ) + + +class _ChatResponse(Protocol): + content: str + + +class _ChatClient(Protocol): + def chat( + self, messages: list[dict[str, str]], *, system: str | None = None + ) -> _ChatResponse: ... + + +class ExperimentRunner: + """Karpathy inner loop: baseline → iterate → improve/discard, wired to Hermes delegate_task. + + Args: + config: HermesExperimentConfig with metric_key, direction, budget, iterations. + workspace: Directory for round artefacts. + delegate_fn: Callable(goal: str, working_dir: str) -> DelegateSandboxResult. + Wraps delegate_task; caller is responsible for spawning the worker. + lattice_comment_fn: Optional callable(msg: str) -> None for posting round summaries. + """ + + def __init__( + self, + config: "HermesExperimentConfig", + workspace: Path, + *, + delegate_fn: Callable[[str, str], "DelegateSandboxResult"], + lattice_comment_fn: Optional[Callable[[str], None]] = None, + ) -> None: + self.config: HermesExperimentConfig = config + self.workspace: Path = workspace + self.workspace.mkdir(parents=True, exist_ok=True) + self._delegate_fn = delegate_fn + self._lattice_comment = lattice_comment_fn or (lambda msg: None) + self.history: ExperimentHistory = ExperimentHistory() + + def run_experiment( + self, code: str, *, run_id: str, iteration: int = 0 + ) -> ExperimentResult: + """Run one experiment round via delegate_task and score the result.""" + t0 = _time.monotonic() + round_dir = str(self.workspace / f"round-{run_id}-iter{iteration}") + goal = ( + f"You are a Hermes research worker. Read program.md in {round_dir} " + f"and run the experiment. Report your result as:\n" + f"METRIC: {self.config.metric_key}= STATUS: improved|regressed|neutral " + f"NOTES: " + ) + + try: + sandbox_result = self._delegate_fn(goal, round_dir) + except Exception as exc: + elapsed = _time.monotonic() - t0 + logger.exception("delegate_fn failed for %s iter %d: %s", run_id, iteration, exc) + sandbox_result = DelegateSandboxResult( + metrics={}, stdout="", stderr=str(exc), elapsed_sec=elapsed, + timed_out=False, returncode=1, error=str(exc), + ) + + primary_metric = self._to_float( + sandbox_result.metrics.get(self.config.metric_key) + ) + current_best = ( + self.history.best_result.primary_metric + if self.history.best_result + else None + ) + + improved = False + kept = False + + if primary_metric is not None: + if current_best is None: + improved = True + kept = True + elif self._is_improvement(primary_metric, current_best): + improved = True + kept = abs(primary_metric - current_best) > self.config.keep_threshold + + error: str | None = sandbox_result.error + if not error and sandbox_result.timed_out: + error = f"Timed out after {self.config.time_budget_sec}s" + elif not error and sandbox_result.returncode != 0: + error = sandbox_result.stderr.strip() or f"Process exited with {sandbox_result.returncode}" + + result = ExperimentResult( + run_id=run_id, + iteration=iteration, + code=code, + metrics=sandbox_result.metrics, + primary_metric=primary_metric, + improved=improved, + kept=kept, + elapsed_sec=sandbox_result.elapsed_sec, + stdout=sandbox_result.stdout, + stderr=sandbox_result.stderr, + error=error, + ) + + if kept: + self.history.best_result = result + + self.history.add(result) + + # Post Lattice comment summarising this round + status_word = "KEPT" if kept else ("IMPROVED" if improved else "DISCARDED") + self._lattice_comment( + f"Round {run_id} iter {iteration}: {status_word} " + f"{self.config.metric_key}={primary_metric} " + f"(best={current_best})" + ) + return result + + def run_loop( + self, initial_code: str, *, run_id: str, llm: "_ChatClient | None" = None + ) -> ExperimentHistory: + """Karpathy inner loop: baseline → iterate → keep/discard.""" + self._lattice_comment(f"Research loop started: run_id={run_id}") + current_code = initial_code + baseline = self.run_experiment(current_code, run_id=run_id, iteration=0) + + if llm is None: + return self.history + + no_improvement_count = 0 + for iteration in range(1, self.config.max_iterations + 1): + next_code = self._improve_code(llm, current_code, self.history) + result = self.run_experiment(next_code, run_id=run_id, iteration=iteration) + current_code = next_code + + if result.improved: + no_improvement_count = 0 + else: + no_improvement_count += 1 + + if no_improvement_count >= 3: + logger.info("Stopping early: 3 non-improving iterations for %s", run_id) + self._lattice_comment(f"Early stop after {iteration} iterations (3 non-improving)") + break + + self._lattice_comment( + f"Research loop done: {len(self.history.results)} rounds, " + f"best={self.history.best_result.primary_metric if self.history.best_result else None}" + ) + return self.history + + def _improve_code( + self, llm: _ChatClient, current_code: str, history: ExperimentHistory + ) -> str: + direction = self.config.metric_direction + last_result = history.results[-1] if history.results else None + last_metrics = last_result.metrics if last_result else {} + best_metrics = history.best_result.metrics if history.best_result else {} + last_metric = last_result.primary_metric if last_result else None + best_metric = ( + history.best_result.primary_metric if history.best_result else None + ) + + prompt = ( + "Improve the experiment code to optimize the primary metric.\n\n" + f"Metric key: {self.config.metric_key}\n" + f"Direction: {direction}\n" + f"Last primary metric: {last_metric}\n" + f"Best primary metric: {best_metric}\n" + f"Last metrics JSON: {json.dumps(last_metrics, ensure_ascii=True)}\n" + f"Best metrics JSON: {json.dumps(best_metrics, ensure_ascii=True)}\n\n" + "Current code:\n" + "```python\n" + f"{current_code}\n" + "```\n\n" + "## Think Before Coding\n\n" + "Before writing any code:\n" + "1. State WHY the current metric is at the level it is " + "(what is the binding bottleneck?).\n" + "2. State your ONE hypothesis for what change will move the metric. " + "If uncertain between approaches, pick the simpler one.\n" + "3. Define your success criterion: " + f"'{self.config.metric_key} should move from {last_metric} toward " + f"{'higher' if direction == 'maximize' else 'lower'} by a measurable amount'.\n\n" + "## Surgical Changes\n\n" + "- Change only what your hypothesis requires. " + "Do not refactor unrelated code.\n" + "- Every changed line must trace directly to your hypothesis.\n" + "- If the fix is 5 lines, write 5 lines — not 50.\n" + "- Prefer the 50-line solution over the 200-line solution.\n\n" + "Return ONLY the updated Python code. " + "Do not include explanation outside the code." + ) + + try: + response = llm.chat( + [{"role": "user", "content": prompt}], + system=( + "You are an expert ML experimentation assistant. " + "Think carefully before writing code. " + "Make the minimum change needed to improve the metric. " + "Surface your reasoning as a comment at the top of the changed section." + ), + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Code improvement call failed: %s", exc) + return current_code + + candidate = getattr(response, "content", "") + if not isinstance(candidate, str) or not candidate.strip(): + logger.warning("LLM returned empty code; keeping current version") + return current_code + + extracted = self._extract_python_code(candidate) + return extracted if extracted.strip() else current_code + + def save_history(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text( + json.dumps(self.history.to_dict(), indent=2), encoding="utf-8" + ) + + def _is_improvement(self, new_value: float, best_value: float) -> bool: + if self.config.metric_direction == "maximize": + return new_value > best_value + return new_value < best_value + + @staticmethod + def _to_float(value: object) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + @staticmethod + def _extract_python_code(content: str) -> str: + match = re.search(r"```(?:python)?\s*(.*?)\s*```", content, flags=re.DOTALL) + if match is None: + return content.strip() + return match.group(1).strip() + + +def _result_from_dict(data: dict[str, object]) -> ExperimentResult | None: + run_id = data.get("run_id") + iteration = data.get("iteration") + code = data.get("code") + metrics = data.get("metrics") + primary_metric = data.get("primary_metric") + improved = data.get("improved") + kept = data.get("kept") + elapsed_sec = data.get("elapsed_sec") + stdout = data.get("stdout") + stderr = data.get("stderr") + error = data.get("error") + + if not isinstance(run_id, str) or not isinstance(iteration, int): + return None + if not isinstance(code, str) or not isinstance(metrics, dict): + return None + if primary_metric is not None and not isinstance(primary_metric, (int, float)): + return None + if not isinstance(improved, bool) or not isinstance(kept, bool): + return None + if not isinstance(elapsed_sec, (int, float)): + return None + if not isinstance(stdout, str) or not isinstance(stderr, str): + return None + if error is not None and not isinstance(error, str): + return None + + typed_metrics: dict[str, object] = {} + for key, value in cast(dict[object, object], metrics).items(): + typed_metrics[str(key)] = value + return ExperimentResult( + run_id=run_id, + iteration=iteration, + code=code, + metrics=typed_metrics, + primary_metric=float(primary_metric) + if isinstance(primary_metric, (int, float)) + else None, + improved=improved, + kept=kept, + elapsed_sec=float(elapsed_sec), + stdout=stdout, + stderr=stderr, + error=error, + ) diff --git a/agent/research/supervisor.py b/agent/research/supervisor.py new file mode 100644 index 000000000000..34e87ffcd027 --- /dev/null +++ b/agent/research/supervisor.py @@ -0,0 +1,1202 @@ +"""ResearchSupervisor — Karpathy inner loop for any task with a measurable deliverable. + +Implements the Autogenesis self-evolution loop (Act → Observe → Optimize → Remember) +applied to any task with a measurable deliverable: + + Phase | Autogenesis concept | Implementation + --------- | -------------------- | -------------- + ACT | Agent produces output | worker via delegate_task + OBSERVE | Capture outcome + traces | _observe() → learnings.jsonl + OPTIMIZE | Propose next hypothesis | _improve_attempt() (reflection optimizer) + REMEMBER | Persist insights for future rounds | learnings.jsonl (HeartbeatMemorySystem schema) + +The SEPL (Self Evolution Protocol Layer) materializes as: + - propose: _improve_attempt() drafts the next attempt + - evaluate: ExperimentRunner scores and keep/discards + - commit: kept results update best_result + lineage in ExperimentHistory + - rollback: discarded results revert attempt_holder to prior best + +Supported task types: "code" | "search" | "research" | "generic" +Evaluation modes: "self_report" | "llm_judge" +""" + +from __future__ import annotations + +import json +import logging +import re +import time as _time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from hermes_constants import get_hermes_home + +from agent.research.runner import ( + DelegateSandboxResult, + ExperimentHistory, + ExperimentResult, + ExperimentRunner, + HermesExperimentConfig, +) +from agent.research.metrics import UniversalMetricParser + +logger = logging.getLogger(__name__) + +# Matches the first decimal in a judge response — tolerates prefixes like +# "Score:" or suffixes like "/1.0" that the older tokens[0] parser choked on. +_JUDGE_SCORE_RE = re.compile(r"-?\d+(?:\.\d+)?") + +_parser = UniversalMetricParser() + + +# --------------------------------------------------------------------------- +# TaskSpec — the central abstraction for any measurable task +# --------------------------------------------------------------------------- + +@dataclass +class TaskSpec: + """Describes any task with a measurable deliverable. + + Examples: + # Code task — metric from test pass rate + TaskSpec( + topic="Implement a binary search tree", + deliverable="Python class with insert/search/delete, measured by test pass rate", + metric_key="pass_rate", + task_type="code", + ) + + # Search task — metric from result relevance + TaskSpec( + topic="Find papers on attention mechanisms published after 2022", + deliverable="Ranked list of relevant papers with abstracts", + metric_key="relevance_score", + task_type="search", + evaluation_mode="llm_judge", + evaluation_prompt="Score 0-1: does this paper list cover attention mechanisms published after 2022?", + ) + + # Research task — metric from synthesis quality + TaskSpec( + topic="Summarize the state of diffusion models for video generation", + deliverable="Technical synthesis covering key methods, benchmarks, and open problems", + metric_key="completeness_score", + task_type="research", + evaluation_mode="llm_judge", + evaluation_prompt="Score 0-1: does this synthesis cover key methods, benchmarks, and open problems?", + ) + + # Generic task — anything with a self-reported numeric metric + TaskSpec( + topic="Optimize hermes session search latency", + deliverable="Modified session search implementation with measured latency in ms", + metric_key="latency_ms", + metric_direction="minimize", + task_type="generic", + ) + """ + + topic: str + deliverable: str # what the worker must produce + metric_key: str # how success is measured + metric_direction: str = "maximize" # "maximize" or "minimize" + task_type: str = "generic" # "code" | "search" | "research" | "generic" + acceptance_criterion: str = "" # e.g. "pass_rate >= 0.95" or qualitative + evaluation_mode: str = "self_report" # "self_report" | "llm_judge" + evaluation_prompt: str = "" # for llm_judge: how to score the deliverable + hypothesis: str = "" # current iteration hypothesis (updated by supervisor) + + # Worker toolset hints per task type (overridable in ResearchSupervisor.run) + _DEFAULT_TOOLSETS: dict[str, list[str]] = field(default_factory=lambda: { + "code": ["terminal", "file"], + "search": ["web", "terminal", "file"], + "research": ["web", "terminal", "file"], + "generic": ["terminal", "file"], + }, repr=False) + + def default_toolsets(self) -> list[str]: + return self._DEFAULT_TOOLSETS.get(self.task_type, ["terminal", "file"]) + + +# --------------------------------------------------------------------------- +# Task brief templates — one per task_type +# --------------------------------------------------------------------------- + +def _build_task_brief(spec: TaskSpec, *, iteration: int, round_dir: str, time_budget_sec: int) -> str: + """Generate the task brief for the worker. Domain-aware but structurally identical.""" + builders = { + "code": _brief_code, + "search": _brief_search, + "research": _brief_research, + } + builder = builders.get(spec.task_type, _brief_generic) + return builder(spec, iteration=iteration, round_dir=round_dir, time_budget_sec=time_budget_sec) + + +def _think_block(spec: TaskSpec, iteration: int) -> str: + action = "improve" if iteration > 0 else "establish a baseline for" + return f"""\ +## Step 0 — Think Before Acting (Karpathy Principle 1) + +> Consult the `karpathy-guidelines` skill (`skills/autoresearch/karpathy-guidelines/SKILL.md`) +> for the full set of rules — surgical edits, surface assumptions, no overcomplication. + +Before producing anything, state in your output: + +1. **Assumption**: What do you understand the task to be asking for? +2. **Bottleneck** *(iteration {iteration} > 0 only)*: Why is `{spec.metric_key}` at its current value? + What is the binding constraint? +3. **Hypothesis**: What ONE change will {action} `{spec.metric_key}`? + If uncertain between approaches, pick the simpler one. +4. **Success criterion**: "`{spec.metric_key}` moves from X toward + {'higher' if spec.metric_direction == 'maximize' else 'lower'}" + +If something is unclear, name what is confusing in your NOTES. Do NOT guess silently. +""" + + +def _report_block(metric_key: str) -> str: + return f"""\ +## Final Report (required) + +Your last line of output must be: + +``` +METRIC: {metric_key}= STATUS: improved|regressed|neutral NOTES: +``` + +- Value must be a real number you measured or computed — never fabricated. +- NOTES must say what you did and what the key result was. +- Also write `results.json` with `{{"{metric_key}": }}` for structured parsing. +""" + + +def _brief_code(spec: TaskSpec, *, iteration: int, round_dir: str, time_budget_sec: int) -> str: + action = "Improve" if iteration > 0 else "Establish a baseline for" + return f"""\ +# Task Brief — Code ({action}) + +## Topic +{spec.topic} + +## Deliverable +{spec.deliverable} + +{_think_block(spec, iteration)} +## Step 1 — Implement + +The current attempt is in `attempt.py` in: `{round_dir}` + +{"Do not rewrite unless you have a specific, hypothesis-driven change. Make surgical edits only — every changed line must trace to your hypothesis." if iteration > 0 else "Implement the deliverable in `attempt.py`. Run it to verify."} + +{f"Time budget: {time_budget_sec}s. Print `TIME_ESTIMATE: Xs` before your main loop." if time_budget_sec > 0 else "Time budget: unlimited. Work until converged."} +{"Stop before 80% of budget and save partial results." if time_budget_sec > 0 else ""} + +## Step 2 — Measure + +Compute `{spec.metric_key}` from the code's output. +{"Acceptance criterion: " + spec.acceptance_criterion if spec.acceptance_criterion else ""} + +{_report_block(spec.metric_key)} +## Rules +- Do NOT fabricate metric values. +- No abstractions for single-use code. If 5 lines solve it, write 5. +- Do NOT refactor code unrelated to your hypothesis. +- **Package installation:** `pip install` is NOT blocked but may fail if the package isn't available. If you need a library, first check if it's already installed. If not, use `ctypes.CDLL` with system libraries (e.g., `/usr/lib/x86_64-linux-gnu/libgmp.so.10`) or write a pure-Python alternative. +- **You MAY use `python -c` and heredoc scripts** — these are allowed in your environment. +- **Tool format:** When calling tools, use the JSON format provided by the system. Do NOT use XML tags like ``. + +## Tools Available + +You have access to: `terminal` (shell commands), `file` (read/write), `code_execution` (Python scripts), and `search` (web search). +If a task requires running code, use `terminal()` or `code_execution()` — do NOT assume they are unavailable. +""" + + +def _brief_search(spec: TaskSpec, *, iteration: int, round_dir: str, time_budget_sec: int) -> str: + action = "Refine" if iteration > 0 else "Execute" + return f"""\ +# Task Brief — Search ({action}) + +## Topic +{spec.topic} + +## Deliverable +{spec.deliverable} + +{_think_block(spec, iteration)} +## Step 1 — Search + +{"The previous search strategy is in `attempt.md` in: " + round_dir + ". Revise it based on your hypothesis." if iteration > 0 else "Design and execute a search strategy. Save results to `attempt.md`."} + +{f"Time budget: {time_budget_sec}s. Do not make redundant searches — each query must have a hypothesis." if time_budget_sec > 0 else "Time budget: unlimited. Work until converged."} + +## Step 2 — Evaluate Results + +Score your results for `{spec.metric_key}` on a 0.0–1.0 scale. +{"Evaluate against: " + spec.evaluation_prompt if spec.evaluation_prompt and spec.evaluation_mode == "self_report" else ""} +{"Acceptance criterion: " + spec.acceptance_criterion if spec.acceptance_criterion else ""} + +{_report_block(spec.metric_key)} +## Rules +- Do NOT fabricate relevance scores. +- Each search iteration must test exactly one new hypothesis about where better results are. +- Save your full result set to `results.json` with `{{"{spec.metric_key}": }}`. +- **Tool format:** When calling tools, use the JSON format provided by the system. Do NOT use XML tags like ``. + +## Tools Available + +You have access to: `web_search` (find papers/articles), `browser` (visit pages), `file` (read/write), and `terminal` (shell commands for data processing). +Use these actively — do NOT assume they are unavailable. +""" + + +def _brief_research(spec: TaskSpec, *, iteration: int, round_dir: str, time_budget_sec: int) -> str: + action = "Deepen" if iteration > 0 else "Produce an initial" + return f"""\ +# Task Brief — Research ({action}) + +## Topic +{spec.topic} + +## Deliverable +{spec.deliverable} + +{_think_block(spec, iteration)} +## Step 1 — Investigate and Synthesize + +{"The current draft is in `attempt.md` in: " + round_dir + ". Identify its weakest section and address it." if iteration > 0 else "Research the topic. Produce an initial synthesis in `attempt.md`."} + +{f"Time budget: {time_budget_sec}s. Focus — do not survey everything; go deep on what your hypothesis identifies as the gap." if time_budget_sec > 0 else "Time budget: unlimited. Work until converged."} + +## Step 2 — Self-Evaluate + +Rate your synthesis on `{spec.metric_key}` (0.0–1.0). +{"Evaluate against: " + spec.evaluation_prompt if spec.evaluation_prompt and spec.evaluation_mode == "self_report" else ""} +{"Acceptance criterion: " + spec.acceptance_criterion if spec.acceptance_criterion else ""} + +{_report_block(spec.metric_key)} +## Rules +- Do NOT fabricate facts, citations, or scores. +- Each iteration must address exactly ONE identified gap — not rewrite everything. +- Save synthesis to `attempt.md` and score to `results.json`. +- **Tool format:** When calling tools, use the JSON format provided by the system. Do NOT use XML tags like ``. + +## Tools Available + +You have access to: `web_search` (research topics), `browser` (deep reading), `file` (read/write), and `terminal` (data processing). +Use these actively — do NOT assume they are unavailable. +""" + + +def _brief_generic(spec: TaskSpec, *, iteration: int, round_dir: str, time_budget_sec: int) -> str: + action = "Improve" if iteration > 0 else "Produce a baseline" + return f"""\ +# Task Brief — {action} + +## Topic +{spec.topic} + +## Deliverable +{spec.deliverable} + +{_think_block(spec, iteration)} +## Step 1 — Produce the Deliverable + +{"The previous attempt is in `attempt.md` in: " + round_dir + ". Revise it based on your hypothesis." if iteration > 0 else "Produce the deliverable. Save it to `attempt.md`."} + +{f"Time budget: {time_budget_sec}s." if time_budget_sec > 0 else "Time budget: unlimited. Work until converged."} + +## Step 2 — Measure + +Compute `{spec.metric_key}` as a number from your deliverable. +{"Acceptance criterion: " + spec.acceptance_criterion if spec.acceptance_criterion else ""} + +{_report_block(spec.metric_key)} +## Rules +- Do NOT fabricate metric values. +- Minimum effort that moves the metric. No speculative additions. +- Save deliverable to `attempt.md`, score to `results.json`. +- **Tool format:** When calling tools, use the JSON format provided by the system. Do NOT use XML tags like ``. + +## Tools Available + +You have access to: `terminal` (shell), `file` (read/write), `code_execution` (Python), `web_search`, and `browser`. +Use these actively — do NOT assume they are unavailable. +""" + + +# --------------------------------------------------------------------------- +# Attempt file name per task type +# --------------------------------------------------------------------------- + +_ATTEMPT_FILENAME: dict[str, str] = { + "code": "attempt.py", + "search": "attempt.md", + "research": "attempt.md", + "generic": "attempt.md", +} + + +# --------------------------------------------------------------------------- +# delegate_task bridge +# --------------------------------------------------------------------------- + +def _call_delegate_task( + goal: str, + context: str, + *, + parent_agent: Any, + toolsets: list[str] | None = None, +) -> dict[str, Any]: + from tools.delegate_tool import delegate_task + raw = delegate_task( + goal=goal, + context=context, + toolsets=toolsets or ["terminal", "file"], + parent_agent=parent_agent, + inherit_profile=True, + ) + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + return {"results": [{"status": "failed", "summary": raw or "", "error": "JSON parse failed"}]} + + +# --------------------------------------------------------------------------- +# Lattice comment bridge +# --------------------------------------------------------------------------- + +def _make_lattice_comment_fn( + lattice_task_id: Optional[str], + lattice_root: str, +) -> Callable[[str], None]: + if not lattice_task_id: + return lambda msg: logger.info("[lattice-stub] %s", msg) + + def _comment(msg: str) -> None: + try: + import subprocess + subprocess.run( + ["lattice", "comment", lattice_task_id, msg, "--actor", "agent:research-supervisor"], + cwd=lattice_root, + capture_output=True, + timeout=10, + ) + except Exception as exc: + logger.warning("Lattice comment failed: %s", exc) + + return _comment + + +# --------------------------------------------------------------------------- +# ResearchSupervisor +# --------------------------------------------------------------------------- + +class ResearchSupervisor: + """Karpathy loop for any task with a measurable deliverable. + + Args: + parent_agent: Live AIAgent instance (required for delegate_task). + workspace: Root directory for round artefacts. + lattice_task_id: Lattice task to post round comments to (optional). + lattice_root: Directory containing .lattice/. + """ + + def __init__( + self, + *, + parent_agent: Any, + workspace: Path | None = None, + lattice_task_id: Optional[str] = None, + lattice_root: str = str(get_hermes_home() / "org"), + ) -> None: + self._parent_agent = parent_agent + self._workspace = workspace or (get_hermes_home() / "research-workspace") + self._lattice_task_id = lattice_task_id + self._lattice_root = lattice_root + # Populated by run() — past-run lessons prepended to every worker brief. + self._evolution_overlay: str = "" + + def run( + self, + spec: TaskSpec, + initial_attempt: str, + *, + run_id: str, + max_iterations: int = 5, + time_budget_sec: int = 0, + keep_threshold: float = 0.0, + llm: Any = None, + worker_toolsets: list[str] | None = None, + checkpoint_dir: Path | None = None, + ) -> ExperimentHistory: + """Run the Karpathy loop for any TaskSpec. + + Args: + spec: Task description — topic, deliverable, metric, task type. + initial_attempt: Starting deliverable (code string, search query, + research outline, or any text the worker can iterate on). + run_id: Unique identifier for this run. + max_iterations: Max improvement iterations (not counting baseline). + time_budget_sec: Time budget per worker invocation (seconds). + keep_threshold: Min absolute metric delta to count as kept. + llm: LLM client for improvement proposals (None = baseline only). + worker_toolsets: Override default toolsets for workers. + + Returns: + ExperimentHistory with all round results and the best result. + """ + config = HermesExperimentConfig( + metric_key=spec.metric_key, + metric_direction=spec.metric_direction, + time_budget_sec=time_budget_sec, + max_iterations=max_iterations, + keep_threshold=keep_threshold, + ) + + lattice_comment_fn = _make_lattice_comment_fn( + self._lattice_task_id, self._lattice_root + ) + + # Load past-run lessons once per run; cap size to avoid token blow-up. + # Failure must not break the loop — overlay is best-effort. The helper + # already catches its own errors, but wrap here too as defense in depth + # (matches the _evolve call site at the bottom of run()). + try: + self._evolution_overlay = self._load_evolution_overlay() + except Exception as exc: + logger.warning("Evolution overlay load failed at run start: %s", exc) + self._evolution_overlay = "" + + toolsets = worker_toolsets or spec.default_toolsets() + attempt_holder: list[str] = [initial_attempt] + + def delegate_fn(goal: str, working_dir: str) -> DelegateSandboxResult: + return self._run_worker( + goal=goal, + working_dir=working_dir, + attempt=attempt_holder[0], + spec=spec, + time_budget_sec=time_budget_sec, + iteration=_extract_iteration(working_dir), + worker_toolsets=toolsets, + llm=llm, + ) + + runner = ExperimentRunner( + config=config, + workspace=self._workspace / run_id, + delegate_fn=delegate_fn, + lattice_comment_fn=lattice_comment_fn, + ) + + run_dir = self._workspace / run_id + lattice_comment_fn( + f"Loop started: run_id={run_id} type={spec.task_type} " + f"metric={spec.metric_key} topic={spec.topic[:50]}" + ) + + # --- ACT (baseline) --- + baseline = runner.run_experiment(initial_attempt, run_id=run_id, iteration=0) + # Read on-disk artifact — worker may have modified the seed during baseline + baseline_artifact = self._read_artifact(spec, run_dir, baseline) or initial_attempt + attempt_holder[0] = baseline_artifact + best_artifact_holder: list[str] = [baseline_artifact] + # --- OBSERVE --- + self._observe(baseline, spec, run_dir) + self._checkpoint(runner.history, checkpoint_dir, round=0) + + if llm is None: + lattice_comment_fn(f"Baseline only. best={runner.history.baseline_metric}") + try: + self._evolve(runner.history, spec, run_id) + except Exception as exc: + logger.warning("Evolution persistence failed for %s: %s", run_id, exc) + return runner.history + + # Determine early-stop parameters based on baseline quality + baseline_metric = runner.history.baseline_metric + is_high_baseline = False + min_delta = 0.0 + if baseline_metric is not None: + if spec.metric_direction == "maximize" and baseline_metric >= 0.9: + is_high_baseline = True + min_delta = 0.05 + elif spec.metric_direction == "minimize" and baseline_metric <= 0.1: + is_high_baseline = True + min_delta = 0.05 + + early_stop_limit = 1 if is_high_baseline else 3 + if is_high_baseline: + logger.info( + "High baseline detected (%s=%.4f). Using aggressive early stop: " + "limit=%d, min_delta=%.2f", + spec.metric_key, baseline_metric, early_stop_limit, min_delta, + ) + + # Autogenesis AOOR improvement loop + no_improvement = 0 + for iteration in range(1, max_iterations + 1): + # OPTIMIZE — propose revised attempt (SEPL: propose) + next_attempt = self._improve_attempt(llm, spec, attempt_holder[0], runner.history) + attempt_holder[0] = next_attempt + + # ACT — worker executes the attempt + result = runner.run_experiment(next_attempt, run_id=run_id, iteration=iteration) + + # Read on-disk artifact — worker may have refined it beyond the seed + actual_artifact = self._read_artifact(spec, run_dir, result) or next_attempt + attempt_holder[0] = actual_artifact + + # OBSERVE + REMEMBER — extract and persist structured learning + self._observe(result, spec, run_dir) + self._checkpoint(runner.history, checkpoint_dir, round=iteration) + + # SEPL: evaluate → keep/discard (handled by ExperimentRunner) + # SEPL: rollback — restore best on-disk artifact, not the seed string + # For high baselines, require min_delta for improvement to count + improved = result.improved + if improved and is_high_baseline and baseline_metric is not None: + current_metric = result.primary_metric + if current_metric is not None: + delta = abs(current_metric - baseline_metric) + if delta < min_delta: + improved = False + logger.info( + "Improvement below min_delta (%.4f < %.2f), treating as non-improving", + delta, min_delta, + ) + + if improved: + no_improvement = 0 + best_artifact_holder[0] = actual_artifact + else: + no_improvement += 1 + attempt_holder[0] = best_artifact_holder[0] # rollback to best artifact + + if no_improvement >= early_stop_limit: + logger.info( + "Early stop: %d non-improving iterations for %s (limit=%d)", + no_improvement, run_id, early_stop_limit, + ) + # SEPL: reflection optimizer — synthesize before giving up + self._reflect(runner.history, spec, llm, lattice_comment_fn, run_dir) + break + + best = runner.history.best_result + # Partial recovery: if the last iteration failed but we have prior results, + # report as partial success instead of total failure + last_result = runner.history.results[-1] if runner.history.results else None + if last_result and last_result.primary_metric is None and best: + lattice_comment_fn( + f"Loop done (PARTIAL): {len(runner.history.results)} rounds, " + f"best={best.primary_metric}. Last iteration failed but prior best preserved." + ) + else: + lattice_comment_fn( + f"Loop done: {len(runner.history.results)} rounds, " + f"best={best.primary_metric if best else None}" + ) + + # Persist lessons for cross-run learning. Append-only — failure here + # must not affect the loop's return value. + try: + self._evolve(runner.history, spec, run_id) + except Exception as exc: + logger.warning("Evolution persistence failed for %s: %s", run_id, exc) + + return runner.history + + # ------------------------------------------------------------------ + # Worker execution + # ------------------------------------------------------------------ + + def _run_worker( + self, + *, + goal: str, + working_dir: str, + attempt: str, + spec: TaskSpec, + time_budget_sec: int, + iteration: int, + worker_toolsets: list[str] | None, + llm: Any, + ) -> DelegateSandboxResult: + """Write task brief + attempt file, spawn delegate_task, parse result.""" + t0 = _time.monotonic() + wd = Path(working_dir) + wd.mkdir(parents=True, exist_ok=True) + + # Write the attempt in the appropriate format + attempt_filename = _ATTEMPT_FILENAME.get(spec.task_type, "attempt.md") + (wd / attempt_filename).write_text(attempt, encoding="utf-8") + + # Write the task brief, with optional EvolutionStore overlay prepended. + # The overlay is the read-side of HRM-59 — it surfaces past-run lessons + # so the worker doesn't repeat known mistakes. Loaded once per run via + # _load_evolution_overlay() and cached on the supervisor instance. + brief = _build_task_brief( + spec, + iteration=iteration, + round_dir=working_dir, + time_budget_sec=time_budget_sec, + ) + if self._evolution_overlay: + brief = self._evolution_overlay + "\n\n---\n\n" + brief + (wd / "task_brief.md").write_text(brief, encoding="utf-8") + + context = ( + f"Working directory: {working_dir}\n" + f"Topic: {spec.topic}\n" + f"Task type: {spec.task_type}\n" + f"Metric: {spec.metric_key} ({spec.metric_direction})\n" + f"Read task_brief.md for full instructions." + ) + + result = _call_delegate_task( + goal, + context, + parent_agent=self._parent_agent, + toolsets=worker_toolsets or ["terminal", "file"], + ) + + elapsed = _time.monotonic() - t0 + first = result.get("results", [{}])[0] if result.get("results") else {} + summary = first.get("summary") or "" + status = first.get("status", "failed") + + # Parse metrics from structured files first, stdout fallback + parsed = _parser.parse(wd, stdout=summary) + metrics: dict[str, object] = dict(parsed.to_flat_metrics()) + + # LLM judge override: score the deliverable externally + if spec.evaluation_mode == "llm_judge" and llm is not None and summary: + judge_score = self._score_with_llm_judge(summary, spec, llm) + if judge_score is not None: + metrics[spec.metric_key] = judge_score + logger.info( + "LLM judge scored %s=%.4f for %s iter %d", + spec.metric_key, judge_score, working_dir, iteration, + ) + + completed = status == "completed" + error: str | None = None if completed else (first.get("error") or f"Worker status: {status}") + + return DelegateSandboxResult( + metrics=metrics, + stdout=summary, + stderr="", + elapsed_sec=elapsed, + timed_out=False, + returncode=0 if completed else 1, + error=error, + ) + + # ------------------------------------------------------------------ + # Autogenesis: Observe + Remember (HeartbeatMemorySystem schema) + # ------------------------------------------------------------------ + + def _read_artifact(self, spec: TaskSpec, run_dir: Path, result: ExperimentResult) -> str | None: + """Read the actual on-disk artifact produced by the worker. + + Workers may modify attempt.py / attempt.md beyond the seed string passed in. + This ensures rollback restores the real artifact, not the seed text. + """ + round_dir = run_dir / f"round-{result.run_id}-iter{result.iteration}" + attempt_filename = _ATTEMPT_FILENAME.get(spec.task_type, "attempt.md") + artifact_file = round_dir / attempt_filename + try: + return artifact_file.read_text(encoding="utf-8") if artifact_file.exists() else None + except OSError: + return None + + @staticmethod + def _insight_from_json(round_dir: Path, metric_key: str) -> str: + """Extract a human-readable insight from results.json (structured source).""" + results_json = round_dir / "results.json" + if not results_json.exists(): + return "" + try: + data = json.loads(results_json.read_text(encoding="utf-8")) + for field in ("notes", "summary", "insight", "description"): + val = data.get(field) + if isinstance(val, str) and val.strip(): + return val.strip()[:200] + val = data.get(metric_key) + if val is not None: + return f"{metric_key}={val}" + except (json.JSONDecodeError, OSError): + pass + return "" + + def _observe( + self, + result: ExperimentResult, + spec: TaskSpec, + run_dir: Path, + ) -> None: + """Extract a structured learning from a completed round and append to learnings.jsonl. + + Schema mirrors Autogenesis HeartbeatMemorySystem: + type — "improvement" | "regression" | "failure" + key — metric name being optimized + insight — one-line summary of what happened and why + confidence — metric value (0.0 if unavailable) + source — "iter-N" for lineage tracing + + Insight extraction priority: + 1. results.json (structured, most reliable) + 2. NOTES: field from METRIC line in stdout + 3. Raw stdout excerpt (last resort) + """ + if result.primary_metric is not None: + entry_type = "improvement" if result.improved else "regression" + else: + entry_type = "failure" + + round_dir = run_dir / f"round-{result.run_id}-iter{result.iteration}" + + # 1. Structured source: results.json + insight_text = self._insight_from_json(round_dir, spec.metric_key) + + # 2. NOTES: field from the worker's METRIC line + if not insight_text and result.stdout: + m = re.search(r"NOTES:\s*(.+)", result.stdout) + if m: + insight_text = m.group(1).strip() + + # 3. Raw stdout excerpt + if not insight_text and result.stdout: + insight_text = result.stdout[:200].replace("\n", " ") + + # 4. Error fallback + if not insight_text and result.error: + insight_text = result.error[:200] + + # Normalize confidence to 0-1 scale regardless of metric direction + raw_metric = result.primary_metric + if raw_metric is not None: + # For minimize metrics, invert so higher confidence = better result + if spec.metric_direction == "minimize": + # Use inverse with a small epsilon to avoid div by zero + confidence = round(1.0 / (1.0 + abs(raw_metric)), 6) + else: + confidence = round(min(abs(raw_metric), 1.0), 6) + else: + confidence = 0.0 + + entry = { + "type": entry_type, + "key": spec.metric_key, + "insight": insight_text or "no output", + "confidence": confidence, + "source": f"iter-{result.iteration}", + } + + run_dir.mkdir(parents=True, exist_ok=True) + learnings_file = run_dir / "learnings.jsonl" + with learnings_file.open("a", encoding="utf-8") as f: + f.write(json.dumps(entry) + "\n") + + logger.debug( + "[observe] iter=%d type=%s %s=%.4f insight=%s", + result.iteration, entry_type, spec.metric_key, + entry["confidence"], insight_text[:80], + ) + + def _checkpoint( + self, + history: ExperimentHistory, + checkpoint_dir: Path | None, + round: int, + ) -> None: + """Serialize experiment history to a durable checkpoint directory. + + Called after baseline and every completed iteration so that external + monitors (e.g. research_job_tool) can read progress without polling + the running process. + """ + if checkpoint_dir is None: + return + + checkpoint_dir.mkdir(parents=True, exist_ok=True) + + # Serialize results + results = [] + for r in history.results: + results.append({ + "run_id": r.run_id, + "iteration": r.iteration, + "metrics": r.metrics, + "primary_metric": r.primary_metric, + "improved": r.improved, + "kept": r.kept, + "elapsed_sec": r.elapsed_sec, + "error": r.error, + }) + + best = None + if history.best_result: + br = history.best_result + best = { + "run_id": br.run_id, + "iteration": br.iteration, + "primary_metric": br.primary_metric, + "metrics": br.metrics, + } + + checkpoint_dir.joinpath("history.json").write_text( + json.dumps({"results": results, "best": best}, indent=2) + ) + + # Lightweight status file for quick polling + checkpoint_dir.joinpath("checkpoint.json").write_text( + json.dumps({ + "round": round, + "total_rounds": len(history.results), + "best_metric": history.best_result.primary_metric if history.best_result else None, + "updated_at": _time.time(), + }, indent=2) + ) + + logger.debug("[checkpoint] round=%d dir=%s", round, checkpoint_dir) + + # ------------------------------------------------------------------ + # Autogenesis: Reflect (SEPL reflection optimizer on early stop) + # ------------------------------------------------------------------ + + def _reflect( + self, + history: "ExperimentHistory", + spec: TaskSpec, + llm: Any, + lattice_comment_fn: "Callable[[str], None]", + run_dir: Path, + ) -> None: + """Synthesis pass after 3 non-improving iterations. + + Reads learnings.jsonl, asks the LLM to diagnose why the metric stalled, + and posts the diagnosis to Lattice. This is the SEPL reflection optimizer: + instead of iterating blindly, we re-examine whether the hypothesis was wrong. + """ + learnings_file = run_dir / "learnings.jsonl" + learnings: list[dict[str, Any]] = [] + if learnings_file.exists(): + for line in learnings_file.read_text(encoding="utf-8").splitlines(): + try: + learnings.append(json.loads(line)) + except json.JSONDecodeError: + pass + + best = history.best_result + best_metric = best.primary_metric if best else None + + if not llm: + lattice_comment_fn( + f"[reflect] Early stop after 3 non-improving rounds. " + f"Best {spec.metric_key}={best_metric}. " + f"llm=None — reflection skipped. Pass an LLM client to enable diagnosis." + ) + return + + if not learnings: + lattice_comment_fn( + f"[reflect] Early stop after 3 non-improving rounds. " + f"Best {spec.metric_key}={best_metric}. " + f"No learnings in learnings.jsonl — re-examine hypothesis manually." + ) + return + + learnings_summary = "\n".join( + f"- iter {e['source']}: {e['type']} | {e['key']}={e['confidence']} | {e['insight']}" + for e in learnings + ) + + prompt = ( + f"A research loop ran {len(learnings)} iterations on the following task:\n\n" + f"Topic: {spec.topic}\n" + f"Deliverable: {spec.deliverable}\n" + f"Metric: {spec.metric_key} ({spec.metric_direction})\n" + f"Best achieved: {best_metric}\n\n" + f"Round-by-round observations:\n{learnings_summary}\n\n" + "The loop stopped because 3 consecutive iterations did not improve the metric.\n\n" + "Diagnose:\n" + "1. Why did the metric stall? What is the fundamental bottleneck?\n" + "2. Was the hypothesis wrong — or was the approach right but the budget too small?\n" + "3. What ONE different approach would you try next if given another budget?\n\n" + "Be specific and concise. This diagnosis will be posted to the task tracker." + ) + + try: + response = llm.chat( + [{"role": "user", "content": prompt}], + system=( + "You are an expert research diagnostician. " + "Identify root causes, not symptoms. Be concrete and actionable." + ), + ) + diagnosis = getattr(response, "content", "").strip()[:1000] + except Exception as exc: + logger.warning("Reflection LLM call failed: %s", exc) + diagnosis = f"LLM reflection failed: {exc}" + + lattice_comment_fn( + f"[reflect] Early stop after {len(learnings)} rounds. " + f"Best {spec.metric_key}={best_metric}.\n\n" + f"Diagnosis:\n{diagnosis}" + ) + + # Persist the reflection as a special learning entry + reflection_entry = { + "type": "reflection", + "key": spec.metric_key, + "insight": diagnosis[:500], + "confidence": best_metric or 0.0, + "source": "reflect-final", + } + with learnings_file.open("a", encoding="utf-8") as f: + f.write(json.dumps(reflection_entry) + "\n") + + # ------------------------------------------------------------------ + # Autogenesis: Optimize — improvement proposal (Karpathy principles) + # ------------------------------------------------------------------ + + def _improve_attempt( + self, + llm: Any, + spec: TaskSpec, + current_attempt: str, + history: ExperimentHistory, + ) -> str: + """Propose a revised attempt using domain-aware Karpathy prompting.""" + last = history.results[-1] if history.results else None + best = history.best_result + last_metric = last.primary_metric if last else None + best_metric = best.primary_metric if best else None + last_stdout = last.stdout if last else "" + + _DOMAIN_VERB = { + "code": "Revise the code", + "search": "Revise your search strategy, queries, or result ranking", + "research": "Deepen or reframe your research synthesis", + "generic": "Revise your approach", + } + domain_verb = _DOMAIN_VERB.get(spec.task_type, "Revise your approach") + + _DOMAIN_HINT = { + "code": "Make surgical edits — every changed line must trace to your hypothesis. " + "No refactoring of unrelated sections.", + "search": "Test exactly ONE new query strategy or source. " + "Don't repeat what didn't work.", + "research": "Address exactly ONE identified gap (missing source, weak argument, " + "uncovered angle). Don't rewrite everything.", + "generic": "Change only what your hypothesis requires. " + "Minimum viable revision.", + } + domain_hint = _DOMAIN_HINT.get(spec.task_type, "") + + prompt = ( + f"Task: {spec.topic}\n" + f"Deliverable: {spec.deliverable}\n" + f"Metric: {spec.metric_key} ({spec.metric_direction})\n" + f"Last score: {last_metric}\n" + f"Best score: {best_metric}\n" + f"Last worker output (excerpt):\n{last_stdout[:800]}\n\n" + "---\n\n" + "Current attempt:\n" + f"{current_attempt}\n\n" + "---\n\n" + "## Think Before Revising (Karpathy Principle 1)\n\n" + "State:\n" + f"1. WHY is `{spec.metric_key}` at {last_metric}? What is the binding bottleneck?\n" + "2. Your ONE hypothesis for what change will move it.\n" + f"3. Success criterion: `{spec.metric_key}` moves from {last_metric} toward " + f"{'higher' if spec.metric_direction == 'maximize' else 'lower'}.\n\n" + "## Simplicity First\n\n" + "If the revision can be 5 lines, make it 5 lines — not 50.\n" + "No speculative additions. No features that don't serve the metric.\n\n" + f"## Your Task\n\n{domain_verb}. {domain_hint}\n\n" + "Return ONLY the revised attempt. " + "Include a brief comment at the top stating your hypothesis and what you changed." + ) + + system = ( + f"You are a {spec.task_type} improvement specialist. " + "Apply the Karpathy loop: think first, make surgical changes, verify the metric moves. " + "Surface your reasoning. Never guess silently." + ) + + try: + response = llm.chat([{"role": "user", "content": prompt}], system=system) + except Exception as exc: + logger.exception("Improvement call failed: %s", exc) + return current_attempt + + candidate = getattr(response, "content", "") + if not isinstance(candidate, str) or not candidate.strip(): + logger.warning("LLM returned empty attempt; keeping current") + return current_attempt + + # For code tasks, extract from code fence if present + if spec.task_type == "code": + from agent.research.runner import ExperimentRunner + extracted = ExperimentRunner._extract_python_code(candidate) + return extracted if extracted.strip() else candidate.strip() + + return candidate.strip() + + # ------------------------------------------------------------------ + # Evolution — persist lessons across runs (HRM-59 v1) + # and surface them to next-run workers (HRM-62 v2) + # ------------------------------------------------------------------ + + _OVERLAY_MAX_CHARS = 1500 + _OVERLAY_MAX_LESSONS = 3 + + def _load_evolution_overlay(self) -> str: + """Load past-run lessons formatted for prepending to worker briefs. + + Reads from EvolutionStore at $HERMES_HOME/evolution and uses + build_overlay() with a research-loop scope. Capped at + _OVERLAY_MAX_CHARS to bound prompt-token cost. Returns empty + string on any failure — the loop must run with or without lessons. + """ + try: + from agent.research.evolution import EvolutionStore + + store_dir = get_hermes_home() / "evolution" + if not store_dir.exists(): + return "" + overlay = EvolutionStore(store_dir).build_overlay( + stage_name="research_loop", + max_lessons=self._OVERLAY_MAX_LESSONS, + ) + if len(overlay) > self._OVERLAY_MAX_CHARS: + overlay = overlay[: self._OVERLAY_MAX_CHARS] + "\n\n[... overlay truncated ...]" + return overlay + except Exception as exc: + logger.warning("Evolution overlay load failed: %s", exc) + return "" + + def _evolve(self, history: Any, spec: TaskSpec, run_id: str) -> None: + """Append per-iteration lessons from this run to the EvolutionStore. + + Adapter between ExperimentResult (Karpathy loop) and LessonEntry + (ResearchClaw schema). v1 only persists; prompt overlay injection + is intentionally deferred to v2 so this hook stays append-only and + cannot affect ongoing or future loops if it misbehaves. + """ + from datetime import datetime, timezone + from agent.research.evolution import ( + EvolutionStore, + LessonEntry, + LessonCategory, + _classify_error, + ) + + results = getattr(history, "results", []) or [] + if not results: + return + + now = datetime.now(timezone.utc).isoformat() + lessons: list[LessonEntry] = [] + + for result in results: + iteration = getattr(result, "iteration", 0) + stage_name = f"iter_{iteration}" + error = getattr(result, "error", None) + improved = getattr(result, "improved", False) + kept = getattr(result, "kept", False) + metric = getattr(result, "primary_metric", None) + + if error: + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=iteration, + category=_classify_error(stage_name, str(error)), + severity="error", + description=f"{spec.metric_key}: {error}", + timestamp=now, + run_id=run_id, + )) + elif improved and kept: + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=iteration, + category=LessonCategory.PIPELINE, + severity="info", + description=f"{spec.metric_key} improved to {metric}", + timestamp=now, + run_id=run_id, + )) + else: + lessons.append(LessonEntry( + stage_name=stage_name, + stage_num=iteration, + category=LessonCategory.PIPELINE, + severity="warning", + description=f"{spec.metric_key}={metric} no improvement, attempt discarded", + timestamp=now, + run_id=run_id, + )) + + store_dir = get_hermes_home() / "evolution" + store = EvolutionStore(store_dir) + store.append_many(lessons) + + # ------------------------------------------------------------------ + # LLM judge evaluator + # ------------------------------------------------------------------ + + def _score_with_llm_judge( + self, + deliverable: str, + spec: TaskSpec, + llm: Any, + ) -> float | None: + """Score a deliverable using an LLM judge. Returns 0.0–1.0 or None.""" + eval_prompt = spec.evaluation_prompt or ( + f"Score the following deliverable for the task '{spec.topic}' " + f"on a scale of 0.0 to 1.0, where 1.0 = perfect. " + f"Return ONLY a decimal number, nothing else." + ) + prompt = f"{eval_prompt}\n\nDeliverable:\n{deliverable[:4000]}\n\nScore (0.0–1.0):" + content = "" + try: + response = llm.chat( + [{"role": "user", "content": prompt}], + system="You are an objective evaluator. Return only a decimal number between 0.0 and 1.0.", + ) + content = (getattr(response, "content", "") or "").strip() + if not content: + logger.warning("LLM judge returned empty response") + return None + # Extract the first decimal found anywhere — tolerates prose like + # "Score: 0.85", "0.8/1.0", "The score is 0.7 because …". + match = _JUDGE_SCORE_RE.search(content) + if match is None: + logger.warning( + "LLM judge response had no numeric score; raw=%r", + content[:200], + ) + return None + return max(0.0, min(1.0, float(match.group()))) + except Exception as exc: + logger.warning( + "LLM judge scoring failed: %s; raw=%r", exc, content[:200] + ) + return None + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +def _extract_iteration(working_dir: str) -> int: + try: + return int(working_dir.rsplit("iter", 1)[-1]) + except (ValueError, IndexError): + return 0 diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py index dcc514b90146..ac362ff04f8a 100644 --- a/agent/subdirectory_hints.py +++ b/agent/subdirectory_hints.py @@ -23,6 +23,9 @@ logger = logging.getLogger(__name__) +# Module-level cache for subdirectory hints to avoid repeated disk reads +_hint_cache: dict[tuple[str, str], Optional[str]] = {} + # Context files to look for in subdirectories, in priority order. # Same filenames as prompt_builder.py but we load ALL found (not first-wins) # since different subdirectories may use different conventions. @@ -170,6 +173,13 @@ def _is_valid_subdir(self, path: Path) -> bool: def _load_hints_for_directory(self, directory: Path) -> Optional[str]: """Load hint files from a directory. Returns formatted text or None.""" + cache_key = (str(directory), str(self.working_dir)) + if cache_key in _hint_cache: + cached = _hint_cache[cache_key] + if cached is not None: + self._loaded_dirs.add(directory) + return cached + self._loaded_dirs.add(directory) found_hints = [] @@ -208,6 +218,7 @@ def _load_hints_for_directory(self, directory: Path) -> Optional[str]: logger.debug("Could not read %s: %s", hint_path, exc) if not found_hints: + _hint_cache[cache_key] = None return None sections = [] @@ -216,9 +227,11 @@ def _load_hints_for_directory(self, directory: Path) -> Optional[str]: f"[Subdirectory context discovered: {rel_path}]\n{content}" ) + result = "\n\n".join(sections) + _hint_cache[cache_key] = result logger.debug( "Loaded subdirectory hints from %s: %s", directory, [h[0] for h in found_hints], ) - return "\n\n".join(sections) + return result diff --git a/gateway/platforms/daemoncraft.py b/gateway/platforms/daemoncraft.py index c994ae60d73b..3b1adf075280 100644 --- a/gateway/platforms/daemoncraft.py +++ b/gateway/platforms/daemoncraft.py @@ -13,18 +13,93 @@ """ import asyncio +import datetime as _dt import json import logging import os import random import time import uuid +from pathlib import Path from typing import Any, Dict, Optional, Set import aiohttp from aiohttp import WSMsgType from gateway.config import Platform, PlatformConfig + +# --------------------------------------------------------------------------- +# CycleDetector — ported from daemoncraft agents/safety.py (stdlib-only) +# --------------------------------------------------------------------------- +import hashlib +import json as _json +from collections import deque +from dataclasses import dataclass, field +from typing import Deque + + +def _cd_canonicalize(args) -> str: + try: + if isinstance(args, str): + try: + args = _json.loads(args) + except Exception: + return args + return _json.dumps(args, sort_keys=True, default=str) + except Exception: + return repr(args) + + +def _cd_signature(name: str, args) -> str: + payload = f"{name}|{_cd_canonicalize(args)}".encode("utf-8") + return hashlib.sha256(payload).hexdigest()[:16] + + +@dataclass +class _CycleResult: + triggered: bool + sig: Optional[str] + count: int + window: int + action: str + + +@dataclass +class CycleDetector: + """Ring-buffer cycle detector for repeated tool-call patterns.""" + n: int = 4 + window: int = 6 + action: str = "warn" + _buf: Deque[str] = field(default_factory=deque) + _last_triggered_sig: Optional[str] = None + + def __post_init__(self) -> None: + self._buf = deque(maxlen=max(self.window, self.n)) + + def record(self, name: str, args) -> _CycleResult: + sig = _cd_signature(name, args) + self._buf.append(sig) + return self._evaluate() + + def _evaluate(self) -> _CycleResult: + if len(self._buf) < self.n: + return _CycleResult(False, None, 0, len(self._buf), self.action) + counts: Dict[str, int] = {} + for s in self._buf: + counts[s] = counts.get(s, 0) + 1 + top_sig, top_count = max(counts.items(), key=lambda kv: kv[1]) + if top_count >= self.n: + if top_sig == self._last_triggered_sig: + return _CycleResult(False, top_sig, top_count, len(self._buf), self.action) + self._last_triggered_sig = top_sig + return _CycleResult(True, top_sig, top_count, len(self._buf), self.action) + if self._last_triggered_sig and self._last_triggered_sig != top_sig: + self._last_triggered_sig = None + return _CycleResult(False, top_sig, top_count, len(self._buf), self.action) + + def reset(self) -> None: + self._buf.clear() + self._last_triggered_sig = None from gateway.platforms.base import BasePlatformAdapter, MessageEvent, MessageType, SendResult from gateway.session import SessionSource, build_session_key @@ -51,6 +126,7 @@ def __init__(self, config: PlatformConfig): self._voice_mode_default: str = "all" # DaemonCraft defaults to TTS for all replies self._last_tts_time: float = 0.0 self._tts_queue: list[dict] = [] # Dedup buffer for rapid-fire messages + self._cycle_detector: Optional[CycleDetector] = None # Load allowlist by UUID (preferred) or username fallback. raw_allow = os.getenv("DAEMONCRAFT_ALLOWED_USERS", "").strip() @@ -77,6 +153,13 @@ async def connect(self) -> bool: self._last_seen_timestamp = int(time.time() * 1000) self._shutdown_event.clear() self._session = aiohttp.ClientSession() + + n = int(os.getenv("MC_CYCLE_N", "0")) + window = int(os.getenv("MC_CYCLE_WINDOW", "20")) + action = os.getenv("MC_CYCLE_ACTION", "warn") + if n > 0: + self._cycle_detector = CycleDetector(n=n, window=window, action=action) + logger.info("[DaemonCraft] CycleDetector enabled: n=%d window=%d action=%s", n, window, action) self._ws_task = asyncio.create_task(self._ws_loop()) self._mark_connected() logger.info("[DaemonCraft] Connected to %s as %s", self._bot_api_url, self._bot_username) @@ -145,6 +228,8 @@ async def _on_ws_message(self, data: str) -> None: elif msg_type == "heartbeat_context": data = payload.get("data", {}) await self._handle_heartbeat_context(data) + elif msg_type == "action_result": + await self._handle_action_result(payload) elif msg_type == "interrupt": # Loop-to-gateway interrupt acknowledgment — no action needed pass @@ -300,6 +385,12 @@ async def _handle_blueprint_updated(self, data: dict) -> None: ) await self.handle_message(event) + async def _handle_action_result(self, payload: dict) -> None: + """Forward action_result events to transform_tool_result hooks.""" + import json as _json + result_str = _json.dumps(payload) + await self.invoke_hook("transform_tool_result", tool_name="mc_action_result", result=result_str) + async def _handle_heartbeat_context(self, data: dict) -> None: """Process heartbeat_context with two-level event architecture. @@ -318,6 +409,10 @@ async def _handle_heartbeat_context(self, data: dict) -> None: logger.debug("[DaemonCraft] Context-only heartbeat injected silently") return + # Cycle guard — skip wake-up if loop is repeating mc_perceive calls + if await self._check_cycle("mc_perceive", {}): + return + # Wake-up event: force an agent turn with tool_choice=required source = self.build_source( chat_id="world", @@ -412,6 +507,28 @@ async def _inject_synthetic_perceive(self, data: dict) -> None: } self._session_store.append_to_transcript(session_id, assistant_msg) + + # Run transform_tool_result hooks so plugins (e.g. altercraft scene-graph) + # can consume synthetic mc_perceive on the same path as real tool results. + try: + from hermes_cli.plugins import invoke_hook + for hook_result in invoke_hook( + "transform_tool_result", + tool_name="mc_perceive", + args={}, + result=payload, + task_id="", + session_id=session_id, + tool_call_id=tool_call_id, + duration_ms=0, + ): + if isinstance(hook_result, str): + payload = hook_result + tool_msg["content"] = payload + break + except Exception as _hook_exc: + logger.debug("[DaemonCraft] transform_tool_result hook error: %s", _hook_exc) + self._session_store.append_to_transcript(session_id, tool_msg) logger.info("[DaemonCraft] Synthetic mc_perceive injected into session %s", session_id) @@ -496,6 +613,143 @@ async def _handle_chat_entry(self, entry: dict) -> None: await self.handle_message(event) + # ------------------------------------------------------------------ + # Cycle detection + # ------------------------------------------------------------------ + + async def _check_cycle(self, tool_name: str, args: dict) -> bool: + """Check tool-call cycle. Returns True if cycle detected and action is 'interrupt'.""" + if self._cycle_detector is None: + return False + result = self._cycle_detector.record(tool_name, args) + if result.triggered: + if result.action == "interrupt": + logger.warning( + "[DaemonCraft] Cycle detected for '%s' (%d/%d) — interrupting agent", + tool_name, result.count, result.window, + ) + await self._interrupt_agent("cycle_detected") + return True + else: + logger.warning( + "[DaemonCraft] Cycle detected for '%s' (%d/%d) — action=%s", + tool_name, result.count, result.window, result.action, + ) + return False + + # ------------------------------------------------------------------ + # Dashboard feed (DC-123) + # ------------------------------------------------------------------ + + async def on_processing_complete(self, event, outcome) -> None: + """POST the last assistant turn to /agent/log so the dashboard Bot Mind panel populates. + + Before DC-112 the agent_loop posted turns directly. After DC-112 cognition + moved to the gateway but no one wired the log relay. This hook restores + visibility without touching the loop. + """ + if not self._bot_api_url or not self._session: + return + try: + session_id = self._get_world_session_id() + if not session_id or not self._session_store: + return + transcript = self._session_store.load_transcript(session_id) + # Find the last assistant message in the transcript + last_assistant = None + tool_calls = [] + for msg in reversed(transcript): + role = msg.get("role", "") + if role == "assistant" and last_assistant is None: + content = msg.get("content", "") + if isinstance(content, list): + # Extract text and tool_use blocks + text_parts = [b.get("text", "") for b in content if b.get("type") == "text"] + tool_calls = [ + {"name": b.get("name"), "input": b.get("input")} + for b in content if b.get("type") == "tool_use" + ] + last_assistant = "\n".join(text_parts).strip() + else: + last_assistant = str(content) + break + + if last_assistant is None and not tool_calls: + return + + await self._session.post( + f"{self._bot_api_url}/agent/log", + json={ + "turn": len(transcript), + "time": int(time.time() * 1000), + "prompt": "", # omit — transcript is large; response + tools is what the panel needs + "response": last_assistant or "", + "tool_calls": tool_calls, + "error": None, + }, + ) + except Exception as e: + logger.debug("[DaemonCraft] on_processing_complete /agent/log post failed: %s", e) + + # DC-132 — emit a turn metric (best-effort; never raises). + # Latency: time since the last user/perceive message in the transcript, + # if we can find one. tokens_in/out: not yet exposed by AIAgent at this + # hook, so we emit zero placeholders rather than fabricate values. + try: + self._emit_metric( + "turn", + tokens_in=0, + tokens_out=0, + latency_ms=None, + tool_call_count=len(tool_calls), + ) + for tc in tool_calls: + self._emit_metric("tool", tool=tc.get("name") or "?", ok=True) + except Exception: + pass + + # ------------------------------------------------------------------ + # DC-132 — JSONL metrics (mirrors agents/agent_loop.py emitter in daemoncraft) + # ------------------------------------------------------------------ + + def _emit_metric(self, kind: str, **fields) -> None: + """Append a JSON line to ~/.hermes/metrics//.jsonl. + + Schema is documented in scripts/agent-metrics-report.py in the + daemoncraft repo. This is the gateway counterpart to the heartbeat + emitter in agent_loop.py — together they cover the four families + the report script aggregates. + + Cast comes from DAEMONCRAFT_METRICS_CAST env var; falls back to the + bot username so events still group sensibly if the operator hasn't + set it. No env var → emitter still fires under the username. + """ + try: + cast = os.getenv("DAEMONCRAFT_METRICS_CAST", "").strip() or self._bot_username or "daemoncraft" + metrics_root = Path(os.getenv("DAEMONCRAFT_METRICS_DIR", str(Path.home() / ".hermes" / "metrics"))) + now = _dt.datetime.utcnow() + cast_dir = metrics_root / cast + cast_dir.mkdir(parents=True, exist_ok=True) + path = cast_dir / f"{now.date().isoformat()}.jsonl" + record = { + "ts": now.isoformat(timespec="seconds") + "Z", + "cast": cast, + "agent": self._bot_username or "?", + "kind": kind, + **fields, + } + # Single os.write() with O_APPEND — POSIX-atomic for writes + # under PIPE_BUF (typically 4 KB on Linux). Prevents truncated + # lines under concurrent writers / mid-write process kill. + line = (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) + try: + os.write(fd, line) + finally: + os.close(fd) + except Exception: + pass + # ------------------------------------------------------------------ # Outbound # ------------------------------------------------------------------ @@ -527,11 +781,49 @@ async def send( body = await resp.text() logger.warning("[DaemonCraft] /chat/send failed: %s %s", resp.status, body) return SendResult(success=False, error=f"HTTP {resp.status}: {body}") - return SendResult(success=True) except Exception as e: logger.warning("[DaemonCraft] /chat/send exception: %s", e) return SendResult(success=False, error=str(e), retryable=True) + # DC-123: relay TTS to dashboard after every successful outbound message. + # Before DC-112 the agent_loop generated TTS explicitly. Now the gateway + # owns all cognition and must drive TTS itself. We skip PASS/empty + # heartbeat responses and metadata-flagged suppression. + if (content and content.strip() not in ("PASS", "") + and not (metadata or {}).get("suppress_tts")): + asyncio.create_task(self._generate_and_relay_tts(content, chat_id)) + + return SendResult(success=True) + + async def _generate_and_relay_tts(self, text: str, chat_id: str) -> None: + """Generate TTS for outbound text and relay audio to the dashboard. + + DC-123 fix: before DC-112 agent_loop called TTS explicitly. After DC-112 + the gateway owns cognition but the TTS relay was never wired. This method + closes that gap — it is called as a fire-and-forget task from send(). + """ + try: + from tools.tts_tool import text_to_speech_tool, check_tts_requirements + if not check_tts_requirements(): + return + import re as _re, json as _json + # Strip Minecraft formatting codes and markdown before synthesis. + clean = _re.sub(r'§[0-9a-fklmnor]', '', text) + clean = _re.sub(r'[*_`#\[\]()]', '', clean).strip() + if not clean: + return + tts_result = await asyncio.to_thread(text_to_speech_tool, text=clean[:4000]) + tts_data = _json.loads(tts_result) + audio_path = tts_data.get("file_path") + if audio_path and os.path.exists(audio_path): + await self._copy_and_relay_tts(audio_path, chat_id) + try: + os.remove(audio_path) + except OSError: + pass + except Exception as e: + logger.debug("[DaemonCraft] TTS generation failed: %s", e) + async def _copy_and_relay_tts(self, audio_path: str, chat_id: str) -> SendResult: """Copy audio to shared TTS cache and POST /tts/play to dashboards.""" try: diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 95a13b88501d..d38d59a4f0cc 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -7773,6 +7773,22 @@ def cmd_profile(args): print(f"Error: {e}") sys.exit(1) + elif action == "setup": + name = args.profile_name + template = getattr(args, "template", None) or name + if template == "researcher": + from hermes_cli.researcher_scaffold import setup_researcher_profile + try: + print(f"\nSetting up '{name}' profile with researcher scaffold...") + setup_researcher_profile(name) + except FileNotFoundError as e: + print(f"Error: {e}") + sys.exit(1) + else: + print(f"No scaffold template available for '{template}'.") + print("Available templates: researcher") + sys.exit(1) + elif action == "delete": name = args.profile_name yes = getattr(args, "yes", False) @@ -9957,6 +9973,16 @@ def cmd_acp(args): "--no-alias", action="store_true", help="Skip wrapper script creation" ) + profile_setup = profile_subparsers.add_parser( + "setup", help="Apply a scaffold template to an existing profile" + ) + profile_setup.add_argument("profile_name", help="Profile to configure") + profile_setup.add_argument( + "--template", + metavar="TEMPLATE", + help="Scaffold template to apply (default: profile name). Available: researcher", + ) + profile_delete = profile_subparsers.add_parser("delete", help="Delete a profile") profile_delete.add_argument("profile_name", help="Profile to delete") profile_delete.add_argument( diff --git a/hermes_cli/researcher_scaffold.py b/hermes_cli/researcher_scaffold.py new file mode 100644 index 000000000000..b189276fa0f0 --- /dev/null +++ b/hermes_cli/researcher_scaffold.py @@ -0,0 +1,401 @@ +"""Bootstrap the 'researcher' profile with research-specific config, SOUL, and memories. + +Usage: + hermes profile create researcher + hermes profile setup researcher + researcher chat +""" + +from __future__ import annotations + +from pathlib import Path + +# --------------------------------------------------------------------------- +# Content constants +# --------------------------------------------------------------------------- + +_CONFIG_YAML = """\ +# Researcher profile — optimised for iterative self-improving research loops +# This agent is a node in the altermundi operational chain. It reads from +# and writes to the shared vault (Markdown + git at $HERMES_VAULT_PATH) and reports progress +# via the shared task tracker (Lattice). +model: + default: kimi-k2.6 + provider: kimi-coding + +toolsets: + - research # run_research: Karpathy + Autogenesis AOOR loop + - web # search/research workers need web access + - file # read/write artifacts + - delegation # run_research uses delegate_task internally + - terminal # code tasks need terminal + - memory # persist research findings across sessions + - session_search + - skills + - todo + +# MCP servers — intentionally none +# Vault interaction goes through standard file + grep + git tools. +# Lattice task tracking goes through the `lattice` CLI in the terminal. +# No MCP layer over either — both already have first-class CLI/text interfaces. +# Set HERMES_VAULT_PATH for the vault root and LATTICE_ROOT for the task tracker. +mcp_servers: {} + +agent: + max_turns: 80 + reasoning_effort: high + verbose: false +""" + +_SOUL_MD = """\ +You are a Research Agent powered by the Karpathy self-improvement loop and the +Autogenesis self-evolution protocol (Act → Observe → Optimize → Remember). + +You are NOT an isolated assistant. You are a node in the **altermundi operational +chain**, connected to two shared systems: + +- **Vault** (plain Markdown + git, at `$HERMES_VAULT_PATH`): The team's shared + LLM-wiki — knowledge graph, specs, runbooks, and accumulated research. Read + with standard file tools (`grep -r`, `cat`, `Read`). Write with `Write`/`Edit` + and commit with `git` so changes are versioned and reviewable. **No MCP layer** + — the vault is just files in a repo. +- **Lattice** (CLI: `lattice`): The team's event-sourced task tracker — every + research run must be tracked as a Lattice task with round-by-round progress + comments. This is the audit trail and coordination layer. Invoke via the + terminal — `lattice create`, `lattice comment`, `lattice complete`, etc. + +## Operational Context + +Before starting any research: +1. **Search the vault** for existing work — `grep -r "" $HERMES_VAULT_PATH` +2. **Read relevant notes directly** with `Read` / `cat` to avoid duplicating effort +3. **Create a Lattice task** for tracking — `lattice create "Research: " --actor agent:researcher` +4. After completion, **write findings into the vault**, **commit with git**, + and **close the Lattice task** + +After research completes: +1. Write a summary note to `$HERMES_VAULT_PATH/Research/.md` +2. `cd $HERMES_VAULT_PATH && git add Research/.md && git commit -m "research: "` +3. Link the note path in the Lattice task comment +4. Close the Lattice task with `complete` (not `status`): + ``` + lattice complete --actor agent:researcher --review "" + ``` + +**Degraded mode**: If the `lattice` CLI is unavailable (binary missing, LATTICE_ROOT +unwritable), declare degraded mode: +- State: "Lattice offline — running without coordination integration" +- Continue research if the core task is still possible +- Vault read/write still works — it's just files +- Verify `lattice doctor` passes before the next research run + +## Core Behavior + +Your primary tool is `run_research`. Use it when a task requires iterative +refinement toward a measurable quality criterion. For simple lookups or +one-shot tasks, use `delegate_task` directly. + +## When to use `run_research` + +- User asks to "research", "investigate", "find the best", "optimize", "study" +- Task has a clear quality criterion: relevance, accuracy, completeness, latency +- A single attempt is unlikely to be sufficient — the topic needs iteration +- You can define a numeric metric (0–1 score, pass rate, ms latency, etc.) + +## When NOT to use `run_research` + +- Simple factual questions → answer directly from knowledge +- One-off file operations or code edits → use `delegate_task` +- Tasks with no measurable outcome → use `delegate_task` + +## CRITICAL: Do NOT manually construct AIAgent + +The old pattern of importing `AIAgent` from `run_agent.py` and calling it +inside `execute_code` is DEPRECATED. `run_research` already spawns workers +via `delegate_task` internally. Just call the tool directly. + +## Choosing parameters (by task type) + +| Situation | Recommended metric_key | evaluation_mode | Notes | +|-----------|------------------------|-----------------|-------| +| Code optimization | `latency_ms` or `throughput` | `self_report` | pass_rate is baseline-only; optimize for speed/memory | +| Code correctness | `pass_rate` | `self_report` | Start here, then switch to latency_ms | +| Literature/web search | `relevance_score` | `llm_judge` | Specific criteria beat generic scoring | +| Research synthesis | `completeness_score` | `llm_judge` | 0–1 scale, evaluate against rubric | +| Algorithm design | `time_to_solution` or `iterations_to_converge` | `self_report` | Measures efficiency, not just correctness | +| Ambiguous quality | (custom) | `llm_judge` | Write a clear evaluation_prompt | + +## Metric selection guide + +- **Code tasks**: Start with `pass_rate` to get a working baseline. Once + baseline = 1.0, run a SECOND `run_research` with `latency_ms` or + `throughput` to optimize performance. This is a manual pivot, not automatic. +- **Search tasks**: `relevance_score` (0–1) with a specific llm_judge prompt + like "Score 0-1: does this list cover X published after 2022?" +- **Research tasks**: `completeness_score` (0–1) with rubric in evaluation_prompt +- **Generic tasks**: Pick the ONE number that best captures "better". If you + can't define it numerically, use `llm_judge`. + +## Lattice tracking workflow + +1. Before calling `run_research`, create a Lattice task for tracking: + ``` + lattice create "Research: " --actor agent:researcher + ``` +2. Pass the task ID as `lattice_task_id` to `run_research` +3. The supervisor auto-posts round-by-round progress comments to Lattice +4. After completion, update Lattice status and link the workspace path + +## Before calling `run_research` + +1. Clarify the metric with the user if unclear ("what does 'good' mean here?") +2. Tell the user: "I'll run a research loop — this may take a few minutes." +3. Set a specific `evaluation_prompt` for llm_judge tasks +4. Start with max_iterations=3, time_budget_sec=0 (unlimited); increase only if needed +5. For code tasks: if pass_rate is already 1.0, use latency_ms or throughput + +## After `run_research` returns + +1. State: best metric achieved + number of iterations +2. Summarize the key finding or deliverable in plain language +3. Offer to run more iterations if the metric didn't converge +4. Point to `workspace` path if the user wants raw artifacts +5. If `lattice_task_id` was set, confirm round comments were posted + +## Research integrity + +- Never fabricate findings — only report what `run_research` actually produced +- If the metric is low, say so honestly and diagnose why +- Cite the `learnings_file` as the audit trail for your conclusions + +## Autonomous execution mode + +When the user prompt contains explicit phrasing like "do not ask", "no preguntes", +"execute autonomously", "no permission", or "iterate without asking": + +- DO NOT offer to "write a script if you'd like" — write it and run it. +- DO NOT request clarification when the task is well-scoped — proceed with reasonable assumptions and document them in the result. +- DO NOT halt on the first tool error — diagnose, attempt one alternative, then proceed with what you have. +- DO NOT escape to the user mid-task — finish the work and report what you did, including failures. + +If a task is genuinely impossible (missing capability, locked file, unreachable +service), STILL complete the protocol: emit the FAIL marker the prompt asked for, +explain the obstacle in the report, do not request input. + +Counterexample (do not do this): "If you'd like me to write the orchestration +script anyway (as a deliverable), I can produce a clean Python script... Just +let me know which path to take." This is bailing in autonomous mode. + +## Tool usage patterns (lessons from prior research swarms) + +These patterns avoid common errors observed in past research sessions. Follow +them by default — they save tool calls and prevent retries. + +### Long Lattice comments — write to file, then heredoc + +Inline comment text with embedded quotes, newlines, or `$()` expansions is +fragile. The reliable pattern: + +``` +write_file /tmp/-comment.txt "" +cd $LATTICE_ROOT && lattice comment "$(cat /tmp/-comment.txt)" --actor agent:researcher +``` + +Skip the inline-first attempt. Go straight to file + cat for anything over +two lines. + +### File reads — generous range, no re-reads + +Read with explicit offset+limit covering what you need on the first pass. +Re-read a file only after you have *edited* it; do not re-read by inertia +to "remember the section." If you genuinely need a different section than +the first read, request it once with the right offset. + +### `grep` alternation — use `-E` or `-P`, never `\|` + +Bash escape of `\|` inside double quotes is fragile and frequently fails. +Always: + +``` +grep -E "pattern_a|pattern_b" # extended regex +grep -P "pattern_a|pattern_b" # perl-compat +``` + +Never `grep "pattern_a\|pattern_b"`. + +### Heredoc tag must not appear in body + +If your content might contain words like `ANALYSIS`, `EOF`, `END`, do not +use them as the heredoc tag. Use a unique, scoped tag: + +``` +cat <<'EOF_HRM57' > /tmp/x.txt +... content that may contain EOF or ANALYSIS literally ... +EOF_HRM57 +``` + +### Do NOT use `execute_code` to import internal Hermes modules + +`from hermes_tools import read_file` and similar do not work — these are +agent tools, not Python modules. Use the `read_file` tool dispatch directly. +`execute_code` is for *running computation*, not for tool routing. +""" + +_MEMORY_MD = """\ +--- +name: Research Agent Bootstrap Memory +description: Initial patterns and workspace info for the researcher profile +type: project +--- + +## Workspace + +Research artifacts live in: ~/.hermes/research-workspace/ +Each `run_research` call creates a subdirectory named by run_id: + - learnings.jsonl — HeartbeatMemorySystem schema: type/key/insight/confidence/source + - round-*/task_brief.md — worker instructions per iteration + - round-*/attempt.py or attempt.md — actual deliverable per round + - round-*/results.json — structured metrics + +## Codebase layout — research subsystem + +When investigating the AutoResearch implementation, these are the canonical +paths. Read directly; do not `find` or `grep` to discover them. + +| Path | Role | +|------|------| +| `agent/research/supervisor.py` | Karpathy loop core — `ResearchSupervisor`, `TaskSpec`, `_build_task_brief`, `_score_with_llm_judge` | +| `agent/research/runner.py` | `ExperimentRunner`, `ExperimentHistory`, `ExperimentResult` | +| `agent/research/job_runner.py` | Detached OS process entrypoint — `_build_agent`, `main` | +| `agent/research/evolution.py` | `EvolutionStore`, `extract_lessons` (vendored, currently unwired) | +| `agent/research/metrics.py` | `UniversalMetricParser` for results.json + stdout | +| `tools/research_tool.py` | `run_research` tool handler + `_LLMBridge` | +| `tools/research_job_tool.py` | `research_job` tool (start/status/collect/resume) | +| `tools/delegate_tool.py` | `delegate_task`, `_build_child_agent` (~line 967) | +| `skills/autoresearch/` | Bundled skills: `karpathy-guidelines`, `a-evolve`, 7 domain skills | +| `tests/agent/test_research_supervisor.py` | 18 integration tests | +| `HERMES_RESEARCH.md`, `RESEARCH_AGENTS.md`, `RESEARCH_OPERATIONS.md` | Top-level docs | + +## Vault integration (plain Markdown + git, no MCP) + +The vault at `$HERMES_VAULT_PATH` is just a git repo of Markdown files. +Interact with standard tools — no abstraction layer. + +- **Pre-flight**: Search for existing research before starting + ``` + grep -ri "fibonacci optimization" $HERMES_VAULT_PATH + ``` +- **During**: Read specs, runbooks, or prior research notes + ``` + cat $HERMES_VAULT_PATH/Research/Fibonacci\ Optimization.md + ``` +- **Post-flight**: Write findings back and commit + ``` + cat >> $HERMES_VAULT_PATH/Research/Fibonacci\ Optimization.md <<'EOF' + + ## Results + ... + EOF + cd $HERMES_VAULT_PATH && git add -A && git commit -m "research: fibonacci optimization results" + ``` +- **History**: Use `git log` / `git blame` to trace who wrote what, when, and why. + +**Naming convention**: `Research/.md` for research outputs. +**Why no MCP**: the vault is plain Markdown; standard text + git tools are +simpler, more debuggable, and let any agent (not just Hermes) interact with it. + +## Lattice integration (CLI) + +Lattice is the coordination layer. Invoke via the terminal — no MCP layer. + +- **Task creation**: Every research run starts with a Lattice task + ``` + lattice create "Research: " --actor agent:researcher + ``` +- **Progress tracking**: The supervisor auto-posts round comments, but you can + also post manual updates + ``` + lattice comment LAT-42 "Baseline complete: pass_rate=1.0" --actor agent:researcher + ``` +- **Completion**: Mark done and link artifacts (use `complete`, not `status`): + ``` + lattice complete --actor agent:researcher --review "" + ``` +- **History/audit**: `lattice show `, `lattice list`, `lattice comments `. + +## Metric patterns by task type + +| Task type | Phase 1 metric | Phase 2 metric | Why | +|-----------|---------------|----------------|-----| +| Code (new) | pass_rate | latency_ms or throughput | Baseline correctness, then optimize | +| Code (existing) | latency_ms | memory_mb | Already correct, optimize speed/resource | +| Search | relevance_score | coverage_score | Quality first, then completeness | +| Research | completeness_score | depth_score | Breadth first, then depth | +| Algorithm | pass_rate | iterations_to_converge | Correctness, then efficiency | + +**Anti-pattern**: Using pass_rate for code optimization after baseline is already +1.0. The supervisor sees no improvement and wastes iterations. Switch to a +performance metric. + +## Lattice integration pattern + +1. `lattice create "Research: " --actor agent:researcher` +2. Capture task_id from output +3. Call `run_research` with `lattice_task_id=` +4. Supervisor auto-posts per-round comments +5. After completion: `lattice complete --actor agent:researcher --review ""` +6. Optional: `lattice comment "Workspace: " --actor agent:researcher` + +## Patterns that work well + +- For literature search: evaluation_mode="llm_judge" with specific criteria beats self_report +- For code tasks: start with a minimal baseline, keep time_budget_sec=0 (unlimited) +- For generic research: metric_key="completeness_score" with 0-1 scale is broadly applicable +- When metric stalls after 3 rounds: read learnings.jsonl to diagnose the bottleneck +- Two-phase code research: first `pass_rate` baseline, then `latency_ms` optimization + +## Toolset notes + +- run_research internally uses delegate_task — both toolsets must be enabled +- search/research task_type workers use web+file toolsets automatically +- code task_type workers use terminal+file toolsets automatically +- DO NOT manually construct AIAgent inside execute_code — use run_research directly +""" + + +# --------------------------------------------------------------------------- +# Setup function +# --------------------------------------------------------------------------- + +def setup_researcher_profile(profile_name: str = "researcher") -> None: + """Write research-specific config, SOUL.md, and MEMORY.md to a profile. + + The profile must already exist (created via `hermes profile create `). + This function overwrites config.yaml, SOUL.md, and memories/MEMORY.md with + researcher-optimised content. + """ + from hermes_cli.profiles import get_profile_dir + + profile_dir = get_profile_dir(profile_name) + if not profile_dir.exists(): + raise FileNotFoundError( + f"Profile '{profile_name}' not found. " + f"Run: hermes profile create {profile_name}" + ) + + # config.yaml + (profile_dir / "config.yaml").write_text(_CONFIG_YAML, encoding="utf-8") + print(f" ✓ config.yaml") + + # SOUL.md + (profile_dir / "SOUL.md").write_text(_SOUL_MD, encoding="utf-8") + print(f" ✓ SOUL.md") + + # memories/MEMORY.md + memories_dir = profile_dir / "memories" + memories_dir.mkdir(exist_ok=True) + (memories_dir / "MEMORY.md").write_text(_MEMORY_MD, encoding="utf-8") + print(f" ✓ memories/MEMORY.md") + + print(f"\nResearcher profile ready at: {profile_dir}") + print(f"Start a session with: {profile_name} chat") diff --git a/model_tools.py b/model_tools.py index b991780a618c..b60df1249244 100644 --- a/model_tools.py +++ b/model_tools.py @@ -649,6 +649,7 @@ def handle_function_call( user_task: Optional[str] = None, enabled_tools: Optional[List[str]] = None, skip_pre_tool_call_hook: bool = False, + parent_agent: Any = None, ) -> str: """ Main function call dispatcher that routes calls to the tool registry. @@ -725,12 +726,14 @@ def handle_function_call( function_name, function_args, task_id=task_id, enabled_tools=sandbox_enabled, + parent_agent=parent_agent, ) else: result = registry.dispatch( function_name, function_args, task_id=task_id, user_task=user_task, + parent_agent=parent_agent, ) duration_ms = int((time.monotonic() - _dispatch_start) * 1000) diff --git a/run_agent.py b/run_agent.py index 98b83beb8ce9..4d4afcda32d2 100644 --- a/run_agent.py +++ b/run_agent.py @@ -949,6 +949,17 @@ def __init__( checkpoints_enabled: bool = False, checkpoint_max_snapshots: int = 50, pass_session_id: bool = False, + persist_session: bool = True, + # Detached / non-CLI invariants. Default values match the historical + # in-process initialization: depth 0, env-derived cwd, lazy hint + # tracker, no spinner, no progress callback. Detached entrypoints + # (research_job_runner, future cron / batch parents) override these + # to avoid the post-init patch block that the agent.factory module + # used to apply manually. See HRM-57. + delegate_depth: int = 0, + terminal_cwd: str | None = None, + cwd: str | None = None, + subdirectory_hints: Any = None, ): """ Initialize the AI Agent. @@ -1189,10 +1200,15 @@ def __init__( self._tool_worker_threads: set[int] = set() self._tool_worker_threads_lock = threading.Lock() - # Subagent delegation state - self._delegate_depth = 0 # 0 = top-level agent, incremented for children - self._active_children = [] # Running child AIAgents (for interrupt propagation) + # Subagent delegation state — caller may seed depth (HRM-57) for + # detached parents that simulate a top-level agent in a non-CLI flow. + self._delegate_depth = delegate_depth # 0 = top-level agent, incremented for children + self._active_children = [] # Running child AIAgents (for interrupt propagation) self._active_children_lock = threading.Lock() + # Spinner is normally attached during run_conversation; pre-init to + # None so detached parents that hand the agent off to delegate_task + # without ever entering an interactive loop don't AttributeError. + self._delegate_spinner = None # Store OpenRouter provider preferences self.providers_allowed = providers_allowed @@ -2072,9 +2088,18 @@ def __init__( except Exception as _ce_err: logger.debug("Context engine on_session_start: %s", _ce_err) - self._subdirectory_hints = SubdirectoryHintTracker( - working_dir=os.getenv("TERMINAL_CWD") or None, - ) + # Detached parents (HRM-57) may pass subdirectory_hints=None or a + # pre-built tracker; otherwise build the env-derived default. + if subdirectory_hints is not None: + self._subdirectory_hints = subdirectory_hints + else: + self._subdirectory_hints = SubdirectoryHintTracker( + working_dir=terminal_cwd or os.getenv("TERMINAL_CWD") or None, + ) + # terminal_cwd / cwd are exposed on the instance for delegate_task + # consumers; default to env-derived values when not passed. + self.terminal_cwd = terminal_cwd or os.getenv("TERMINAL_CWD") or os.getcwd() + self.cwd = cwd or os.getcwd() self._user_turn_count = 0 # Cumulative token usage for the session @@ -9358,6 +9383,7 @@ def _invoke_tool(self, function_name: str, function_args: dict, effective_task_i session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + parent_agent=self, ) @staticmethod @@ -9688,7 +9714,9 @@ def _run_tool(index, tool_call, function_name, function_args): env=get_active_env(effective_task_id), ) - subdir_hints = self._subdirectory_hints.check_tool_call(name, args) + subdir_hints = None + if self._subdirectory_hints: + subdir_hints = self._subdirectory_hints.check_tool_call(name, args) if subdir_hints: function_result += subdir_hints @@ -9984,6 +10012,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + parent_agent=self, ) _spinner_result = function_result except Exception as tool_error: @@ -10004,6 +10033,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + parent_agent=self, ) except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" @@ -10052,9 +10082,10 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe ) # Discover subdirectory context files from tool arguments - subdir_hints = self._subdirectory_hints.check_tool_call(function_name, function_args) - if subdir_hints: - function_result += subdir_hints + if self._subdirectory_hints: + subdir_hints = self._subdirectory_hints.check_tool_call(function_name, function_args) + if subdir_hints: + function_result += subdir_hints tool_msg = { "role": "tool", diff --git a/skills/autoresearch/a-evolve/SKILL.md b/skills/autoresearch/a-evolve/SKILL.md new file mode 100644 index 000000000000..7e986a33e038 --- /dev/null +++ b/skills/autoresearch/a-evolve/SKILL.md @@ -0,0 +1,202 @@ +--- +name: a-evolve +description: > + Apply A-Evolve's agentic evolution methodology to improve AI agent performance + across runs. Use when the user wants to diagnose agent failures, generate + targeted skills from error patterns, evolve system prompts, or accumulate + episodic knowledge. Works standalone or inside AutoResearchClaw pipelines. + Triggers on: "evolve", "self-improve", "diagnose failures", "generate skills + from errors", "what went wrong and how to fix it", or any mention of A-Evolve. +--- + +# A-Evolve: Agentic Evolution Skill + +Apply the **Solve → Observe → Evolve → Gate → Reload** methodology from +[A-Evolve](https://github.com/A-EVO-Lab/a-evolve) to iteratively improve +agent performance. This skill is prompt-based — no external dependencies, +no harness changes. You analyze failures, propose workspace mutations, and +generate durable artifacts (skills, prompt patches, knowledge entries) that +the agent can load in future runs. + +## Core Loop + +When asked to evolve or improve agent performance, follow this 5-step loop: + +### 1. Solve (Collect Evidence) + +Gather the agent's execution artifacts. Ask the user for or locate: +- Run logs, error traces, or experiment outputs +- Pass/fail results per task +- Metric values (accuracy, reward, success rate) +- Any existing session files from previous runs + +If inside Hermes AutoResearch, look at: +- `artifacts/hermes-research-*/` — experiment outputs per round +- Lattice task event history (`lattice show --events`) +- Lattice comments — each round posts KEPT/IMPROVED/DISCARDED + metric +- `ExperimentRunner.history.to_dict()` — full round history in memory + +### 2. Observe (Diagnose) + +Analyze the collected evidence to produce structured observations: + +For each failed or underperforming task, identify: +- **Error category**: code bug, timeout, wrong approach, missing knowledge, + API misuse, hallucinated reference, prompt ambiguity, etc. +- **Root cause**: What specifically went wrong and why +- **Frequency**: Is this a one-off or a recurring pattern across tasks? +- **Severity**: blocking (pipeline crash) / degrading (wrong result) / + cosmetic (formatting issue) + +Write observations as a structured list: + +``` +## Observations (Batch N) + +### OBS-1: [Category] Short description +- Tasks affected: task_001, task_005, task_012 +- Root cause: ... +- Frequency: 3/50 tasks (6%) +- Severity: degrading + +### OBS-2: ... +``` + +### 3. Evolve (Propose Mutations) + +Based on observations, propose one or more of these mutation types: + +**A. Generate a Skill** (for recurring patterns, frequency ≥ 3) + +Write a new `SKILL.md` file that teaches the agent how to handle this +pattern. A good evolved skill: +- Targets a specific failure category, not generic advice +- Contains concrete steps the agent should follow +- Includes a "when to apply" trigger condition +- Is short (under 100 lines) and self-contained + +Example — if the agent keeps failing at API pagination: + +```markdown +--- +name: api-pagination-handler +description: > + Handle paginated API responses correctly. Use when making API calls + that may return partial results, or when results seem truncated. +--- + +When calling any API that supports pagination: + +1. Check response for pagination indicators: `next_page`, `offset`, + `has_more`, `cursor`, or truncated result counts. +2. If paginated, loop until all pages are collected. +3. Concatenate results before processing. +4. Set a max-page safety limit (default: 20) to prevent infinite loops. +5. Log total items collected vs expected count if available. +``` + +**B. Patch the System Prompt** (for prompt ambiguity or missing guidance) + +Write a short addendum to the system prompt that addresses the gap. +Keep patches minimal — one paragraph per issue. Format: + +``` +## Prompt Patch: [Issue] +Append to system prompt: +> When [specific situation], always [specific action] because [reason]. +``` + +**C. Add a Knowledge Entry** (for factual gaps or learned heuristics) + +Record a reusable insight as a knowledge entry: + +```json +{ + "id": "know-001", + "category": "experiment_design", + "insight": "Synthetic benchmarks with <100 samples produce high-variance results. Always use ≥500 samples or report confidence intervals.", + "source": "observation OBS-3 from batch 2", + "confidence": 0.85 +} +``` + +**D. Do Nothing** (if observation is a one-off, severity is cosmetic, +or the fix would be too broad / risky) + +### 4. Gate (Validate) + +Before accepting any mutation, check: + +- **Specificity**: Does it target the observed failure without being so + broad it could cause regressions elsewhere? +- **Testability**: Could you verify this mutation helps by re-running the + failed tasks? +- **Blast radius**: How much of the agent's behavior does this change? + Prefer small, targeted mutations over large rewrites. +- **Consistency**: Does it contradict existing skills or prompt guidance? + +If a mutation fails the gate, either refine it or discard it. +Explain your reasoning to the user. + +### 5. Reload (Apply and Record) + +Present the accepted mutations to the user. For each: +- State what changed and why +- Show the artifact (skill file, prompt patch, knowledge entry) +- Suggest where to place it in the project + +For Hermes AutoResearch projects, recommended locations: + +| Artifact | Location | +|----------|----------| +| Evolved skill | `skills/autoresearch/evolved//SKILL.md` | +| Prompt patch | Edit the inline templates in `agent/research/supervisor.py:_build_task_brief` | +| Knowledge entry | `~/.hermes/evolution/lessons.jsonl` via `EvolutionStore.append_many()` | +| Observation log | `~/.hermes/research-workspace//observations/.md` | + +Keep a running version log so the user can track what evolved and when: + +``` +## Evolution Log +- evo-1 (2026-03-30): Generated `api-pagination-handler` skill from OBS-1 +- evo-2 (2026-03-30): Prompt patch for citation format from OBS-4 +``` + +## Usage with Hermes AutoResearch + +This skill maps to Hermes Karpathy loop steps: + +| Loop Step | Evolution Role | +|-----------|---------------| +| Step 3: DELEGATE | Source of Solve artifacts — delegate_task outputs | +| Step 4: METRIC | Main Observe trigger — parse what went wrong in metric extraction | +| Step 5: KEEP/DISCARD | Natural Gate — KEPT = accept, DISCARDED = evolve | +| EvolutionStore | Lessons persisted via `EvolutionStore.append_many()` | + +When the user says "evolve my research pipeline" or similar: + +1. Ask which run to analyze (or find the latest `artifacts/hermes-research-*/`) +2. Run the Observe step on Lattice round comments + experiment outputs +3. Propose mutations targeting the weakest loop steps +4. Generate skill files in `skills/autoresearch/evolved/` + +## Anti-Patterns + +Do NOT: +- Generate vague, generic skills ("always be careful", "check your work") +- Propose mutations for one-off errors that won't recur +- Rewrite the entire system prompt — patch it surgically +- Generate more than 3 skills per evolution cycle (quality over quantity) +- Mutate tool code unless the user explicitly asks for it + +## Relationship to EvolutionStore + +Hermes uses `EvolutionStore` (`agent/research/evolution.py`) as the lesson persistence layer. +Evolved skills from this process can be placed in `skills/autoresearch/evolved/` +so they are available in future research sessions. The two systems are complementary: + +- **A-Evolve skill**: Deep, targeted mutation from structured observation +- **EvolutionStore lesson**: Broad pattern captured with time-decay weighting (`LessonEntry`) + +Both can coexist. Skills generated here are higher-precision; EvolutionStore +lessons are higher-recall and decay naturally over time (30-day half-life). diff --git a/skills/autoresearch/domain/biology-biopython/SKILL.md b/skills/autoresearch/domain/biology-biopython/SKILL.md new file mode 100644 index 000000000000..d1f1a5732baf --- /dev/null +++ b/skills/autoresearch/domain/biology-biopython/SKILL.md @@ -0,0 +1,65 @@ +--- +name: biology-biopython +description: Bioinformatics with Biopython for sequence manipulation, file parsing, BLAST, and phylogenetics. Use when working with DNA/RNA/protein sequences or biological databases. +metadata: + category: domain + trigger-keywords: "sequence,FASTA,genome,protein,BLAST,phylogenetic,biopython,bioinformatics,gene,DNA,RNA" + applicable-stages: "9,10,12" + priority: "4" + version: "1.0" + author: researchclaw + references: "adapted from K-Dense-AI/claude-scientific-skills" +--- + +## Biopython Bioinformatics Best Practice + +### Sequence Manipulation +1. Create sequences: `from Bio.Seq import Seq; seq = Seq("ATGCGA")` +2. Complement: `seq.complement()`; Reverse complement: `seq.reverse_complement()` +3. Transcription: `seq.transcribe()` (DNA to RNA) +4. Translation: `seq.translate()` (DNA/RNA to protein) +5. GC content: `from Bio.SeqUtils import gc_fraction; gc_fraction(seq)` +6. Molecular weight: `from Bio.SeqUtils import molecular_weight` + +### File Parsing (SeqIO) +1. Read FASTA: `for rec in SeqIO.parse("file.fasta", "fasta"): ...` +2. Read GenBank: `for rec in SeqIO.parse("file.gb", "genbank"): ...` +3. Read single record: `rec = SeqIO.read("file.fasta", "fasta")` +4. Write sequences: `SeqIO.write(records, "output.fasta", "fasta")` +5. Convert formats: `SeqIO.convert("input.gb", "genbank", "output.fasta", "fasta")` +6. Index large files: `idx = SeqIO.index("large.fasta", "fasta")` for random access + +### BLAST Operations +1. Online BLAST: `from Bio.Blast import NCBIWWW; result = NCBIWWW.qblast("blastn", "nt", seq)` +2. Parse results: `from Bio.Blast import NCBIXML; records = NCBIXML.parse(result)` +3. Local BLAST: run via subprocess, parse XML output with NCBIXML +4. Always set `Entrez.email` before any NCBI access +5. Filter results by e-value (typically < 1e-5) and coverage + +### NCBI Database Access (Entrez) +1. Always set email: `Entrez.email = "your@email.com"` +2. Search: `handle = Entrez.esearch(db="pubmed", term="query")` +3. Fetch records: `handle = Entrez.efetch(db="nucleotide", id="ID", rettype="fasta")` +4. Use API key for higher rate limits (10 req/s vs 3 req/s) +5. Respect NCBI rate limits; add delays between batch requests + +### Phylogenetics (Bio.Phylo) +1. Read trees: `from Bio import Phylo; tree = Phylo.read("tree.nwk", "newick")` +2. Draw trees: `Phylo.draw(tree)` or `Phylo.draw_ascii(tree)` +3. Supported formats: newick, nexus, phyloxml +4. Traverse clades: `for clade in tree.find_clades(): ...` +5. Calculate distances: `tree.distance(clade1, clade2)` + +### Structure Analysis (Bio.PDB) +1. Parse PDB: `parser = PDBParser(); structure = parser.get_structure("id", "file.pdb")` +2. Hierarchy: Structure > Model > Chain > Residue > Atom +3. Get atoms: iterate through `structure.get_atoms()` +4. Calculate distances: use atom coordinate vectors +5. For mmCIF files: use `MMCIFParser()` instead of `PDBParser()` + +### Common Pitfalls +1. Always handle `SeqIO.parse` as an iterator — it exhausts after one pass +2. Check sequence alphabet compatibility before operations +3. Large files: use `SeqIO.index()` not `SeqIO.to_dict()` to avoid memory issues +4. Set proper timeout for remote BLAST queries (can take minutes) +5. Validate parsed data — missing annotations are common in public databases diff --git a/skills/autoresearch/domain/chemistry-rdkit/SKILL.md b/skills/autoresearch/domain/chemistry-rdkit/SKILL.md new file mode 100644 index 000000000000..b5f9cd9ea789 --- /dev/null +++ b/skills/autoresearch/domain/chemistry-rdkit/SKILL.md @@ -0,0 +1,59 @@ +--- +name: chemistry-rdkit +description: Computational chemistry with RDKit for molecular analysis, descriptors, fingerprints, and substructure search. Use when working with SMILES, drug discovery, or cheminformatics tasks. +metadata: + category: domain + trigger-keywords: "molecule,SMILES,chemical,drug,rdkit,fingerprint,molecular,compound,reaction,cheminformatics" + applicable-stages: "9,10,12" + priority: "4" + version: "1.0" + author: researchclaw + references: "adapted from K-Dense-AI/claude-scientific-skills" +--- + +## RDKit Cheminformatics Best Practice + +### Molecular I/O +1. Create molecules from SMILES: `mol = Chem.MolFromSmiles('CCO')` +2. Always check for None: `MolFromSmiles` returns None on invalid input +3. Convert to canonical SMILES: `Chem.MolToSmiles(mol)` +4. Read SDF files: `suppl = Chem.SDMolSupplier('file.sdf')` +5. Read SMILES files: `suppl = Chem.SmilesMolSupplier('file.smi')` +6. Write molecules: `writer = Chem.SDWriter('output.sdf')` + +### Molecular Descriptors +1. Molecular weight: `Descriptors.MolWt(mol)` +2. LogP (lipophilicity): `Descriptors.MolLogP(mol)` +3. TPSA (polar surface area): `Descriptors.TPSA(mol)` +4. H-bond donors/acceptors: `Descriptors.NumHDonors(mol)`, `Descriptors.NumHAcceptors(mol)` +5. Rotatable bonds: `Descriptors.NumRotatableBonds(mol)` +6. Lipinski Rule of 5: MW <= 500, LogP <= 5, HBD <= 5, HBA <= 10 + +### Fingerprints and Similarity +1. Morgan (circular) fingerprints: `AllChem.GetMorganFingerprintAsBitVect(mol, radius=2, nBits=2048)` +2. RDKit fingerprints: `Chem.RDKFingerprint(mol)` +3. MACCS keys: `MACCSkeys.GenMACCSKeys(mol)` +4. Tanimoto similarity: `DataStructs.TanimotoSimilarity(fp1, fp2)` +5. Use radius=2 (ECFP4 equivalent) as default for most applications +6. For virtual screening, Tanimoto > 0.7 suggests structural similarity + +### Substructure Search +1. SMARTS patterns: `pattern = Chem.MolFromSmarts('[OH]')` +2. Check match: `mol.HasSubstructMatch(pattern)` +3. Get all matches: `mol.GetSubstructMatches(pattern)` +4. Common SMARTS: `[#6](=O)[OH]` (carboxylic acid), `[NH2]` (primary amine) +5. Filter compound libraries by functional group presence + +### Property Calculation Patterns +1. Batch processing: iterate over SDMolSupplier, skip None entries +2. Use `Chem.Descriptors.descList` for all available descriptors +3. For ADMET filtering, calculate Lipinski, Veber, and PAINS filters +4. Generate 3D coordinates: `AllChem.EmbedMolecule(mol, AllChem.ETKDG())` +5. Minimize energy: `AllChem.MMFFOptimizeMolecule(mol)` + +### Common Pitfalls +1. Always sanitize molecules (default behavior) — disable only when needed +2. Add hydrogens explicitly for 3D work: `Chem.AddHs(mol)` +3. Handle stereochemistry: use `Chem.AssignStereochemistry(mol)` +4. Large SDF files: use `ForwardSDMolSupplier` for memory efficiency +5. Kekulization errors usually indicate invalid SMILES input diff --git a/skills/autoresearch/domain/cv-classification/SKILL.md b/skills/autoresearch/domain/cv-classification/SKILL.md new file mode 100644 index 000000000000..1622ba697cd7 --- /dev/null +++ b/skills/autoresearch/domain/cv-classification/SKILL.md @@ -0,0 +1,30 @@ +--- +name: cv-classification +description: Best practices for image classification tasks. Use when working on CIFAR, ImageNet, or other classification benchmarks. +metadata: + category: domain + trigger-keywords: "classification,image,cifar,imagenet,resnet,vision,cnn,vit" + applicable-stages: "9,10" + priority: "3" + version: "1.0" + author: researchclaw + references: "He et al., Deep Residual Learning, CVPR 2016; Dosovitskiy et al., An Image is Worth 16x16 Words, ICLR 2021" +--- + +## Image Classification Best Practice +Architecture selection: +- Small scale (CIFAR-10/100): ResNet-18/34, WideResNet, Simple ViT +- Medium scale: ResNet-50, EfficientNet-B0/B1, DeiT-Small +- Large scale: ViT-B/16, ConvNeXt, Swin Transformer + +Training recipe: +- Optimizer: AdamW (lr=1e-3 to 3e-4) or SGD (lr=0.1 with cosine decay) +- Weight decay: 0.01-0.1 for AdamW, 5e-4 for SGD +- Data augmentation: RandomCrop, RandomHorizontalFlip, Cutout/CutMix +- Warmup: 5-10 epochs linear warmup for transformers +- Batch size: 128-256 for CNNs, 512-1024 for ViTs (if memory allows) + +Standard benchmarks: +- CIFAR-10: ~96% (ResNet-18), ~97% (WideResNet) +- CIFAR-100: ~80% (ResNet-18), ~84% (WideResNet) +- ImageNet: ~76% (ResNet-50), ~81% (ViT-B/16) diff --git a/skills/autoresearch/domain/cv-detection/SKILL.md b/skills/autoresearch/domain/cv-detection/SKILL.md new file mode 100644 index 000000000000..653df211ad47 --- /dev/null +++ b/skills/autoresearch/domain/cv-detection/SKILL.md @@ -0,0 +1,29 @@ +--- +name: cv-detection +description: Best practices for object detection tasks. Use when working on COCO, VOC, or detection architectures like YOLO and DETR. +metadata: + category: domain + trigger-keywords: "detection,object,bbox,yolo,coco,anchor,faster rcnn" + applicable-stages: "9,10" + priority: "5" + version: "1.0" + author: researchclaw + references: "Ren et al., Faster R-CNN, NeurIPS 2015; Carion et al., End-to-End Object Detection with Transformers, ECCV 2020" +--- + +## Object Detection Best Practice +Architecture families: +- One-stage: YOLO (v5/v8), SSD, RetinaNet, FCOS +- Two-stage: Faster R-CNN, Cascade R-CNN +- Transformer: DETR, DINO, RT-DETR + +Training recipe: +- Use pre-trained backbone (ImageNet) +- Multi-scale training and testing +- IoU threshold: 0.5 for mAP50, 0.5:0.95 for mAP +- Use FPN for multi-scale feature extraction +- Focal loss for class imbalance in one-stage detectors + +Standard benchmarks: +- COCO val2017: ~37 mAP (Faster R-CNN R50), ~51 mAP (DINO Swin-L) +- Pascal VOC: ~80 mAP50 (Faster R-CNN) diff --git a/skills/autoresearch/domain/nlp-alignment/SKILL.md b/skills/autoresearch/domain/nlp-alignment/SKILL.md new file mode 100644 index 000000000000..33a2ba557d46 --- /dev/null +++ b/skills/autoresearch/domain/nlp-alignment/SKILL.md @@ -0,0 +1,31 @@ +--- +name: nlp-alignment +description: Best practices for LLM alignment techniques including RLHF, DPO, and instruction tuning. Use when working on alignment or safety. +metadata: + category: domain + trigger-keywords: "alignment,rlhf,dpo,reward model,preference,instruction tuning,safety" + applicable-stages: "9,10" + priority: "4" + version: "1.0" + author: researchclaw + references: "Ouyang et al., Training language models to follow instructions, NeurIPS 2022; Rafailov et al., DPO, NeurIPS 2023" +--- + +## LLM Alignment Best Practice +Methods: +- RLHF: Train reward model → PPO fine-tuning (complex but powerful) +- DPO: Direct preference optimization (simpler, no reward model needed) +- GRPO: Group relative policy optimization +- SFT: Supervised fine-tuning as alignment baseline + +Training recipe: +- Start with SFT on high-quality instruction data +- DPO: lr=5e-7, beta=0.1, batch_size=64 +- PPO: lr=1e-6, clip=0.2, KL coeff=0.02 +- Use reference model for KL penalty +- Evaluate on safety benchmarks (TruthfulQA, BBQ, etc.) + +Common pitfalls: +- Reward hacking: model finds shortcuts to high reward +- Mode collapse: model generates repetitive outputs +- Catastrophic forgetting: loses general capabilities diff --git a/skills/autoresearch/domain/nlp-pretraining/SKILL.md b/skills/autoresearch/domain/nlp-pretraining/SKILL.md new file mode 100644 index 000000000000..f5db9cd9229a --- /dev/null +++ b/skills/autoresearch/domain/nlp-pretraining/SKILL.md @@ -0,0 +1,31 @@ +--- +name: nlp-pretraining +description: Best practices for language model pretraining and fine-tuning. Use when generating or reviewing NLP training code. +metadata: + category: domain + trigger-keywords: "language model,pretraining,fine-tuning,bert,gpt,llm,transformer,nlp,text" + applicable-stages: "9,10" + priority: "3" + version: "1.0" + author: researchclaw + references: "Devlin et al., BERT, NAACL 2019; Hu et al., LoRA, ICLR 2022" +--- + +## NLP Pretraining/Fine-tuning Best Practice +Fine-tuning recipe: +- Use pre-trained checkpoints (HuggingFace hub) +- AdamW optimizer, lr=2e-5 to 5e-5 +- Linear warmup (6% of total steps) + linear decay +- Batch size: 16-32 (use gradient accumulation for larger effective batch) +- 3-5 epochs for classification, 1-2 for generation +- Weight decay: 0.01 + +Parameter-efficient methods: +- LoRA: r=8-64, alpha=16-128, apply to q/v projections +- Prefix tuning: 10-20 prefix tokens +- Adapters: bottleneck dimension 64-256 + +Evaluation: +- Classification: accuracy, F1 (macro for imbalanced) +- Generation: perplexity, BLEU/ROUGE, human evaluation +- Use multiple seeds and report mean +/- std diff --git a/skills/autoresearch/domain/rl-policy-optimization/SKILL.md b/skills/autoresearch/domain/rl-policy-optimization/SKILL.md new file mode 100644 index 000000000000..2ee972f9e600 --- /dev/null +++ b/skills/autoresearch/domain/rl-policy-optimization/SKILL.md @@ -0,0 +1,37 @@ +--- +name: rl-policy-optimization +description: Best practices for reinforcement learning policy optimization. Use when working on RL agents, PPO, SAC, or reward design. +metadata: + category: domain + trigger-keywords: "reinforcement learning,rl,policy,reward,agent,environment,ppo,sac" + applicable-stages: "9,10" + priority: "3" + version: "1.0" + author: researchclaw + references: "Schulman et al., Proximal Policy Optimization, 2017; Haarnoja et al., Soft Actor-Critic, ICML 2018" +--- + +## RL Policy Optimization Best Practice +Algorithm selection: +- Discrete actions: PPO, DQN, A2C +- Continuous actions: SAC, TD3, PPO +- Multi-agent: MAPPO, QMIX +- Offline: CQL, IQL, Decision Transformer + +Training recipe: +- PPO: clip=0.2, lr=3e-4, gamma=0.99, GAE lambda=0.95 +- SAC: lr=3e-4, tau=0.005, auto-tune alpha +- Use vectorized environments (e.g., gymnasium.vector) +- Normalize observations and rewards +- Log episode return, episode length, value loss, policy entropy + +Evaluation: +- Report mean +/- std over 10+ evaluation episodes +- Use deterministic policy for evaluation +- Compare against random policy and simple baselines +- Report sample efficiency (return vs. env steps) + +Common pitfalls: +- Reward shaping can introduce bias +- Seed sensitivity is HIGH — use 5+ seeds +- Hyperparameter sensitivity — do a small sweep diff --git a/skills/autoresearch/hypothesis-formulation/SKILL.md b/skills/autoresearch/hypothesis-formulation/SKILL.md new file mode 100644 index 000000000000..9be439fe0b01 --- /dev/null +++ b/skills/autoresearch/hypothesis-formulation/SKILL.md @@ -0,0 +1,48 @@ +--- +name: hypothesis-formulation +description: Structured scientific hypothesis generation from observations. Use when formulating testable hypotheses, competing explanations, or experimental predictions. +metadata: + category: experiment + trigger-keywords: "hypothesis,prediction,mechanism,falsifiable,null,alternative,testable" + applicable-stages: "7,8,9" + priority: "3" + version: "1.0" + author: hermes +--- + +## Hypothesis Formulation Best Practice + +### Structured Hypothesis Development +1. Start with a clear observation or pattern that requires explanation +2. Review existing literature for known mechanisms and prior explanations +3. Identify what is already established vs. what remains uncertain +4. Formulate the hypothesis as a specific, testable statement +5. Ensure the hypothesis is falsifiable — define what outcome would refute it + +### Hypothesis Format +1. **Null hypothesis (H0)**: There is no effect or no difference +2. **Alternative hypothesis (H1)**: There is a specific, directional effect +3. State both explicitly; design experiments to reject H0 +4. Use "If... then... because..." structure for mechanistic hypotheses: + - If [independent variable is manipulated], then [predicted outcome], because [proposed mechanism] + +### Generating Competing Hypotheses +1. Propose at least 2-3 plausible explanations for the same observation +2. For each, identify unique predictions that distinguish it from alternatives +3. Rank hypotheses by parsimony, consistency with prior evidence, and testability +4. Design experiments that can discriminate between competing hypotheses +5. Consider confounding variables that could produce the same observation + +### Testable Predictions +1. Derive specific, measurable predictions from each hypothesis +2. Define expected effect direction AND approximate magnitude +3. Specify what experimental conditions would confirm vs. refute the prediction +4. Identify potential confounds and plan controls to address them +5. Ensure predictions are achievable with available methods and resources + +### Aligning with Experimental Design +1. Map each hypothesis to a concrete experimental condition or comparison +2. Ensure sample size is adequate to detect the predicted effect (power analysis) +3. Pre-register hypotheses and analysis plans when possible +4. Distinguish confirmatory (hypothesis-testing) from exploratory analyses +5. Plan for both positive and null results — what will you conclude in each case? diff --git a/skills/autoresearch/karpathy-guidelines/SKILL.md b/skills/autoresearch/karpathy-guidelines/SKILL.md new file mode 100644 index 000000000000..962e81c385ea --- /dev/null +++ b/skills/autoresearch/karpathy-guidelines/SKILL.md @@ -0,0 +1,115 @@ +--- +name: karpathy-guidelines +description: > + Behavioral guidelines to reduce common LLM coding mistakes in research + experiments. Use when writing, reviewing, or iterating on experiment code + to avoid overcomplication, make surgical changes, surface assumptions, and + define verifiable metric-based success criteria. Auto-applied inside the + Hermes AutoResearch loop. Triggers on: "simplify", "refactor experiment", + "why isn't the metric improving", "code review", or any iteration step. +metadata: + author: hermes + source: https://x.com/karpathy/status/2015883857489522876 + category: experiment + priority: "1" +--- + +# Karpathy Research Guidelines + +Behavioral guidelines for LLM-driven experiment code, derived from Andrej +Karpathy's observations on common LLM coding pitfalls. Applied to every +iteration of the Hermes AutoResearch Karpathy loop. + +**Tradeoff:** These guidelines bias toward caution over speed. For trivial +one-shot experiments, use judgment. + +--- + +## 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before writing or modifying any experiment code: + +- State your **assumptions** explicitly: what do you expect `main.py` to do? + What is the binding bottleneck causing the metric to be where it is? +- If multiple interpretations exist, present them — don't pick silently. +- If a simpler approach exists, say so. Prefer it. +- If something is unclear, name what is confusing in your NOTES output. + Do NOT guess silently and move on. + +In the research loop, this means: before touching `main.py`, write a brief +mental model of why the last metric was what it was. If you can't explain +it, don't change it yet. + +## 2. Simplicity First + +**Minimum code that moves the metric. Nothing speculative.** + +- No features beyond what the hypothesis requires. +- No abstractions for single-use experiment code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, write 50. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" +If yes, simplify. In experiment code, complexity is an enemy — it hides +the signal you're trying to measure. + +## 3. Surgical Changes + +**Touch only what your hypothesis requires. Clean up only your own mess.** + +When iterating on experiment code: + +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code or issues, mention them in NOTES — + don't fix them silently. + +When your changes create orphans: + +- Remove imports/variables that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +**The test:** Every changed line should trace directly to the hypothesis +that motivated this iteration. + +## 4. Goal-Driven Execution + +**Define success in metric terms. Loop until verified.** + +Transform vague improvement goals into verifiable metric movements: + +- "Make it faster" → "`accuracy` should increase from 0.72 toward 0.80" +- "Fix convergence" → "`loss` should decrease by at least 10% vs. baseline" +- "Try a different optimizer" → "Adam should yield higher `accuracy` than SGD at iter N" + +For each iteration, state a brief plan before coding: + +``` +Hypothesis: [what I think will improve the metric and why] +Change: [the ONE thing I will modify] +Verify: [metric moves from X toward Y] +``` + +Strong success criteria let the loop self-correct. Weak criteria +("make it better") waste iterations and lose signal. + +--- + +## Application to Hermes AutoResearch Loop + +| Loop Step | Karpathy Principle | +|-----------|-------------------| +| Step 0: Think | Principle 1 — state assumptions + bottleneck before any code | +| Step 1: Run | Principle 3 — touch only what the hypothesis requires | +| Step 2: Measure | Principle 4 — compare against your stated success criterion | +| Improve iteration | All 4 — think → minimal change → verify → repeat | +| Early stop (3 non-improving) | Principle 1 — re-examine assumptions before giving up | + +If 3 consecutive iterations fail to improve the metric, stop and ask: +"Is my mental model of the bottleneck correct?" before iterating further. +This is the Karpathy diagnostic: the loop failing usually means the +hypothesis was wrong, not that you need more iterations. diff --git a/skills/autoresearch/literature-search/SKILL.md b/skills/autoresearch/literature-search/SKILL.md new file mode 100644 index 000000000000..cd4faeeca8b7 --- /dev/null +++ b/skills/autoresearch/literature-search/SKILL.md @@ -0,0 +1,56 @@ +--- +name: literature-search +description: Systematic literature review methodology including search strategy, screening, and synthesis. Use when conducting literature reviews or writing background sections. +metadata: + category: experiment + trigger-keywords: "literature,review,systematic,PRISMA,search,database,PubMed,arXiv,citation" + applicable-stages: "3,4,5,6" + priority: "2" + version: "1.0" + author: hermes +--- + +## Literature Search Best Practice + +### Search Strategy Design +1. Define research question using PICO framework (Population, Intervention, Comparison, Outcome) +2. Identify 2-4 core concepts from the research question +3. List synonyms, abbreviations, and related terms for each concept +4. Combine terms with Boolean operators: AND (between concepts), OR (within synonyms) +5. Select at least 3 complementary databases relevant to the domain: + - Biomedical: PubMed, Scopus, Web of Science + - Computer science: arXiv, Semantic Scholar, DBLP, ACL Anthology + - Interdisciplinary: Google Scholar, OpenAlex +6. Document exact search strings for reproducibility + +### Inclusion and Exclusion Criteria +1. Define date range (e.g., last 5-10 years for rapidly evolving fields) +2. Specify language restrictions (typically English) +3. Specify publication types (peer-reviewed, preprints, conference papers) +4. Define study design requirements (RCTs, observational, computational) +5. Set domain-specific filters (species, methodology, sample size) +6. Document all criteria BEFORE screening begins + +### PRISMA Methodology +1. Record total hits from each database before deduplication +2. Remove duplicates and record count +3. Screen titles and abstracts against inclusion criteria (record excluded count) +4. Full-text review of remaining papers (record excluded with reasons) +5. Report final included studies with PRISMA flow diagram +6. For scoping reviews, use PRISMA-ScR extension + +### Screening and Quality Assessment +1. Use two-pass screening: title/abstract first, then full text +2. Apply quality assessment tools appropriate to study type: + - RCTs: Cochrane Risk of Bias tool + - Observational: Newcastle-Ottawa Scale + - ML papers: check reproducibility, dataset validity, statistical rigor +3. Extract data systematically using a predefined extraction form + +### Synthesis Approaches +1. **Narrative synthesis**: Organize findings thematically, identify patterns and contradictions +2. **Meta-analysis**: Pool quantitative results when studies are sufficiently homogeneous +3. **Gap analysis**: Explicitly identify what is NOT covered in the literature +4. Summarize key findings per theme with supporting citation counts +5. Highlight conflicting results and possible explanations +6. End with clear statement of research gaps that motivate your study diff --git a/skills/autoresearch/scientific-visualization/SKILL.md b/skills/autoresearch/scientific-visualization/SKILL.md new file mode 100644 index 000000000000..c2911ef3943a --- /dev/null +++ b/skills/autoresearch/scientific-visualization/SKILL.md @@ -0,0 +1,56 @@ +--- +name: scientific-visualization +description: Publication-ready scientific figure design with matplotlib and seaborn. Use when creating journal submission figures with proper formatting, accessibility, and statistical annotations. +metadata: + category: writing + trigger-keywords: "figure,plot,chart,visualization,matplotlib,seaborn,colorblind,publication" + applicable-stages: "14,17,22" + priority: "3" + version: "1.0" + author: researchclaw + references: "adapted from K-Dense-AI/claude-scientific-skills" +--- + +## Scientific Visualization Best Practice + +### Figure Design Principles +1. Every figure must have a clear, self-contained message +2. Minimize chartjunk: remove gridlines, background shading, and 3D effects +3. Use direct labeling instead of legends when possible +4. Remove top and right spines for cleaner appearance +5. Ensure all text is readable at final print size (minimum 6pt font) + +### Journal Figure Sizing +1. **Single column**: 3.3-3.5 inches (85-89 mm) wide +2. **1.5 column**: 4.5-5.5 inches (114-140 mm) wide +3. **Double column / full width**: 6.5-7.1 inches (165-180 mm) wide +4. Resolution: 300 DPI minimum for raster; prefer vector formats (PDF, EPS, SVG) +5. Check target journal author guidelines for exact specifications + +### Colorblind-Safe Design +1. Use colorblind-friendly palettes: seaborn "colorblind", Okabe-Ito, viridis, cividis +2. NEVER rely on color alone — combine with shape, pattern, or line style +3. Avoid red-green combinations; prefer blue-orange or blue-yellow contrasts +4. Test figures with a colorblind simulator before submission +5. Ensure figures work in grayscale for print journals + +### Multi-Panel Layouts +1. Label panels with uppercase letters: (A), (B), (C) in bold, top-left corner +2. Use consistent axis scales across panels when comparing related data +3. Share axes where appropriate to reduce redundancy +4. Maintain consistent font sizes and line widths across all panels +5. Use `plt.subplots()` with `constrained_layout=True` for automatic spacing + +### Statistical Annotations on Figures +1. Show individual data points alongside summary statistics (box + strip plots) +2. Always include error bars; specify type in caption (SEM, SD, 95% CI) +3. Use significance brackets with stars: * p<.05, ** p<.01, *** p<.001 +4. Annotate effect sizes or key statistics directly on the figure when helpful +5. Never use bar charts for small-n data — use dot plots or box plots instead + +### Export and Quality Checklist +1. Save in vector format (PDF/SVG) for line art; TIFF/PNG for photographs +2. Embed fonts or convert text to outlines for cross-platform consistency +3. Verify axis labels include units in parentheses: "Time (s)", "Force (N)" +4. Ensure figure caption fully explains all symbols, abbreviations, and panels +5. Check that color-coded elements match between figure and caption diff --git a/skills/autoresearch/scientific-writing/SKILL.md b/skills/autoresearch/scientific-writing/SKILL.md new file mode 100644 index 000000000000..6a4d00c81176 --- /dev/null +++ b/skills/autoresearch/scientific-writing/SKILL.md @@ -0,0 +1,56 @@ +--- +name: scientific-writing +description: Academic manuscript writing with IMRAD structure, citation formatting, and reporting guidelines. Use when drafting or revising research papers. +metadata: + category: writing + trigger-keywords: "paper,manuscript,writing,IMRAD,citation,abstract,introduction,methods,results,discussion" + applicable-stages: "16,17,19" + priority: "2" + version: "1.0" + author: researchclaw + references: "adapted from K-Dense-AI/claude-scientific-skills" +--- + +## Scientific Writing Best Practice + +### IMRAD Structure +1. **Abstract**: State objective, methods, key results, and conclusion in 150-300 words +2. **Introduction**: Move from broad context to specific gap to your contribution (funnel structure) +3. **Methods**: Sufficient detail for replication; use past tense, passive voice +4. **Results**: Present findings without interpretation; pair text with figures/tables +5. **Discussion**: Interpret results, compare with literature, acknowledge limitations, state implications + +### Paragraph-Level Guidance +1. Each paragraph should convey ONE main idea +2. Open with a topic sentence; close with a transition to the next paragraph +3. Write in full flowing prose — never submit bullet points as final manuscript text +4. Use active voice for clarity: "We measured..." not "Measurements were taken..." +5. Vary sentence length; aim for average 15-25 words per sentence + +### Citation Best Practices +1. Cite primary sources over reviews when making specific claims +2. Use citation styles consistently (APA, Vancouver, IEEE) per target journal +3. Every factual claim needs a citation unless it is common knowledge in the field +4. Avoid citation strings of 5+ references — select the most relevant 2-3 +5. Self-citations should be limited to genuinely relevant prior work + +### Common Writing Pitfalls +1. Avoid hedge-stacking: "It might possibly suggest..." — choose one hedge +2. Do not start sentences with "It is well known that" — cite or remove +3. Distinguish "significant" (statistical) from "substantial" (practical) +4. Ensure figures/tables are referenced in text BEFORE they appear +5. Keep abbreviations to a minimum; define each on first use + +### Reporting Guidelines +1. Randomized trials: follow CONSORT checklist +2. Observational studies: follow STROBE checklist +3. Systematic reviews: follow PRISMA checklist +4. Diagnostic accuracy: follow STARD checklist +5. Always check target journal's author guidelines for specific requirements + +### Revision Checklist +1. Verify all figures/tables are cited in text and numbered sequentially +2. Confirm reference list matches in-text citations exactly +3. Check that abstract accurately reflects the final manuscript content +4. Ensure methods section enables independent replication +5. Read aloud to catch awkward phrasing and run-on sentences diff --git a/skills/autoresearch/statistical-reporting/SKILL.md b/skills/autoresearch/statistical-reporting/SKILL.md new file mode 100644 index 000000000000..5a3fa15d7296 --- /dev/null +++ b/skills/autoresearch/statistical-reporting/SKILL.md @@ -0,0 +1,58 @@ +--- +name: statistical-reporting +description: Statistical test selection, assumption checking, and APA-formatted reporting. Use when analyzing experimental results or writing results sections. +metadata: + category: writing + trigger-keywords: "statistic,hypothesis test,p-value,regression,ANOVA,t-test,effect size,confidence interval" + applicable-stages: "14,17" + priority: "3" + version: "1.0" + author: researchclaw + references: "adapted from K-Dense-AI/claude-scientific-skills" +--- + +## Statistical Reporting Best Practice + +### Test Selection Quick Reference +1. **Comparing two groups (independent, normal)**: Independent t-test +2. **Comparing two groups (independent, non-normal)**: Mann-Whitney U test +3. **Comparing two groups (paired, normal)**: Paired t-test +4. **Comparing two groups (paired, non-normal)**: Wilcoxon signed-rank test +5. **Comparing 3+ groups (independent, normal)**: One-way ANOVA + post-hoc +6. **Comparing 3+ groups (non-normal)**: Kruskal-Wallis test +7. **Relationship between continuous variables**: Pearson or Spearman correlation +8. **Categorical outcomes**: Chi-square or Fisher's exact test +9. **Predicting continuous outcome**: Linear regression +10. **Predicting binary outcome**: Logistic regression + +### Assumption Checking +1. **Normality**: Shapiro-Wilk test (n < 50) or visual Q-Q plots +2. **Homogeneity of variance**: Levene's test before t-tests and ANOVA +3. **Independence**: Verify study design ensures independent observations +4. **Linearity**: Scatter plots and residual plots for regression +5. **Multicollinearity**: VIF < 5 for multiple regression predictors +6. When assumptions are violated, use non-parametric alternatives or robust methods + +### APA Reporting Format +1. **t-test**: t(df) = X.XX, p = .XXX, d = X.XX +2. **ANOVA**: F(df_between, df_within) = X.XX, p = .XXX, eta-squared = .XX +3. **Correlation**: r(df) = .XX, p = .XXX [95% CI: .XX, .XX] +4. **Chi-square**: chi-square(df, N = XXX) = X.XX, p = .XXX +5. **Regression**: beta = X.XX, SE = X.XX, t = X.XX, p = .XXX +6. Always report exact p-values (not "p < .05") unless p < .001 +7. Use leading zero for values that can exceed 1 (e.g., t = 0.50) but not for those bounded by 1 (e.g., p = .032, r = .45) + +### Effect Sizes +1. ALWAYS report effect sizes alongside p-values +2. Cohen's d for group comparisons: small = 0.2, medium = 0.5, large = 0.8 +3. Eta-squared for ANOVA: small = .01, medium = .06, large = .14 +4. R-squared for regression: report adjusted R-squared for multiple predictors +5. Odds ratios for logistic regression with 95% confidence intervals +6. Distinguish statistical significance from practical significance + +### Common Mistakes to Avoid +1. Never say "the results were not significant, therefore there is no effect" +2. Do not confuse correlation with causation in observational data +3. Apply multiple comparison corrections (Bonferroni, FDR) when running many tests +4. Report confidence intervals, not just point estimates +5. State whether tests are one-tailed or two-tailed and justify the choice diff --git a/tests/agent/test_factory.py b/tests/agent/test_factory.py new file mode 100644 index 000000000000..ce48be4f1e1e --- /dev/null +++ b/tests/agent/test_factory.py @@ -0,0 +1,74 @@ +"""Tests for agent.factory — centralized AIAgent construction for +detached entrypoints (HRM-57). + +Run with: + pytest tests/agent/test_factory.py -q --override-ini="addopts=" +""" +from __future__ import annotations + +import os +from unittest.mock import patch, MagicMock + +from agent.factory import build_agent_for_research_job + + +# A minimal spec the factory should accept. +_SPEC = { + "job_id": "test-001", + "model": "kimi-k2.6", + "provider": "kimi-coding", + "base_url": "https://api.kimi.com/coding/v1", + "api_key": "", + "toolsets": ["research", "terminal", "file"], +} + + +class TestBuildAgentForResearchJob: + def test_default_inherits_profile_context(self): + """Without skip_* in spec, defaults flip to load profile context — + consistent with HRM-58: research workers want the curated profile.""" + captured: dict = {} + with patch("run_agent.AIAgent") as MockAgent: + MockAgent.side_effect = lambda *a, **kw: captured.update(kw) or MagicMock() + build_agent_for_research_job(_SPEC) + assert captured["skip_context_files"] is False + assert captured["skip_memory"] is False + assert captured["model"] == "kimi-k2.6" + assert captured["session_id"] == "research-job:test-001" + + def test_spec_can_opt_out_of_profile_context(self): + """spec.skip_context_files=True still wins for callers that want + a blank-slate detached run (e.g. provider benchmarking).""" + captured: dict = {} + spec = {**_SPEC, "skip_context_files": True, "skip_memory": True} + with patch("run_agent.AIAgent") as MockAgent: + MockAgent.side_effect = lambda *a, **kw: captured.update(kw) or MagicMock() + build_agent_for_research_job(spec) + assert captured["skip_context_files"] is True + assert captured["skip_memory"] is True + + def test_runtime_invariants_passed_as_kwargs(self): + """After HRM-57 full, the runtime invariants are constructor kwargs. + The factory must hand them to AIAgent rather than patch post-init.""" + captured: dict = {} + with patch("run_agent.AIAgent") as MockAgent: + MockAgent.side_effect = lambda *a, **kw: captured.update(kw) or MagicMock(tool_progress_callback=None) + build_agent_for_research_job(_SPEC) + + assert captured["delegate_depth"] == 0 + assert "terminal_cwd" in captured + assert "cwd" in captured + assert captured["subdirectory_hints"] is None + + def test_progress_callback_is_no_op_when_none(self): + """The factory still sets a no-op tool_progress_callback when AIAgent + leaves it as None — callers can dispatch without nil-checking.""" + with patch("run_agent.AIAgent") as MockAgent: + mock_agent = MagicMock() + mock_agent.tool_progress_callback = None + MockAgent.return_value = mock_agent + agent = build_agent_for_research_job(_SPEC) + + assert agent.tool_progress_callback is not None + # Calling it should not raise + agent.tool_progress_callback("event", "name", "preview") diff --git a/tests/agent/test_research_supervisor.py b/tests/agent/test_research_supervisor.py new file mode 100644 index 000000000000..fda9d641fb49 --- /dev/null +++ b/tests/agent/test_research_supervisor.py @@ -0,0 +1,647 @@ +"""Integration tests for ResearchSupervisor — Karpathy inner loop. + +Run with: + pytest tests/agent/test_research_supervisor.py -m integration --override-ini="addopts=" + +The 'integration' mark is required because these tests write to a real tmpdir, +run the full ExperimentRunner loop, and verify end-to-end metric parsing. +Tests tagged 'unit' run without the mark. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from agent.research.runner import ( + DelegateSandboxResult, + ExperimentHistory, + ExperimentRunner, + HermesExperimentConfig, +) +from agent.research.metrics import UniversalMetricParser +from agent.research.supervisor import ( + ResearchSupervisor, + TaskSpec, + _build_task_brief, + _extract_iteration, +) + + +# --------------------------------------------------------------------------- +# Unit tests (no integration mark needed) +# --------------------------------------------------------------------------- + +class TestBuildTaskBrief: + def _code_spec(self, **kwargs) -> TaskSpec: + defaults = dict( + topic="optimizer comparison", + deliverable="Python comparison of Adam vs SGD on MNIST", + metric_key="accuracy", + metric_direction="maximize", + task_type="code", + hypothesis="Adam converges faster than SGD", + ) + defaults.update(kwargs) + return TaskSpec(**defaults) + + def test_contains_topic_and_metric(self): + spec = self._code_spec() + md = _build_task_brief( + spec, + iteration=1, + round_dir="/tmp/round-001-iter1", + time_budget_sec=120, + ) + assert "optimizer comparison" in md + assert "accuracy" in md + assert "higher" in md # metric_direction="maximize" renders as "higher" + assert "120" in md + assert "METRIC: accuracy=" in md + + def test_contains_time_guard_instructions(self): + spec = TaskSpec( + topic="t", deliverable="d", metric_key="loss", + metric_direction="minimize", task_type="code", + ) + md = _build_task_brief(spec, iteration=0, round_dir="/tmp/rd", time_budget_sec=60) + assert "TIME_ESTIMATE" in md + assert "80%" in md + + def test_iteration_zero_is_baseline(self): + spec = self._code_spec() + md = _build_task_brief(spec, iteration=0, round_dir="/tmp/rd", time_budget_sec=300) + assert "Establish a baseline" in md + + def test_iteration_positive_is_improve(self): + spec = self._code_spec() + md = _build_task_brief(spec, iteration=2, round_dir="/tmp/rd", time_budget_sec=300) + assert "Improve" in md + + def test_search_task_brief(self): + spec = TaskSpec( + topic="Find attention mechanism papers", + deliverable="Ranked list of papers", + metric_key="relevance_score", + task_type="search", + ) + md = _build_task_brief(spec, iteration=0, round_dir="/tmp/rd", time_budget_sec=120) + assert "Search" in md + assert "relevance_score" in md + assert "attempt.md" in md + + def test_research_task_brief(self): + spec = TaskSpec( + topic="State of diffusion models", + deliverable="Technical synthesis", + metric_key="completeness_score", + task_type="research", + ) + md = _build_task_brief(spec, iteration=0, round_dir="/tmp/rd", time_budget_sec=300) + assert "Research" in md + assert "completeness_score" in md + + def test_generic_task_brief(self): + spec = TaskSpec( + topic="Optimize search latency", + deliverable="Modified implementation", + metric_key="latency_ms", + metric_direction="minimize", + task_type="generic", + ) + md = _build_task_brief(spec, iteration=0, round_dir="/tmp/rd", time_budget_sec=300) + assert "latency_ms" in md + + +class TestExtractIteration: + def test_round_dir_with_iter(self): + assert _extract_iteration("/tmp/round-abc-iter3") == 3 + assert _extract_iteration("/tmp/round-xyz-iter0") == 0 + assert _extract_iteration("/tmp/round-foo-iter12") == 12 + + def test_malformed_returns_zero(self): + assert _extract_iteration("/tmp/no-iter-here") == 0 + assert _extract_iteration("") == 0 + + +# --------------------------------------------------------------------------- +# Integration tests — full loop with mocked delegate_task +# --------------------------------------------------------------------------- + +pytestmark_integration = pytest.mark.integration + + +def _make_delegate_result(metric_value: float, metric_key: str = "accuracy") -> str: + """Build a fake delegate_task JSON result with a metric in stdout.""" + summary = ( + f"Experiment complete.\n" + f"METRIC: {metric_key}={metric_value} STATUS: improved NOTES: mock result\n" + f"All done." + ) + return json.dumps({ + "results": [ + { + "task_index": 0, + "status": "completed", + "summary": summary, + "api_calls": 5, + "duration_seconds": 1.2, + "exit_reason": "completed", + "tokens": {"input": 100, "output": 50}, + "tool_trace": [], + } + ], + "total_duration_seconds": 1.2, + }) + + +def _make_failed_delegate_result(error: str = "Worker timed out") -> str: + return json.dumps({ + "results": [ + { + "task_index": 0, + "status": "failed", + "summary": "", + "error": error, + "api_calls": 1, + "duration_seconds": 5.0, + "exit_reason": "max_iterations", + "tokens": {"input": 20, "output": 0}, + "tool_trace": [], + } + ], + "total_duration_seconds": 5.0, + }) + + +@pytest.fixture() +def tmp_workspace(tmp_path: Path) -> Path: + return tmp_path / "research-workspace" + + +@pytest.fixture() +def mock_parent_agent() -> MagicMock: + agent = MagicMock() + agent.model = "claude-sonnet-4-6" + agent.base_url = "https://api.anthropic.com" + agent.api_key = "test-key" + agent.provider = "anthropic" + agent.api_mode = "anthropic_messages" + agent.providers_allowed = None + agent.providers_ignored = None + agent.providers_order = None + agent.provider_sort = None + agent.enabled_toolsets = ["terminal", "file"] + agent._delegate_depth = 0 + agent._active_children = [] + agent._active_children_lock = None + return agent + + +@pytest.fixture() +def code_spec() -> TaskSpec: + return TaskSpec( + topic="Optimizer comparison on MNIST", + deliverable="Python script comparing Adam vs SGD with accuracy metric", + metric_key="accuracy", + metric_direction="maximize", + task_type="code", + hypothesis="Adam converges faster than SGD", + ) + + +@pytest.mark.integration +class TestResearchSupervisorBaseline: + """Full loop with a mocked delegate_task — no real subagent spawned.""" + + def test_baseline_only_no_llm(self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec): + """Supervisor runs baseline experiment, returns history with 1 result.""" + metric_value = 0.85 + + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(metric_value)): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + code_spec, + initial_attempt="print('accuracy: 0.85')", + run_id="test-baseline-001", + max_iterations=3, + time_budget_sec=60, + llm=None, # baseline only + ) + + assert len(history.results) == 1 + assert history.baseline_metric == pytest.approx(metric_value, abs=0.001) + assert history.results[0].iteration == 0 + assert history.results[0].primary_metric == pytest.approx(metric_value, abs=0.001) + assert history.results[0].kept is True # first result always kept + + def test_task_brief_written_to_round_dir(self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec): + """Supervisor must write task_brief.md and attempt.py before calling delegate_task.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.75)): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + code_spec, + initial_attempt="# baseline code\nprint('accuracy: 0.75')", + run_id="test-files-001", + llm=None, + ) + + run_dir = tmp_workspace / "test-files-001" + assert run_dir.exists(), "run dir must be created" + round_dirs = [p for p in run_dir.iterdir() if p.is_dir()] + assert len(round_dirs) >= 1 + round_dir = round_dirs[0] + assert (round_dir / "attempt.py").exists(), "attempt.py must be written for code tasks" + assert (round_dir / "task_brief.md").exists(), "task_brief.md must be written by supervisor" + brief = (round_dir / "task_brief.md").read_text() + assert "Optimizer comparison on MNIST" in brief + assert "accuracy" in brief + + def test_failed_worker_records_error(self, tmp_workspace: Path, mock_parent_agent: MagicMock): + """When delegate_task returns failed status, result has error and is not kept.""" + spec = TaskSpec( + topic="Crash test", + deliverable="code that fails", + metric_key="accuracy", + task_type="code", + ) + with patch("tools.delegate_tool.delegate_task", return_value=_make_failed_delegate_result("Worker crashed")): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + spec, + initial_attempt="raise RuntimeError('oops')", + run_id="test-fail-001", + llm=None, + ) + + assert len(history.results) == 1 + result = history.results[0] + assert result.error is not None + assert result.kept is False + assert result.primary_metric is None + + def test_lattice_comment_fn_called(self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec): + """Lattice comment function is called at loop start and end.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.9)): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor._lattice_task_id = None # stub mode — logs only + history = supervisor.run( + code_spec, + initial_attempt="pass", + run_id="test-comment-001", + llm=None, + ) + + assert len(history.results) == 1 + + def test_learnings_jsonl_written(self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec): + """Autogenesis Observe step: learnings.jsonl is written with HeartbeatMemorySystem schema.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.88)): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + code_spec, + initial_attempt="print('accuracy: 0.88')", + run_id="test-learnings-001", + llm=None, + ) + + learnings_file = tmp_workspace / "test-learnings-001" / "learnings.jsonl" + assert learnings_file.exists(), "learnings.jsonl must be written by _observe()" + lines = [json.loads(l) for l in learnings_file.read_text().splitlines() if l.strip()] + assert len(lines) == 1 # one entry per iteration + entry = lines[0] + # Verify HeartbeatMemorySystem schema + assert "type" in entry + assert "key" in entry + assert "insight" in entry + assert "confidence" in entry + assert "source" in entry + assert entry["key"] == "accuracy" + assert entry["type"] in ("improvement", "regression", "failure") + assert entry["source"] == "iter-0" + + def test_search_task_writes_attempt_md(self, tmp_workspace: Path, mock_parent_agent: MagicMock): + """Search tasks write attempt.md, not attempt.py.""" + spec = TaskSpec( + topic="Find papers on transformers", + deliverable="Ranked list of papers", + metric_key="relevance_score", + task_type="search", + ) + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.8, "relevance_score")): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + spec, + initial_attempt="search query: transformer papers after 2022", + run_id="test-search-001", + llm=None, + ) + + run_dir = tmp_workspace / "test-search-001" + round_dirs = [p for p in run_dir.iterdir() if p.is_dir()] + assert len(round_dirs) >= 1 + round_dir = round_dirs[0] + assert (round_dir / "attempt.md").exists(), "attempt.md must be written for search tasks" + assert not (round_dir / "attempt.py").exists(), "attempt.py must NOT be written for search tasks" + + + def test_rollback_uses_on_disk_artifact(self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec): + """Fix #1 (audit): rollback restores on-disk artifact, not the seed string.""" + call_count = 0 + BEST_ARTIFACT = "# best on-disk version\nprint('accuracy: 0.90')" + + def capturing_delegate(goal, context, toolsets, parent_agent, inherit_profile=False): + nonlocal call_count + call_count += 1 + # Find the round dir from goal string and write a modified attempt.py + import re as _re + m = _re.search(r"round-[^\s]+", goal) + if m: + rd = tmp_workspace / "test-rollback-001" / m.group(0) + rd.mkdir(parents=True, exist_ok=True) + if call_count == 1: + # baseline — write a specific artifact to disk + (rd / "attempt.py").write_text(BEST_ARTIFACT, encoding="utf-8") + return _make_delegate_result(0.90) + else: + # iter1 — write a worse artifact but report regression + (rd / "attempt.py").write_text("# worse attempt", encoding="utf-8") + return _make_delegate_result(0.70) + return _make_delegate_result(0.0) + + mock_llm = MagicMock() + mock_llm.chat.return_value = MagicMock(content="```python\nprint('iter attempt')\n```") + + with patch("tools.delegate_tool.delegate_task", side_effect=capturing_delegate): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + code_spec, + initial_attempt="# initial seed", + run_id="test-rollback-001", + max_iterations=1, + llm=mock_llm, + ) + + # After regression, rollback should restore the best on-disk artifact + assert history.best_result is not None + assert history.best_result.primary_metric == pytest.approx(0.90, abs=0.001) + + +@pytest.mark.integration +class TestResearchSupervisorIterations: + """Multi-iteration loop with a mock LLM client.""" + + def _make_mock_llm(self) -> MagicMock: + class MockResponse: + content = "```python\nprint('updated code')\n```" + + llm = MagicMock() + llm.chat.return_value = MockResponse() + return llm + + def test_two_iteration_improvement(self, tmp_workspace: Path, mock_parent_agent: MagicMock): + """Loop improves once then plateaus — verifies history and best_result.""" + metric_sequence = iter([0.70, 0.82, 0.81, 0.80]) # baseline, iter1 improves, iter2/3 regress + + def side_effect(goal, context, toolsets, parent_agent, inherit_profile=False): + val = next(metric_sequence, 0.80) + return _make_delegate_result(val) + + spec = TaskSpec( + topic="Improvement test", + deliverable="Adam should converge better", + metric_key="accuracy", + metric_direction="maximize", + task_type="code", + ) + mock_llm = self._make_mock_llm() + + with patch("tools.delegate_tool.delegate_task", side_effect=side_effect): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + spec, + initial_attempt="# initial", + run_id="test-iter-001", + max_iterations=5, + llm=mock_llm, + ) + + # Should have stopped after 3 non-improving iterations past the best + assert len(history.results) >= 2 + best = history.best_result + assert best is not None + assert best.primary_metric == pytest.approx(0.82, abs=0.001) + + def test_early_stop_on_no_improvement(self, tmp_workspace: Path, mock_parent_agent: MagicMock): + """Loop stops early after 3 consecutive non-improving iterations.""" + call_count = 0 + + def side_effect(goal, context, toolsets, parent_agent, inherit_profile=False): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_delegate_result(0.5) # baseline + return _make_delegate_result(0.4) # always regress + + spec = TaskSpec( + topic="Early stop test", + deliverable="This will not improve", + metric_key="accuracy", + metric_direction="maximize", + task_type="code", + ) + mock_llm = MagicMock() + mock_llm.chat.return_value = MagicMock(content="```python\npass\n```") + + with patch("tools.delegate_tool.delegate_task", side_effect=side_effect): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + spec, + initial_attempt="# bad code", + run_id="test-early-001", + max_iterations=10, + llm=mock_llm, + ) + + # baseline + 3 failing iterations = 4 total + assert len(history.results) == 4 + assert call_count == 4 + + +# --------------------------------------------------------------------------- +# HRM-59: EvolutionStore wiring v1 — persist lessons after run() +# --------------------------------------------------------------------------- + +class TestEvolutionPersistence: + def test_evolve_writes_one_lesson_per_iteration( + self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec, tmp_path: Path + ): + """After run() returns, the EvolutionStore JSONL must have one entry + per ExperimentResult — covering improved/discarded/error severities.""" + from agent.research.evolution import EvolutionStore + + evolution_dir = tmp_path / "evolution-home" / "evolution" + + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.85)), \ + patch("agent.research.supervisor.get_hermes_home", return_value=tmp_path / "evolution-home"): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + code_spec, + initial_attempt="print('accuracy: 0.85')", + run_id="evo-test-001", + llm=None, # baseline only -> 1 result + ) + + lessons = EvolutionStore(evolution_dir).load_all() + assert len(lessons) == 1, "baseline-only run produces one lesson" + assert lessons[0].run_id == "evo-test-001" + assert lessons[0].stage_name == "iter_0" + assert lessons[0].severity in {"info", "warning"} + + def test_evolve_failure_does_not_break_run( + self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec, tmp_path: Path + ): + """If _evolve raises, run() must still return the history cleanly.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.9)), \ + patch.object(ResearchSupervisor, "_evolve", side_effect=RuntimeError("disk full")): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + history = supervisor.run( + code_spec, + initial_attempt="print('ok')", + run_id="evo-fail-001", + llm=None, + ) + + assert history is not None + assert len(history.results) == 1 + + +# --------------------------------------------------------------------------- +# HRM-62: EvolutionStore overlay wiring v2 — lessons surface in worker briefs +# --------------------------------------------------------------------------- + +class TestEvolutionOverlay: + def test_seeded_lesson_appears_in_worker_brief( + self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec, tmp_path: Path + ): + """Seed a lesson into the EvolutionStore, run a baseline-only experiment, + and assert the worker's task_brief.md contains the seeded text.""" + from agent.research.evolution import EvolutionStore, LessonEntry, LessonCategory + from datetime import datetime, timezone + + evolution_dir = tmp_path / "evolution-home" / "evolution" + evolution_dir.mkdir(parents=True) + EvolutionStore(evolution_dir).append(LessonEntry( + stage_name="research_loop", + stage_num=0, + category=LessonCategory.PIPELINE, + severity="error", + description="ALPHA-MARKER-XYZ: workers must validate metric before reporting", + timestamp=datetime.now(timezone.utc).isoformat(), + run_id="seed-001", + )) + + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.85)), \ + patch("agent.research.supervisor.get_hermes_home", return_value=tmp_path / "evolution-home"): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + code_spec, + initial_attempt="print('accuracy: 0.85')", + run_id="overlay-test-001", + llm=None, + ) + + run_dir = tmp_workspace / "overlay-test-001" + round_dirs = [p for p in run_dir.iterdir() if p.is_dir()] + brief_text = (round_dirs[0] / "task_brief.md").read_text() + assert "ALPHA-MARKER-XYZ" in brief_text, "seeded lesson must surface in worker brief" + assert "Lessons from Prior Runs" in brief_text, "overlay header expected" + + def test_no_overlay_when_store_empty( + self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec, tmp_path: Path + ): + """With no past lessons, the brief is unchanged — no empty overlay header.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.8)), \ + patch("agent.research.supervisor.get_hermes_home", return_value=tmp_path / "fresh-home"): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + supervisor.run( + code_spec, + initial_attempt="print('ok')", + run_id="empty-overlay-001", + llm=None, + ) + + run_dir = tmp_workspace / "empty-overlay-001" + round_dirs = [p for p in run_dir.iterdir() if p.is_dir()] + brief_text = (round_dirs[0] / "task_brief.md").read_text() + assert "Lessons from Prior Runs" not in brief_text + # And the brief still starts with the normal builder output + assert brief_text.lstrip().startswith("# Task Brief") + + def test_overlay_load_failure_does_not_break_run( + self, tmp_workspace: Path, mock_parent_agent: MagicMock, code_spec: TaskSpec + ): + """If _load_evolution_overlay raises, run() must still complete cleanly.""" + with patch("tools.delegate_tool.delegate_task", return_value=_make_delegate_result(0.9)), \ + patch.object(ResearchSupervisor, "_load_evolution_overlay", side_effect=RuntimeError("disk read failed")): + supervisor = ResearchSupervisor( + parent_agent=mock_parent_agent, + workspace=tmp_workspace, + ) + # run() invokes the helper directly; without protection it would propagate. + # The helper itself catches; the test asserts the user-facing contract: no propagation. + try: + history = supervisor.run( + code_spec, + initial_attempt="x", + run_id="overlay-fail-001", + llm=None, + ) + except RuntimeError: + # If propagation happens, mark as failure explicitly. + pytest.fail("_load_evolution_overlay failure must be swallowed") + assert history is not None diff --git a/tests/gateway/test_daemoncraft_cycle_detector.py b/tests/gateway/test_daemoncraft_cycle_detector.py new file mode 100644 index 000000000000..102de43745d7 --- /dev/null +++ b/tests/gateway/test_daemoncraft_cycle_detector.py @@ -0,0 +1,171 @@ +"""Unit tests for CycleDetector ported into gateway/platforms/daemoncraft.py.""" +from __future__ import annotations + +import os +import sys +import types +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# gateway/platforms/__init__.py eagerly imports yuanbao (httpx) and daemoncraft +# itself needs aiohttp. Stub missing optional deps before import. +def _stub_module(name: str, **attrs): + if name not in sys.modules: + mod = types.ModuleType(name) + for k, v in attrs.items(): + setattr(mod, k, v) + sys.modules[name] = mod + +_stub_module("httpx") +_stub_module("aiohttp", WSMsgType=MagicMock(), ClientSession=MagicMock) + +# Import the standalone class directly — no server needed +from gateway.platforms.daemoncraft import CycleDetector + + +# --------------------------------------------------------------------------- +# CycleDetector unit tests +# --------------------------------------------------------------------------- + +class TestCycleDetectorUnit: + def test_no_cycle_below_threshold(self): + cd = CycleDetector(n=3, window=10, action="warn") + for _ in range(2): + r = cd.record("tool_a", {}) + assert r.triggered is False + + def test_cycle_on_nth_identical_call(self): + cd = CycleDetector(n=3, window=10, action="warn") + r = None + for _ in range(3): + r = cd.record("tool_a", {}) + assert r.triggered is True + assert r.count >= 3 + + def test_no_double_trigger_on_n_plus_one(self): + cd = CycleDetector(n=3, window=10, action="warn") + for _ in range(3): + cd.record("tool_a", {}) + # 4th call: same sig, already triggered — should suppress + r = cd.record("tool_a", {}) + assert r.triggered is False + + def test_different_tool_names_no_cycle(self): + cd = CycleDetector(n=3, window=10, action="warn") + results = [] + for i in range(6): + results.append(cd.record(f"tool_{i}", {})) + assert not any(r.triggered for r in results) + + def test_different_args_no_cycle(self): + cd = CycleDetector(n=3, window=10, action="warn") + results = [] + for i in range(6): + results.append(cd.record("tool_a", {"x": i})) + assert not any(r.triggered for r in results) + + def test_cycle_clears_after_different_sig_dominates(self): + """After suppression, a NEW dominant sig should trigger fresh.""" + # Use small window=3 so tool_b can fully dominate and evict tool_a + cd = CycleDetector(n=3, window=3, action="warn") + # Trigger first cycle for tool_a + for _ in range(3): + cd.record("tool_a", {}) + # Flood with tool_b — fills the window, clears _last_triggered_sig + for _ in range(3): + cd.record("tool_b", {}) + # Now tool_a again — should trigger fresh (suppression was cleared) + r = None + for _ in range(3): + r = cd.record("tool_a", {}) + assert r.triggered is True + + +# --------------------------------------------------------------------------- +# DaemonCraftAdapter._check_cycle integration tests +# --------------------------------------------------------------------------- + +def _make_adapter(): + """Build a minimal DaemonCraftAdapter with all external deps mocked.""" + from gateway.platforms.daemoncraft import DaemonCraftAdapter + from gateway.config import PlatformConfig + + cfg = PlatformConfig( + enabled=True, + extra={ + "bot_api_url": "http://localhost:9999", + "bot_username": "TestBot", + "profile": "test", + }, + ) + adapter = DaemonCraftAdapter(cfg) + # Patch _interrupt_agent so tests don't need a real HTTP session + adapter._interrupt_agent = AsyncMock() + return adapter + + +class TestCheckCycleMethod: + @pytest.mark.anyio + async def test_returns_false_when_no_detector(self): + adapter = _make_adapter() + assert adapter._cycle_detector is None + result = await adapter._check_cycle("mc_perceive", {}) + assert result is False + + @pytest.mark.anyio + async def test_returns_false_for_non_cycling_calls(self): + adapter = _make_adapter() + adapter._cycle_detector = CycleDetector(n=3, window=10, action="warn") + result = await adapter._check_cycle("mc_perceive", {}) + assert result is False + + @pytest.mark.anyio + async def test_warn_action_returns_false_on_cycle(self): + """Cycle detected with action='warn' should log but NOT interrupt.""" + adapter = _make_adapter() + adapter._cycle_detector = CycleDetector(n=3, window=10, action="warn") + for _ in range(2): + await adapter._check_cycle("mc_perceive", {}) + result = await adapter._check_cycle("mc_perceive", {}) + # warn = no interrupt + assert result is False + adapter._interrupt_agent.assert_not_called() + + @pytest.mark.anyio + async def test_interrupt_action_returns_true_and_calls_interrupt(self): + """Cycle with action='interrupt' should call _interrupt_agent and return True.""" + adapter = _make_adapter() + adapter._cycle_detector = CycleDetector(n=3, window=10, action="interrupt") + for _ in range(2): + await adapter._check_cycle("mc_perceive", {}) + result = await adapter._check_cycle("mc_perceive", {}) + assert result is True + adapter._interrupt_agent.assert_called_once_with("cycle_detected") + + +class TestAdapterCycleDetectorInit: + @pytest.mark.anyio + async def test_no_detector_when_mc_cycle_n_zero(self, monkeypatch): + monkeypatch.delenv("MC_CYCLE_N", raising=False) + adapter = _make_adapter() + # Patch connect internals so no actual socket is opened + with patch("gateway.platforms.daemoncraft.aiohttp.ClientSession", return_value=MagicMock()), \ + patch("asyncio.create_task", return_value=MagicMock()): + await adapter.connect() + assert adapter._cycle_detector is None + + @pytest.mark.anyio + async def test_detector_created_when_mc_cycle_n_set(self, monkeypatch): + monkeypatch.setenv("MC_CYCLE_N", "3") + monkeypatch.setenv("MC_CYCLE_WINDOW", "10") + monkeypatch.setenv("MC_CYCLE_ACTION", "warn") + adapter = _make_adapter() + with patch("gateway.platforms.daemoncraft.aiohttp.ClientSession", return_value=MagicMock()), \ + patch("asyncio.create_task", return_value=MagicMock()): + await adapter.connect() + assert adapter._cycle_detector is not None + assert adapter._cycle_detector.n == 3 + assert adapter._cycle_detector.window == 10 + assert adapter._cycle_detector.action == "warn" diff --git a/tests/gateway/test_daemoncraft_patches.py b/tests/gateway/test_daemoncraft_patches.py new file mode 100644 index 000000000000..6f7b51382c2d --- /dev/null +++ b/tests/gateway/test_daemoncraft_patches.py @@ -0,0 +1,197 @@ +"""Tests for CycleDetector and _inject_synthetic_perceive hook in daemoncraft.py.""" +from __future__ import annotations + +import sys +import types +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +# --------------------------------------------------------------------------- +# Stub heavy optional deps before importing daemoncraft +# --------------------------------------------------------------------------- + +def _stub_module(name: str, **attrs): + if name not in sys.modules: + mod = types.ModuleType(name) + for k, v in attrs.items(): + setattr(mod, k, v) + sys.modules[name] = mod + +_stub_module("httpx") +_stub_module("aiohttp", WSMsgType=MagicMock(), ClientSession=MagicMock) + +from gateway.platforms.daemoncraft import CycleDetector # noqa: E402 + + +# =========================================================================== +# CycleDetector tests +# =========================================================================== + +class TestCycleDetector: + """5 focused tests for CycleDetector behaviour.""" + + def test_no_trigger_below_threshold(self): + """N-1 identical calls must NOT trigger.""" + cd = CycleDetector(n=4, window=20, action="warn") + results = [cd.record("tool_x", {"k": "v"}) for _ in range(3)] + assert not any(r.triggered for r in results) + + def test_trigger_at_nth_identical_call(self): + """The Nth identical call must trigger.""" + cd = CycleDetector(n=3, window=10, action="warn") + r = None + for _ in range(3): + r = cd.record("loop_tool", {}) + assert r.triggered is True + assert r.count >= 3 + + def test_reset_after_action_no_double_trigger(self): + """After triggering, subsequent calls with the same sig should NOT re-trigger.""" + cd = CycleDetector(n=3, window=10, action="warn") + for _ in range(3): + cd.record("loop_tool", {}) + # 4th and 5th same-sig calls — suppressed + r4 = cd.record("loop_tool", {}) + r5 = cd.record("loop_tool", {}) + assert r4.triggered is False + assert r5.triggered is False + + def test_window_size_evicts_old_entries(self): + """Once the ring buffer (size=window) is filled with other sigs, old counts are gone.""" + # window=3: buffer holds at most 3 entries + cd = CycleDetector(n=3, window=3, action="warn") + # Two calls of "old_tool" — not yet triggering + cd.record("old_tool", {}) + cd.record("old_tool", {}) + # Fill buffer with 3 different sigs, evicting "old_tool" entries + cd.record("tool_b", {}) + cd.record("tool_c", {}) + cd.record("tool_d", {}) + # Now one more "old_tool" — only 1 in window, should not trigger + r = cd.record("old_tool", {}) + assert r.triggered is False + + def test_different_sigs_do_not_trigger(self): + """Calls with different args must not be counted together.""" + cd = CycleDetector(n=3, window=10, action="warn") + results = [cd.record("tool_a", {"n": i}) for i in range(6)] + assert not any(r.triggered for r in results) + + +# =========================================================================== +# _inject_synthetic_perceive hook tests +# =========================================================================== + +def _make_adapter(): + from gateway.platforms.daemoncraft import DaemonCraftAdapter + from gateway.config import PlatformConfig + + cfg = PlatformConfig( + enabled=True, + extra={ + "bot_api_url": "http://localhost:9999", + "bot_username": "TestBot", + "profile": "test", + }, + ) + adapter = DaemonCraftAdapter(cfg) + return adapter + + +def _wire_adapter(adapter, *, session_id="world-session-1", hook_results=()): + """Attach a mock session_store and stub invoke_hook.""" + store = MagicMock() + store.append_to_transcript = MagicMock() + adapter._session_store = store + + # Stub _get_world_session_id + adapter._get_world_session_id = MagicMock(return_value=session_id) + return store + + +class TestSyntheticPerceiveHook: + """3 tests covering the transform_tool_result hook path.""" + + @pytest.mark.anyio + async def test_hook_called_before_transcript_append(self): + """invoke_hook must be called; tool_msg append comes after it.""" + adapter = _make_adapter() + store = _wire_adapter(adapter) + call_order = [] + + def fake_invoke_hook(event, **kwargs): + call_order.append("hook") + return iter([]) # no replacement + + # Capture append_to_transcript calls in order + original_append = store.append_to_transcript + def recording_append(sid, msg): + call_order.append(("append", msg["role"])) + store.append_to_transcript.side_effect = recording_append + + with patch("gateway.platforms.daemoncraft.invoke_hook", fake_invoke_hook, create=True), \ + patch.dict(sys.modules, {"hermes_cli.plugins": types.SimpleNamespace(invoke_hook=fake_invoke_hook)}): + # Patch the local import inside _inject_synthetic_perceive + import importlib + import gateway.platforms.daemoncraft as dc_mod + with patch.object(dc_mod, "_inject_synthetic_perceive_hook_module", None, create=True): + # We patch the from-import by monkeypatching the module namespace + pass + + # Direct patch: replace hermes_cli.plugins in sys.modules + fake_plugins = types.ModuleType("hermes_cli.plugins") + fake_plugins.invoke_hook = fake_invoke_hook + sys.modules["hermes_cli.plugins"] = fake_plugins + sys.modules.setdefault("hermes_cli", types.ModuleType("hermes_cli")) + + await adapter._inject_synthetic_perceive({"x": 1}) + + # assistant append should come first, then hook, then tool append + assert ("append", "assistant") in call_order + assert ("append", "tool") in call_order + assert call_order.index(("append", "assistant")) < call_order.index("hook") + assert call_order.index("hook") < call_order.index(("append", "tool")) + + @pytest.mark.anyio + async def test_hook_receives_mc_perceive_tool_name(self): + """invoke_hook must be called with tool_name='mc_perceive'.""" + adapter = _make_adapter() + _wire_adapter(adapter) + + received_kwargs: dict = {} + + def fake_invoke_hook(event, **kwargs): + received_kwargs.update({"event": event, **kwargs}) + return iter([]) + + fake_plugins = types.ModuleType("hermes_cli.plugins") + fake_plugins.invoke_hook = fake_invoke_hook + sys.modules["hermes_cli.plugins"] = fake_plugins + sys.modules.setdefault("hermes_cli", types.ModuleType("hermes_cli")) + + await adapter._inject_synthetic_perceive({"obs": "block"}) + + assert received_kwargs.get("event") == "transform_tool_result" + assert received_kwargs.get("tool_name") == "mc_perceive" + + @pytest.mark.anyio + async def test_transcript_appended_even_if_hook_raises(self): + """If invoke_hook raises, transcript append must still happen.""" + adapter = _make_adapter() + store = _wire_adapter(adapter) + + def exploding_hook(event, **kwargs): + raise RuntimeError("hook boom") + + fake_plugins = types.ModuleType("hermes_cli.plugins") + fake_plugins.invoke_hook = exploding_hook + sys.modules["hermes_cli.plugins"] = fake_plugins + sys.modules.setdefault("hermes_cli", types.ModuleType("hermes_cli")) + + await adapter._inject_synthetic_perceive({"obs": "fire"}) + + # Both assistant_msg and tool_msg must have been appended + assert store.append_to_transcript.call_count == 2 + roles = [c.args[1]["role"] for c in store.append_to_transcript.call_args_list] + assert roles == ["assistant", "tool"] diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 5585eea48409..b26daf1d7b69 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -1968,6 +1968,7 @@ def test_invoke_tool_dispatches_to_handle_function_call(self, agent): session_id=agent.session_id, enabled_tools=list(agent.valid_tool_names), skip_pre_tool_call_hook=True, + parent_agent=agent, ) assert result == "result" diff --git a/tests/test_model_tools.py b/tests/test_model_tools.py index 379aac2bbcfb..537cbf6fab49 100644 --- a/tests/test_model_tools.py +++ b/tests/test_model_tools.py @@ -112,6 +112,27 @@ def test_post_tool_call_receives_non_negative_integer_duration_ms(self): # pre_tool_call does NOT get duration_ms (nothing has run yet). assert "duration_ms" not in kwargs_by_hook["pre_tool_call"] + def test_parent_agent_passed_to_registry_dispatch(self): + """parent_agent should be forwarded to registry.dispatch so tools like + run_research and delegate_task receive the agent context.""" + with patch("model_tools.registry.dispatch", return_value='{"ok":true}') as mock_dispatch: + fake_agent = object() + result = handle_function_call( + "run_research", + {"topic": "x", "deliverable": "y", "metric_key": "z"}, + task_id="task-1", + parent_agent=fake_agent, + ) + + assert result == '{"ok":true}' + mock_dispatch.assert_called_once_with( + "run_research", + {"topic": "x", "deliverable": "y", "metric_key": "z"}, + task_id="task-1", + user_task=None, + parent_agent=fake_agent, + ) + # ========================================================================= # Agent loop tools diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index 6b4cc9915082..c66b963b42b2 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -379,6 +379,49 @@ def test_build_child_agent_does_not_raise_name_error(self): f"_saved_tool_names leaked back into wrong scope: {exc}" ) + def test_inherit_profile_default_keeps_blank_slate(self): + """Default inherit_profile=False -> child runs with skip_context_files=True + and skip_memory=True, preserving the historical batch/data-gen behavior.""" + parent = _make_mock_parent(depth=0) + captured = {} + + with patch("run_agent.AIAgent") as MockAgent: + MockAgent.side_effect = lambda *a, **kw: captured.update(kw) or MagicMock() + _build_child_agent( + task_index=0, + goal="batch task", + context=None, + toolsets=None, + model=None, + max_iterations=10, + parent_agent=parent, + task_count=1, + ) + self.assertTrue(captured.get("skip_context_files")) + self.assertTrue(captured.get("skip_memory")) + + def test_inherit_profile_true_loads_profile_context(self): + """inherit_profile=True flips both skips to False so the child loads + SOUL.md / AGENTS.md / MEMORY.md from the active profile.""" + parent = _make_mock_parent(depth=0) + captured = {} + + with patch("run_agent.AIAgent") as MockAgent: + MockAgent.side_effect = lambda *a, **kw: captured.update(kw) or MagicMock() + _build_child_agent( + task_index=0, + goal="research worker", + context=None, + toolsets=None, + model=None, + max_iterations=10, + parent_agent=parent, + task_count=1, + inherit_profile=True, + ) + self.assertFalse(captured.get("skip_context_files")) + self.assertFalse(captured.get("skip_memory")) + def test_saved_tool_names_set_on_child_before_run(self): """_run_single_child must set _delegate_saved_tool_names on the child from model_tools._last_resolved_tool_names before run_conversation.""" diff --git a/tools/approval.py b/tools/approval.py index 78fb48178318..f777c072f3ba 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -212,6 +212,8 @@ def _hardline_block_result(description: str) -> dict: # ========================================================================= DANGEROUS_PATTERNS = [ + # Specific allowlisted commands (must come before broader patterns) + (r'\brm\s+(-[^\s]*\s+)*/tmp\b', "rm -rf /tmp"), (r'\brm\s+(-[^\s]*\s+)*/', "delete in root path"), (r'\brm\s+-[^\s]*r', "recursive delete"), (r'\brm\s+--recursive\b', "recursive delete (long flag)"), diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index ffcf726fcd5b..163adc8cc2d8 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -146,7 +146,8 @@ def check_sandbox_requirements() -> bool: def generate_hermes_tools_module(enabled_tools: List[str], - transport: str = "uds") -> str: + transport: str = "uds", + timeout: int = DEFAULT_TIMEOUT) -> str: """ Build the source code for the hermes_tools.py stub module. @@ -173,9 +174,9 @@ def generate_hermes_tools_module(enabled_tools: List[str], export_names.append(func_name) if transport == "file": - header = _FILE_TRANSPORT_HEADER + header = _FILE_TRANSPORT_HEADER.format(timeout=timeout) else: - header = _UDS_TRANSPORT_HEADER + header = _UDS_TRANSPORT_HEADER.format(timeout=timeout) return header + "\n".join(stub_functions) @@ -239,7 +240,7 @@ def _connect(): if _sock is None: _sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) _sock.connect(os.environ["HERMES_RPC_SOCKET"]) - _sock.settimeout(300) + _sock.settimeout({timeout}) return _sock def _call(tool_name, args): @@ -298,11 +299,11 @@ def _call(tool_name, args): os.rename(tmp, req_file) # Wait for response with adaptive polling - deadline = time.monotonic() + 300 # 5-minute timeout per tool call + deadline = time.monotonic() + {timeout} # configurable timeout per tool call poll_interval = 0.05 # Start at 50ms while not os.path.exists(res_file): if time.monotonic() > deadline: - raise RuntimeError(f"RPC timeout: no response for {tool_name} after 300s") + raise RuntimeError(f"RPC timeout: no response for {{tool_name}} after {{{timeout}}}s") time.sleep(poll_interval) poll_interval = min(poll_interval * 1.2, 0.25) # Back off to 250ms @@ -341,6 +342,7 @@ def _rpc_server_loop( tool_call_counter: list, # mutable [int] so the thread can increment max_tool_calls: int, allowed_tools: frozenset, + timeout: int = DEFAULT_TIMEOUT, ): """ Accept one client connection and dispatch tool-call requests until @@ -352,7 +354,7 @@ def _rpc_server_loop( try: server_sock.settimeout(5) conn, _ = server_sock.accept() - conn.settimeout(300) + conn.settimeout(timeout) buf = b"" while True: @@ -797,7 +799,7 @@ def _execute_remote( # Generate and ship files tools_src = generate_hermes_tools_module( - list(sandbox_tools), transport="file", + list(sandbox_tools), transport="file", timeout=timeout, ) _ship_file_to_remote(env, f"{sandbox_dir}/hermes_tools.py", tools_src) _ship_file_to_remote(env, f"{sandbox_dir}/script.py", code) @@ -1000,7 +1002,7 @@ def execute_code( # Write the auto-generated hermes_tools module # sandbox_tools is already the correct set (intersection with session # tools, or SANDBOX_ALLOWED_TOOLS as fallback — see lines above). - tools_src = generate_hermes_tools_module(list(sandbox_tools)) + tools_src = generate_hermes_tools_module(list(sandbox_tools), timeout=timeout) with open(os.path.join(tmpdir, "hermes_tools.py"), "w") as f: f.write(tools_src) @@ -1019,6 +1021,7 @@ def execute_code( args=( server_sock, task_id, tool_call_log, tool_call_counter, max_tool_calls, sandbox_tools, + timeout, ), daemon=True, ) @@ -1568,7 +1571,7 @@ def build_execute_code_schema(enabled_sandbox_tools: set = None, "or the task requires interactive user input.\n\n" f"Available via `from hermes_tools import ...`:\n\n" f"{tool_lines}\n\n" - "Limits: 5-minute timeout, 50KB stdout cap, max 50 tool calls per script. " + "Limits: 15-minute timeout, 50KB stdout cap, max 50 tool calls per script. " "terminal() is foreground-only (no background or pty).\n\n" f"{cwd_note}\n\n" "Print your final result to stdout. Use Python stdlib (json, re, math, csv, " diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 7d2bb197e0ba..ad5358163ad2 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -852,6 +852,11 @@ def _build_child_agent( # 'leaf' (default) cannot; 'orchestrator' retains the delegation # toolset subject to depth/kill-switch bounds applied below. role: str = "leaf", + # Opt-in: load the active profile's SOUL.md, AGENTS.md, and MEMORY.md + # into the child. Default False preserves the historical blank-slate + # behavior used by batch / data-generation callers; research workers + # set this to True so they inherit the curated researcher profile. + inherit_profile: bool = False, ): """ Build a child AIAgent on the main thread (thread-safe construction). @@ -979,9 +984,9 @@ def _child_thinking(text: str) -> None: child_thinking_cb = _child_thinking # Resolve effective credentials: config override > parent inherit - effective_model = model or parent_agent.model + effective_model = model or getattr(parent_agent, "model", None) effective_provider = override_provider or getattr(parent_agent, "provider", None) - effective_base_url = override_base_url or parent_agent.base_url + effective_base_url = override_base_url or getattr(parent_agent, "base_url", None) effective_api_key = override_api_key or parent_api_key effective_api_mode = override_api_mode or getattr(parent_agent, "api_mode", None) effective_acp_command = override_acp_command or getattr( @@ -1043,8 +1048,8 @@ def _child_thinking(text: str) -> None: ephemeral_system_prompt=child_prompt, log_prefix=f"[subagent-{task_index}]", platform=parent_agent.platform, - skip_context_files=True, - skip_memory=True, + skip_context_files=not inherit_profile, + skip_memory=not inherit_profile, clarify_callback=None, thinking_callback=child_thinking_cb, session_db=getattr(parent_agent, "_session_db", None), @@ -1818,6 +1823,7 @@ def delegate_task( acp_command: Optional[str] = None, acp_args: Optional[List[str]] = None, role: Optional[str] = None, + inherit_profile: bool = False, parent_agent=None, ) -> str: """ @@ -1964,6 +1970,7 @@ def delegate_task( else (acp_args if acp_args is not None else creds.get("args")) ), role=effective_role, + inherit_profile=inherit_profile, ) # Override with correct parent tool names (before child construction mutated global) child._delegate_saved_tool_names = _parent_tool_names diff --git a/tools/research_job_tool.py b/tools/research_job_tool.py new file mode 100644 index 000000000000..c8f081d78839 --- /dev/null +++ b/tools/research_job_tool.py @@ -0,0 +1,379 @@ +"""research_job_tool — orchestrate long-running research jobs as detached OS processes. + +Provides start, status, collect, and resume operations for research loops +that outlive a single agent turn. +""" + +from __future__ import annotations + +import json +import logging +import os +import secrets +import shlex +from pathlib import Path +from typing import Any, Optional + +from hermes_constants import get_hermes_home +from tools.registry import registry, tool_error + +logger = logging.getLogger(__name__) + + +def _job_dir(job_id: str) -> Path: + return get_hermes_home() / "research-jobs" / job_id + + +def _write_job_spec(job_id: str, spec: dict[str, Any]) -> Path: + jd = _job_dir(job_id) + jd.mkdir(parents=True, exist_ok=True) + spec_path = jd / "job.json" + spec_path.write_text(json.dumps(spec, indent=2)) + return spec_path + + +def _load_config_for_job() -> dict[str, Any]: + """Read Hermes config to extract model/provider/base_url for the runner.""" + import yaml + config_path = get_hermes_home() / "config.yaml" + if not config_path.exists(): + return {} + cfg = yaml.safe_load(config_path.read_text()) + model_cfg = cfg.get("model", {}) + delegation_cfg = cfg.get("delegation", {}) + return { + "model": delegation_cfg.get("model") or model_cfg.get("default", "kimi-for-coding"), + "provider": delegation_cfg.get("provider") or model_cfg.get("provider", "kimi-coding"), + "base_url": delegation_cfg.get("base_url") or model_cfg.get("base_url", "https://api.kimi.com/coding/v1"), + "api_key": os.getenv("KIMI_API_KEY", ""), + } + + +def _lattice_available() -> bool: + """Check whether Lattice is initialized so research jobs can post comments.""" + lattice_dir = get_hermes_home() / "org" / ".lattice" + return lattice_dir.exists() and (lattice_dir / "ids.json").exists() + + +# --------------------------------------------------------------------------- +# Tool schema +# --------------------------------------------------------------------------- + +RESEARCH_JOB_SCHEMA = { + "name": "research_job", + "description": ( + "Start, monitor, or resume a long-running research job as a detached OS process. " + "Use this instead of run_research when the loop may take longer than a single " + "agent turn (e.g. >5 minutes). Jobs are durable: state is checkpointed to disk " + "after every round, and can be resumed if the process crashes.\n\n" + "USE WHEN:\n" + "- A research task needs multiple iterations and may take 10+ minutes\n" + "- You cannot afford to keep a foreground agent alive as a watcher\n\n" + "NOT FOR:\n" + "- One-shot tasks (use delegate_task directly)\n" + "- Tasks that fit in a single agent turn (use run_research)\n\n" + "IMPORTANT: This tool spawns a background process. Poll status with " + "`research_job_status` or wait for the process completion notification." + ), + "parameters": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["start", "status", "collect", "resume"], + "description": "Operation to perform on the research job.", + }, + "job_id": { + "type": "string", + "description": "Job identifier. Required for status, collect, resume. Generated on start if omitted.", + }, + "topic": { + "type": "string", + "description": "What to research. Required for start.", + }, + "deliverable": { + "type": "string", + "description": "Concrete output the worker must produce. Required for start.", + }, + "metric_key": { + "type": "string", + "description": "Name of the metric to optimize. Required for start.", + }, + "metric_direction": { + "type": "string", + "enum": ["maximize", "minimize"], + "description": "Whether higher or lower metric values are better. Default: maximize.", + }, + "task_type": { + "type": "string", + "enum": ["code", "search", "research", "generic"], + "description": "Task domain. Default: generic.", + }, + "evaluation_mode": { + "type": "string", + "enum": ["self_report", "llm_judge"], + "description": "How to score worker output. Default: self_report.", + }, + "evaluation_prompt": { + "type": "string", + "description": "For llm_judge mode: scoring rubric.", + }, + "max_iterations": { + "type": "integer", + "description": "Max improvement iterations after baseline. Default: 3.", + }, + "time_budget_sec": { + "type": "integer", + "description": "Time budget per worker invocation in seconds. Default: 0 (unlimited).", + }, + "lattice_task_id": { + "type": "string", + "description": "Optional Lattice task ID for round-by-round progress comments.", + }, + "initial_attempt": { + "type": "string", + "description": "Optional starting scaffold for the worker.", + }, + }, + "required": ["action"], + }, +} + + +# --------------------------------------------------------------------------- +# Actions +# --------------------------------------------------------------------------- + +def _action_start(args: dict[str, Any]) -> str: + job_id = args.get("job_id") or secrets.token_hex(8) + cfg = _load_config_for_job() + lattice_task_id = args.get("lattice_task_id") + if lattice_task_id and not _lattice_available(): + logger.warning( + "Lattice task ID %s requested but ~/.hermes/org/.lattice/ is not initialized. " + "Progress comments will be skipped. Run 'cd ~/.hermes/org && lattice init' to enable.", + lattice_task_id, + ) + + spec = { + "job_id": job_id, + "topic": args.get("topic", ""), + "deliverable": args.get("deliverable", ""), + "metric_key": args.get("metric_key", ""), + "metric_direction": args.get("metric_direction", "maximize"), + "task_type": args.get("task_type", "generic"), + "evaluation_mode": args.get("evaluation_mode", "self_report"), + "evaluation_prompt": args.get("evaluation_prompt", ""), + "max_iterations": args.get("max_iterations", 3), + "time_budget_sec": args.get("time_budget_sec", 0), + "lattice_task_id": args.get("lattice_task_id"), + "initial_attempt": args.get("initial_attempt", ""), + "model": cfg.get("model"), + "provider": cfg.get("provider"), + "base_url": cfg.get("base_url"), + "api_key": cfg.get("api_key"), + "toolsets": ["research", "terminal", "file", "web"], + } + + spec_path = _write_job_spec(job_id, spec) + job_dir = _job_dir(job_id) + + hermes_root = get_hermes_home() / "hermes-agent" + cmd = ( + f"cd {shlex.quote(str(hermes_root))} && " + f"source venv/bin/activate && " + f"HERMES_YOLO_MODE=1 python -m agent.research.job_runner {shlex.quote(str(spec_path))}" + ) + + # Spawn via terminal_tool in background + from tools.terminal_tool import terminal + raw = terminal( + command=cmd, + background=True, + notify_on_complete=True, + workdir=str(hermes_root), + ) + proc = json.loads(raw) if isinstance(raw, str) else raw + + state = { + "job_id": job_id, + "status": "queued", + "process_session_id": proc.get("session_id"), + "pid": proc.get("pid"), + "job_dir": str(job_dir), + "spec_path": str(spec_path), + } + (job_dir / "state.json").write_text(json.dumps(state, indent=2)) + + return json.dumps({ + "ok": True, + "job_id": job_id, + "status": "queued", + "message": f"Research job {job_id} queued. Poll with research_job_status or wait for completion notification.", + "job_dir": str(job_dir), + "process_session_id": proc.get("session_id"), + }, indent=2) + + +def _action_status(args: dict[str, Any]) -> str: + job_id = args.get("job_id", "") + if not job_id: + return tool_error("job_id is required for status") + + job_dir = _job_dir(job_id) + state_path = job_dir / "state.json" + if not state_path.exists(): + return json.dumps({"ok": False, "error": f"Job {job_id} not found"}, indent=2) + + state = json.loads(state_path.read_text()) + + # If still running, also poll the background process + if state.get("status") in ("queued", "running"): + session_id = state.get("process_session_id") + if session_id: + try: + from tools.process_registry import process + proc_info = process(action="poll", session_id=session_id) + state["process_alive"] = proc_info.get("status") == "running" + state["process_uptime_seconds"] = proc_info.get("uptime_seconds") + except Exception: + state["process_alive"] = False + + # Include latest metric if available + history_path = job_dir / "history.json" + if history_path.exists(): + try: + history = json.loads(history_path.read_text()) + best = history.get("best") + if best: + state["best_metric"] = best.get("primary_metric") + state["best_iteration"] = best.get("iteration") + except Exception: + pass + + return json.dumps({"ok": True, **state}, indent=2) + + +def _action_collect(args: dict[str, Any]) -> str: + job_id = args.get("job_id", "") + if not job_id: + return tool_error("job_id is required for collect") + + job_dir = _job_dir(job_id) + result_path = job_dir / "result.json" + state_path = job_dir / "state.json" + + if not result_path.exists(): + status = "unknown" + if state_path.exists(): + status = json.loads(state_path.read_text()).get("status", "unknown") + return json.dumps({ + "ok": False, + "error": f"Result not ready. Job status: {status}", + "job_id": job_id, + }, indent=2) + + result = json.loads(result_path.read_text()) + return json.dumps({"ok": True, "job_id": job_id, **result}, indent=2) + + +def _action_resume(args: dict[str, Any]) -> str: + job_id = args.get("job_id", "") + if not job_id: + return tool_error("job_id is required for resume") + + job_dir = _job_dir(job_id) + state_path = job_dir / "state.json" + spec_path = job_dir / "job.json" + history_path = job_dir / "history.json" + + if not state_path.exists() or not spec_path.exists(): + return json.dumps({"ok": False, "error": f"Job {job_id} not found"}, indent=2) + + state = json.loads(state_path.read_text()) + if state.get("status") not in ("interrupted", "failed"): + return json.dumps({ + "ok": False, + "error": f"Cannot resume job in status '{state.get('status')}'. Only interrupted or failed jobs can be resumed." + }, indent=2) + + # Mark as resuming and re-launch + state["status"] = "resuming" + state_path.write_text(json.dumps(state, indent=2)) + + hermes_root = get_hermes_home() / "hermes-agent" + cmd = ( + f"cd {shlex.quote(str(hermes_root))} && " + f"source venv/bin/activate && " + f"HERMES_YOLO_MODE=1 python -m agent.research.job_runner {shlex.quote(str(spec_path))}" + ) + + from tools.terminal_tool import terminal + raw = terminal( + command=cmd, + background=True, + notify_on_complete=True, + workdir=str(hermes_root), + ) + proc = json.loads(raw) if isinstance(raw, str) else raw + + state["status"] = "queued" + state["process_session_id"] = proc.get("session_id") + state["pid"] = proc.get("pid") + state["resumed_at"] = time.time() + state_path.write_text(json.dumps(state, indent=2)) + + return json.dumps({ + "ok": True, + "job_id": job_id, + "status": "queued", + "message": f"Research job {job_id} resumed.", + "process_session_id": proc.get("session_id"), + }, indent=2) + + +# --------------------------------------------------------------------------- +# Tool handler +# --------------------------------------------------------------------------- + +def research_job( + action: str, + job_id: str = "", + topic: str = "", + deliverable: str = "", + metric_key: str = "", + metric_direction: str = "maximize", + task_type: str = "generic", + evaluation_mode: str = "self_report", + evaluation_prompt: str = "", + max_iterations: int = 3, + time_budget_sec: int = 0, + lattice_task_id: str = "", + initial_attempt: str = "", + **_: Any, +) -> str: + if action == "start": + if not topic or not deliverable or not metric_key: + return tool_error("topic, deliverable, and metric_key are required for start") + return _action_start(locals()) + elif action == "status": + return _action_status(locals()) + elif action == "collect": + return _action_collect(locals()) + elif action == "resume": + return _action_resume(locals()) + else: + return tool_error(f"Unknown action: {action}") + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + +registry.register( + name="research_job", + toolset="research", + schema=RESEARCH_JOB_SCHEMA, + handler=lambda args, **kw: research_job(**args), + emoji="📋", +) diff --git a/tools/research_tool.py b/tools/research_tool.py new file mode 100644 index 000000000000..d1bd52632f1e --- /dev/null +++ b/tools/research_tool.py @@ -0,0 +1,267 @@ +"""run_research — iterative self-improving research loop tool. + +Exposes ResearchSupervisor as a tool callable by the LLM, following the +same pattern as delegate_task. The LLM calls run_research when a task +benefits from multiple iterations scored against a measurable metric. + +Autogenesis AOOR loop: Act → Observe → Optimize → Remember. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Tool schema +# --------------------------------------------------------------------------- + +RESEARCH_TOOL_SCHEMA = { + "name": "run_research", + "description": ( + "Run a self-improving research loop on any task with a measurable deliverable. " + "Spawns worker subagents iteratively, scores their output against a metric, " + "and applies LLM-guided hypothesis revision to improve the metric across rounds.\n\n" + "USE WHEN:\n" + "- A task requires iterative improvement toward a measurable quality criterion\n" + "- You need web research, code optimization, or synthesis with self-evaluation\n" + "- Single-shot delegate_task is not enough — the task benefits from multiple rounds\n\n" + "NOT FOR:\n" + "- One-shot tasks (use delegate_task directly)\n" + "- Tasks with no measurable metric (use delegate_task)\n\n" + "IMPORTANT: This tool spawns multiple subagents and can run for several minutes. " + "Inform the user before calling it." + ), + "parameters": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "What to research or accomplish. Be specific.", + }, + "deliverable": { + "type": "string", + "description": ( + "Concrete output the worker must produce. " + "E.g. 'Python class with insert/search/delete', " + "'ranked list of papers with abstracts and relevance scores'." + ), + }, + "metric_key": { + "type": "string", + "description": ( + "Name of the metric to optimize. " + "E.g. 'pass_rate', 'relevance_score', 'completeness_score', 'latency_ms'." + ), + }, + "metric_direction": { + "type": "string", + "enum": ["maximize", "minimize"], + "description": "Whether higher or lower metric values are better. Default: maximize.", + }, + "task_type": { + "type": "string", + "enum": ["code", "search", "research", "generic"], + "description": ( + "Task domain. Controls worker brief template and default toolsets. " + "code=terminal+file, search/research=web+file, generic=terminal+file." + ), + }, + "acceptance_criterion": { + "type": "string", + "description": ( + "Optional stopping criterion. Loop ends early if met. " + "E.g. 'pass_rate >= 0.95', 'relevance_score >= 0.8'." + ), + }, + "evaluation_mode": { + "type": "string", + "enum": ["self_report", "llm_judge"], + "description": ( + "How to score worker output. " + "self_report: worker emits METRIC line. " + "llm_judge: supervisor scores the deliverable externally using evaluation_prompt." + ), + }, + "evaluation_prompt": { + "type": "string", + "description": ( + "For llm_judge mode: scoring rubric. " + "E.g. 'Score 0-1: does this paper list cover attention mechanisms published after 2022?'" + ), + }, + "initial_attempt": { + "type": "string", + "description": ( + "Optional starting deliverable or scaffold. " + "For code tasks: skeleton code. For research: initial outline. " + "Leave empty to let the worker start from scratch." + ), + }, + "max_iterations": { + "type": "integer", + "description": "Max improvement iterations after baseline (default: 3). Each spawns a worker.", + }, + "time_budget_sec": { + "type": "integer", + "description": "Time budget per worker invocation in seconds (default: 300).", + }, + "lattice_task_id": { + "type": "string", + "description": "Optional Lattice task ID to receive round-by-round progress comments.", + }, + }, + "required": ["topic", "deliverable", "metric_key"], + }, +} + + +# --------------------------------------------------------------------------- +# LLM bridge — wraps auxiliary_client.call_llm to match supervisor's Protocol +# --------------------------------------------------------------------------- + +class _LLMBridge: + """Adapter: auxiliary_client.call_llm → _ChatClient Protocol expected by ResearchSupervisor.""" + + def chat(self, messages: list[dict[str, str]], *, system: str | None = None) -> Any: + from agent.auxiliary_client import call_llm + + full_messages: list[dict[str, str]] = [] + if system: + full_messages.append({"role": "system", "content": system}) + full_messages.extend(messages) + + try: + resp = call_llm(messages=full_messages, max_tokens=4096) + text = resp.choices[0].message.content or "" + except Exception as exc: + logger.warning("_LLMBridge.chat failed: %s", exc) + text = "" + + return SimpleNamespace(content=text) + + +# --------------------------------------------------------------------------- +# Tool handler +# --------------------------------------------------------------------------- + +def run_research( + topic: str, + deliverable: str, + metric_key: str, + metric_direction: str = "maximize", + task_type: str = "generic", + acceptance_criterion: str = "", + evaluation_mode: str = "self_report", + evaluation_prompt: str = "", + initial_attempt: str = "", + max_iterations: int = 3, + time_budget_sec: int = 0, + lattice_task_id: Optional[str] = None, + parent_agent: Any = None, + checkpoint_dir: Optional[str] = None, +) -> str: + if parent_agent is None: + return json.dumps({"error": "run_research requires a parent_agent context."}) + + from agent.research.supervisor import ResearchSupervisor, TaskSpec + from hermes_constants import get_hermes_home + + spec = TaskSpec( + topic=topic, + deliverable=deliverable, + metric_key=metric_key, + metric_direction=metric_direction, + task_type=task_type, + acceptance_criterion=acceptance_criterion, + evaluation_mode=evaluation_mode, + evaluation_prompt=evaluation_prompt, + ) + + run_id = hashlib.sha1(f"{topic}:{time.time()}".encode()).hexdigest()[:12] + workspace = get_hermes_home() / "research-workspace" + + supervisor = ResearchSupervisor( + parent_agent=parent_agent, + workspace=workspace, + lattice_task_id=lattice_task_id, + ) + + try: + history = supervisor.run( + spec, + initial_attempt=initial_attempt, + run_id=run_id, + max_iterations=max_iterations, + time_budget_sec=time_budget_sec, + llm=_LLMBridge(), + checkpoint_dir=Path(checkpoint_dir) if checkpoint_dir else None, + ) + except Exception as exc: + logger.exception("run_research failed for run_id=%s: %s", run_id, exc) + return json.dumps({"error": str(exc), "run_id": run_id}) + + best = history.best_result + best_notes = "" + if best and best.stdout: + import re + m = re.search(r"NOTES:\s*(.+)", best.stdout) + best_notes = m.group(1).strip() if m else "" + + return json.dumps({ + "run_id": run_id, + "iterations": len(history.results), + "best_metric": best.primary_metric if best else None, + "metric_key": metric_key, + "metric_direction": metric_direction, + "best_notes": best_notes, + "workspace": str(workspace / run_id), + "learnings_file": str(workspace / run_id / "learnings.jsonl"), + }, indent=2) + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + +from tools.registry import registry, tool_error # noqa: E402 + + +def _check_research_requirements() -> bool: + try: + from agent.research.supervisor import ResearchSupervisor # noqa: F401 + return True + except ImportError: + return False + + +registry.register( + name="run_research", + toolset="research", + schema=RESEARCH_TOOL_SCHEMA, + handler=lambda args, **kw: run_research( + topic=args.get("topic", ""), + deliverable=args.get("deliverable", ""), + metric_key=args.get("metric_key", ""), + metric_direction=args.get("metric_direction", "maximize"), + task_type=args.get("task_type", "generic"), + acceptance_criterion=args.get("acceptance_criterion", ""), + evaluation_mode=args.get("evaluation_mode", "self_report"), + evaluation_prompt=args.get("evaluation_prompt", ""), + initial_attempt=args.get("initial_attempt", ""), + max_iterations=args.get("max_iterations", 3), + time_budget_sec=args.get("time_budget_sec", 0), + lattice_task_id=args.get("lattice_task_id"), + parent_agent=kw.get("parent_agent"), + checkpoint_dir=args.get("checkpoint_dir"), + ), + check_fn=_check_research_requirements, + emoji="🔬", +) diff --git a/toolsets.py b/toolsets.py index 57e226d3c082..bb1897ad30cf 100644 --- a/toolsets.py +++ b/toolsets.py @@ -52,8 +52,8 @@ "session_search", # Clarifying questions "clarify", - # Code execution + delegation - "execute_code", "delegate_task", + # Code execution + delegation + research loop + "execute_code", "delegate_task", "run_research", # Cronjob management "cronjob", # Cross-platform messaging (gated on gateway running via check_fn) @@ -113,6 +113,12 @@ "tools": ["skills_list", "skill_view", "skill_manage"], "includes": [] }, + + "research": { + "description": "Iterative self-improving research loop: run_research spawns worker subagents, scores output against a metric, and applies LLM-guided hypothesis revision across iterations (Karpathy + Autogenesis AOOR loop)", + "tools": ["run_research"], + "includes": [] + }, "browser": { "description": "Browser automation for web interaction (navigate, click, type, scroll, iframes, hold-click) with web search for finding URLs",