feat: add offload_job tool — systemd-run isolation for heavy workloads

tests/tools/test_offload_job_tool.py

@@ -0,0 +1,135 @@
+import json
+import time
+
+from tools.registry import registry
+from tools import offload_job_tool as offload
+
+
+def _decode(raw):
+    return json.loads(raw)
+
+
+def _wait_for_terminal_status(job_id, wanted, timeout=5):
+    deadline = time.time() + timeout
+    last = None
+    while time.time() < deadline:
+        last = _decode(offload.offload_job_tool("status", job_id=job_id))
+        if last["job"]["status"] in wanted:
+            return last
+        time.sleep(0.05)
+    return last
+
+
+def test_offload_job_fallback_runs_command_and_tails_log(tmp_path, monkeypatch):
+    monkeypatch.setattr(offload, "_systemd_run_available", lambda: False)
+
+    result = _decode(
+        offload.offload_job_tool(
+            "start",
+            command="echo hello-offload",
+            workdir=str(tmp_path),
+            label="unit",
+        )
+    )
+
+    assert result["success"] is True
+    job = result["job"]
+    assert job["runner"] == "subprocess"
+    assert job["status"] == "running"
+    assert job["workdir"] == str(tmp_path)
+
+    status = _wait_for_terminal_status(job["job_id"], {"succeeded", "failed", "unknown"})
+    assert status["job"]["status"] == "succeeded"
+    assert status["job"]["exit_code"] == 0
+
+    tail = _decode(offload.offload_job_tool("tail", job_id=job["job_id"], tail_lines=20))
+    assert tail["success"] is True
+    assert "hello-offload" in tail["tail"]
+
+
+def test_offload_job_list_returns_newest_first(tmp_path, monkeypatch):
+    monkeypatch.setattr(offload, "_systemd_run_available", lambda: False)
+    first = _decode(offload.offload_job_tool("start", command="echo first", workdir=str(tmp_path), label="first"))["job"]
+    second = _decode(offload.offload_job_tool("start", command="echo second", workdir=str(tmp_path), label="second"))["job"]
+
+    _wait_for_terminal_status(first["job_id"], {"succeeded", "failed", "unknown"})
+    _wait_for_terminal_status(second["job_id"], {"succeeded", "failed", "unknown"})
+
+    listed = _decode(offload.offload_job_tool("list", limit=10))
+    ids = [job["job_id"] for job in listed["jobs"]]
+    assert ids.index(second["job_id"]) < ids.index(first["job_id"])
+
+
+def test_offload_job_cancel_unknown_job_returns_structured_error():
+    result = _decode(offload.offload_job_tool("cancel", job_id="missing-job"))
+    assert result["success"] is False
+    assert "Unknown offload job" in result["error"]
+
+
+def test_offload_job_cancel_running_fallback_job(tmp_path, monkeypatch):
+    monkeypatch.setattr(offload, "_systemd_run_available", lambda: False)
+
+    result = _decode(
+        offload.offload_job_tool(
+            "start",
+            command="sleep 30",
+            workdir=str(tmp_path),
+            label="cancel",
+        )
+    )
+    assert result["success"] is True
+
+    cancelled = _decode(offload.offload_job_tool("cancel", job_id=result["job"]["job_id"]))
+    assert cancelled["success"] is True
+    assert cancelled["job"]["status"] == "cancelled"
+
+    status = _decode(offload.offload_job_tool("status", job_id=result["job"]["job_id"]))
+    assert status["job"]["status"] == "cancelled"
+
+
+def test_offload_job_uses_systemd_when_available(tmp_path, monkeypatch):
+    calls = []
+
+    def fake_run(cmd, **kwargs):
+        calls.append(cmd)
+
+        class Result:
+            returncode = 0
+            stdout = "Running as unit hermes-offload-demo.service"
+            stderr = ""
+
+        return Result()
+
+    monkeypatch.setattr(offload, "_systemd_run_available", lambda: True)
+    monkeypatch.setattr(offload.subprocess, "run", fake_run)
+
+    result = _decode(
+        offload.offload_job_tool(
+            "start",
+            command="echo systemd",
+            workdir=str(tmp_path),
+            memory_max="32G",
+            cpu_quota="800%",
+            label="systemd",
+        )
+    )
+
+    assert result["success"] is True
+    job = result["job"]
+    assert job["runner"] == "systemd"
+    assert job["unit_name"].startswith("hermes-offload-")
+    flat = " ".join(calls[0])
+    assert "MemoryMax=32G" in flat
+    assert "CPUQuota=800%" in flat
+    assert f"WorkingDirectory={tmp_path}" in flat
+
+
+def test_offload_job_registered_in_terminal_toolset():
+    import toolsets
+
+    entry = registry.get_entry("offload_job")
+    assert entry is not None
+    assert entry.toolset == "terminal"
+    assert "offload" in entry.schema["name"]
+    assert "offload_job" in toolsets.resolve_toolset("terminal")
+    assert "offload_job" in toolsets.resolve_toolset("hermes-gateway")

tools/delegate_tool.py

@@ -1048,6 +1048,8 @@ def _child_thinking(text: str) -> None:
         provider_sort=parent_agent.provider_sort,
         tool_progress_callback=child_progress_cb,
         iteration_budget=None,  # fresh budget per subagent
+        fallback_model=getattr(parent_agent, "_fallback_chain", None)
+        or getattr(parent_agent, "_fallback_model", None),
     )
     child._print_fn = getattr(parent_agent, "_print_fn", None)
     # Set delegation depth so children can't spawn grandchildren

tools/hbackup/.gitignore

@@ -0,0 +1,3 @@
+/target
+Cargo.lock
+*.tar.zst

tools/hbackup/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "hbackup"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+clap = { version = "4", features = ["derive"] }
+zstd = "0.13"
+tar = "0.4"
+walkdir = "2"
+indicatif = "0.17"
+chrono = "0.4"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+anyhow = "1"
+toml = "0.8"
+dirs = "5"

tools/hbackup/README.md

@@ -0,0 +1,120 @@
+# hbackup
+
+Backup and restore tool for Hermes Agent + OpenClaw installations.
+
+## Features
+
+- **Incremental-aware discovery**: Backs up `~/work/hermes-agent`, `~/work/openclaw`, `~/.hermes`, `~/.openclaw`, systemd units, and more
+- **SQLite-safe**: Uses `sqlite3 .backup` for live database copies
+- **Compressed archives**: `tar.zst` format with zstd compression
+- **Google Drive upload**: Via `rclone` integration
+- **Cross-platform home dir**: Uses `dirs` crate — no hardcoded paths
+- **Auto workspace discovery**: Scans `~/.openclaw/workspace*` dynamically
+
+## Install
+
+```bash
+cd tools/hbackup
+cargo build --release
+cp target/release/hbackup ~/.local/bin/
+```
+
+## Usage
+
+### Backup
+
+```bash
+# Create backup
+hbackup backup
+
+# Dry run
+hbackup backup --dry-run
+
+# Custom output
+hbackup backup -o /mnt/external/my-backup.tar.zst
+
+# Exclude patterns
+hbackup backup -x target -x .git
+```
+
+### Restore
+
+```bash
+# Restore from archive
+hbackup restore ~/backups/hermes-openclaw-backup-20260101-120000.tar.zst
+
+# Dry run
+hbackup restore --dry-run ~/backups/...
+
+# Force overwrite
+hbackup restore --force ~/backups/...
+```
+
+### List Backups
+
+```bash
+hbackup list
+```
+
+### Upload to Google Drive
+
+```bash
+# Setup guide
+hbackup setup drive
+
+# Upload to Google Drive (requires rclone configured)
+hbackup upload --drive ~/backups/hermes-openclaw-backup-*.tar.zst
+
+# Custom remote name / folder
+hbackup upload --drive --drive-remote mygdrive --drive-folder backups/2026 ~/backups/...
+```
+
+### Auto (backup + upload)
+
+```bash
+# Create backup then upload to configured destination
+hbackup auto
+```
+
+Configure `~/.config/hbackup/config.toml`:
+
+```toml
+[upload]
+destination = "user@server:/backups/"  # scp/rsync
+drive_remote = "gdrive"                  # Google Drive remote name
+drive_folder = "backups/hermes"          # Google Drive folder
+```
+
+## Cron Setup
+
+```bash
+# Daily backup at 3 AM
+0 3 * * * /home/USER/.local/bin/hbackup auto >> /home/USER/.local/state/hermes/hbackup-cron.log 2>&1
+```
+
+## Config File
+
+`~/.config/hbackup/config.toml`:
+
+```toml
+[upload]
+# For scp/rsync upload:
+method = "scp"  # or "rsync"
+destination = "user@backup-server:/backups/"
+
+# For Google Drive upload:
+drive_remote = "gdrive"
+drive_folder = "backups/hermes"
+```
+
+## Open Source Notes
+
+This tool was extracted from a personal setup. Before publishing:
+
+- All hardcoded paths removed (uses `dirs::home_dir()`)
+- Workspace names auto-discovered from `~/.openclaw/workspace*`
+- No user-specific paths remain
+
+## License
+
+MIT