Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent/scaffolde_capability_prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def build_scaffolde_capabilities_prompt(valid_tool_names: Iterable[str]) -> str:
return ""
lines = [
"# Available Scaffolde capabilities",
"Use the native `scaffolde_capability` tool for Scaffolde-owned capabilities and PAI accounts. Routing precedence: explicit user override > matching Scaffolde authority/account > generic skills.",
"Use the native `scaffolde_capability` tool for declared Scaffolde-owned capabilities. Routing precedence: explicit user override > matching Scaffolde authority > generic skills.",
"If a declared capability fails, describe it as `capability_degraded` rather than claiming the mailbox/service is unavailable.",
]
for cap in registry.capabilities.values():
Expand Down
60 changes: 45 additions & 15 deletions tests/tools/test_scaffolde_capability_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ def _descriptor(home: Path, *, runner: Path | None = None, risk="read", extra=No
runner_arg = str(runner)
desc = {
"version": 1,
"id": "scaffolde.gmail.pai",
"id": "scaffolde.gmail",
"tool_name": "scaffolde_gmail",
"authority": "scaffolde",
"kind": "gmail",
"description": "Canonical Scaffolde Gmail capability for PAI account email.",
"triggers": ["pai gmail", "scaffolde gmail", "email sarah"],
"description": "Canonical Scaffolde Gmail capability.",
"triggers": ["scaffolde gmail", "email sarah"],
"entrypoint": {
"command": "bun",
"args": [runner_arg],
Expand Down Expand Up @@ -126,7 +126,7 @@ def test_loader_reports_absent_valid_malformed_and_degraded_status(hermes_home):
_write_registry(hermes_home, _descriptor(hermes_home))
valid = load_capability_registry()
assert valid.status == "valid"
assert list(valid.capabilities) == ["scaffolde.gmail.pai"]
assert list(valid.capabilities) == ["scaffolde.gmail"]

(hermes_home / "scaffolde" / "capabilities.json").write_text("{not-json", encoding="utf-8")
malformed = load_capability_registry()
Expand All @@ -140,6 +140,19 @@ def test_loader_reports_absent_valid_malformed_and_degraded_status(hermes_home):
assert degraded.errors[0]["code"] == "capability_schema"


def test_registry_rejects_the_retired_gmail_capability_identity(hermes_home):
from tools.scaffolde_capabilities import load_capability_registry

retired = _descriptor(hermes_home)
retired["id"] = "scaffolde.gmail.pai"
_write_registry(hermes_home, retired)

registry = load_capability_registry()
assert registry.status == "degraded"
assert "scaffolde.gmail.pai" not in registry.capabilities
assert any(error["code"] == "retired_capability" for error in registry.errors)


def test_schema_path_and_secret_rejection_fail_closed(hermes_home):
from tools.scaffolde_capabilities import load_capability_registry

Expand Down Expand Up @@ -225,17 +238,34 @@ def test_native_tool_visible_in_default_schema_even_without_valid_registry(herme
assert "scaffolde_capability" in names


def test_tool_contract_advertises_only_the_canonical_gmail_identity(hermes_home):
from tools.scaffolde_capability_tool import SCAFFOLDE_CAPABILITY_SCHEMA

schema = json.dumps(SCAFFOLDE_CAPABILITY_SCHEMA)
assert "scaffolde.gmail" in schema
assert "scaffolde.gmail.pai" not in schema

_write_registry(hermes_home, _descriptor(hermes_home))
result = _call_tool(
action="invoke",
capability_id="scaffolde.gmail.pai",
operation="search",
arguments={"query": "Sarah"},
)
assert result["error_type"] == "unknown_capability"


def test_argument_validation_no_shell_and_env_allowlist_for_read_operation(hermes_home, monkeypatch):
runner = _write_fake_runner(hermes_home)
_write_registry(hermes_home, _descriptor(hermes_home, runner=runner))
monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/tmp/should-not-leak")
monkeypatch.setenv("SECRET_TOKEN", "should-not-inherit")

extra = _call_tool(action="invoke", capability_id="scaffolde.gmail.pai", operation="search", arguments={"query": "Sarah; touch /tmp/pwned", "limit": 2, "extra": "nope"})
extra = _call_tool(action="invoke", capability_id="scaffolde.gmail", operation="search", arguments={"query": "Sarah; touch /tmp/pwned", "limit": 2, "extra": "nope"})
assert extra["status"] == "error"
assert extra["error_type"] == "invalid_arguments"

result = _call_tool(action="invoke", capability_id="scaffolde.gmail.pai", operation="search", arguments={"query": "Sarah; touch /tmp/pwned", "limit": 2})
result = _call_tool(action="invoke", capability_id="scaffolde.gmail", operation="search", arguments={"query": "Sarah; touch /tmp/pwned", "limit": 2})
assert result["status"] == "ok"
child = json.loads(result["stdout"])
assert child["argv"] == ["search", "--query", "Sarah; touch /tmp/pwned", "--limit", "2", "--untrusted-literal", "Sarah; touch /tmp/pwned"]
Expand All @@ -258,14 +288,14 @@ def deny(tool_name, reason, *, rule_key="", approval_callback=None):
return {"approved": False, "message": "denied"}

monkeypatch.setattr("tools.approval.request_tool_approval", deny)
result = _call_tool(action="invoke", capability_id="scaffolde.gmail.pai", operation=operation, arguments={})
result = _call_tool(action="invoke", capability_id="scaffolde.gmail", operation=operation, arguments={})
assert result["status"] == "error"
assert result["error_type"] == "approval_required"
assert len(calls) == 1
assert calls[0][0] == "scaffolde_capability"
assert operation in calls[0][1]
assert "Arguments: {}" in calls[0][1]
assert calls[0][2].startswith(f"scaffolde:scaffolde.gmail.pai:{operation}:")
assert calls[0][2].startswith(f"scaffolde:scaffolde.gmail:{operation}:")
assert len(calls[0][2].rsplit(":", 1)[1]) == 12


Expand All @@ -288,15 +318,15 @@ def deny_after_inspection(_tool, reason, *, rule_key="", approval_callback=None)
body = "full-body-" * 300
result = _call_tool(
action="invoke",
capability_id="scaffolde.gmail.pai",
capability_id="scaffolde.gmail",
operation="send",
arguments={"to": "person@example.com", "body": body},
)
assert result["error_type"] == "approval_required"
assert observed["payload"]["body"] == body
assert observed["mode"] == 0o600
assert "Exact payload SHA256=" in observed["reason"]
assert observed["rule_key"].startswith("scaffolde:scaffolde.gmail.pai:send:")
assert observed["rule_key"].startswith("scaffolde:scaffolde.gmail:send:")
assert not observed["path"].exists()


Expand All @@ -315,8 +345,8 @@ def test_prompt_routing_block_precedes_skill_guidance(hermes_home):

prompt = build_scaffolde_capabilities_prompt({"scaffolde_capability", "skill_view"})
assert "Available Scaffolde capabilities" in prompt
assert "explicit user override > matching Scaffolde authority/account > generic skills" in prompt
assert "scaffolde.gmail.pai" in prompt
assert "explicit user override > matching Scaffolde authority > generic skills" in prompt
assert "scaffolde.gmail" in prompt
assert "capability_degraded" in prompt


Expand All @@ -327,7 +357,7 @@ def test_incident_regression_fake_scaffolde_gmail_succeeds_without_generic_invoc
encoding="utf-8",
)
_write_registry(hermes_home, _descriptor(hermes_home, runner=runner))
result = _call_tool(action="invoke", capability_id="scaffolde.gmail.pai", operation="search", arguments={"query": "Sarah", "limit": 1})
result = _call_tool(action="invoke", capability_id="scaffolde.gmail", operation="search", arguments={"query": "Sarah", "limit": 1})
assert result["status"] == "ok"
assert json.loads(result["stdout"])["from"] == "Sarah"
assert json.loads(result["stdout"])["generic_invoked"] is False
Expand All @@ -338,13 +368,13 @@ def test_runtime_failures_are_capability_degraded(hermes_home):

failing = _write_fake_runner(hermes_home, exit_code=9)
_write_registry(hermes_home, _descriptor(hermes_home, runner=failing))
failed = invoke_capability("scaffolde.gmail.pai", "search", {"query": "Sarah", "limit": 1})
failed = invoke_capability("scaffolde.gmail", "search", {"query": "Sarah", "limit": 1})
assert failed["status"] == "capability_degraded"
assert failed["error_type"] == "nonzero_exit"

slow = hermes_home / "slow.ts"
slow.write_text("import time\ntime.sleep(5)\n", encoding="utf-8")
_write_registry(hermes_home, _descriptor(hermes_home, runner=slow))
timed_out = invoke_capability("scaffolde.gmail.pai", "search", {"query": "Sarah", "limit": 1}, timeout=1)
timed_out = invoke_capability("scaffolde.gmail", "search", {"query": "Sarah", "limit": 1}, timeout=1)
assert timed_out["status"] == "capability_degraded"
assert timed_out["error_type"] == "timeout"
3 changes: 3 additions & 0 deletions tools/scaffolde_capabilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
MAX_STDERR_CHARS = 16_000
DEFAULT_TIMEOUT_SECONDS = 60
_ALLOWED_PARAM_TYPES = {"string", "integer", "boolean"}
_RETIRED_CAPABILITY_IDS = frozenset({"scaffolde.gmail.pai"})
_ALLOWED_RISKS = {"read", "write"}
_SECRET_NAME_RE = re.compile(r"(TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|CREDENTIAL)", re.I)
_SECRET_VALUE_RE = re.compile(
Expand Down Expand Up @@ -128,6 +129,8 @@ def _validate_capability(
if not isinstance(raw, dict):
return None, None, [_err("capability_schema", "capability must be an object")]
cid = _validate_string(raw.get("id"), "id", errors, None)
if cid in _RETIRED_CAPABILITY_IDS:
errors.append(_err("retired_capability", f"capability id is retired: {cid}", cid))
Comment on lines +132 to +133

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep legacy descriptors valid during the consumer-first rollout

When an existing installation still has the previously supported scaffolde.gmail.pai descriptor, this new validation error makes the registry degraded; invoke_capability() then returns capability_degraded before looking up any operation, and the capability prompt is omitted. Because this commit is intended to land before the coordinated producer migration, upgrading Hermes immediately disables Gmail for every installation whose producer has not yet rewritten the registry. Accept or translate the legacy ID during the transition, and retire it only after producers have migrated.

Useful? React with 👍 / 👎.

if raw.get("version") != 1:
errors.append(_err("capability_version", "capability.version must be 1", cid or None))
for field in ("tool_name", "authority", "kind", "description"):
Expand Down
8 changes: 4 additions & 4 deletions tools/scaffolde_capability_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@
SCAFFOLDE_CAPABILITY_SCHEMA = {
"name": "scaffolde_capability",
"description": (
"Authoritative native runtime for Scaffolde-owned capabilities and PAI accounts. "
"Authoritative native runtime for Scaffolde-owned capabilities. "
"Use action=status to inspect descriptor health, action=list to see available capabilities/operations, "
"and action=invoke with capability_id, operation, and arguments for execution. Prefer this over generic "
"Gmail/Google clients for Scaffolde-owned or pai@scaffolde.ai requests unless the user explicitly overrides."
"and action=invoke with capability_id, operation, and arguments for execution. Prefer this over "
"generic clients when a matching Scaffolde capability is declared unless the user explicitly overrides."
),
"parameters": {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["status", "list", "invoke"], "description": "Operation on the Scaffolde capability registry."},
"capability_id": {"type": "string", "description": "Capability id, e.g. scaffolde.gmail.pai. Required for invoke."},
"capability_id": {"type": "string", "description": "Capability id, e.g. scaffolde.gmail. Required for invoke."},
"operation": {"type": "string", "description": "Descriptor operation to invoke. Required for invoke."},
"arguments": {"type": "object", "description": "Exact descriptor-declared arguments for the operation; no extra keys."},
},
Expand Down
Loading