Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions evals/run-evals.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1030,10 +1030,11 @@ assert_tool_file_list() {
}

assert_tool_timeout_arg_recovery() {
# Loud arg validation: if the model emits a near-miss timeout key
# (TimeoutSeconds, timeout_seconds), the rejection's did-you-mean must
# steer it to the canonical _timeout_seconds within the turn — the
# command actually running is the proof of recovery.
# Spelling-tolerant meta keys: a near-miss timeout key (TimeoutSeconds,
# timeout_seconds, Timeout) now resolves onto _timeout_seconds and is
# consumed directly — no rejection round-trip needed. If the model instead
# emits the canonical key, that works too. Either way the command running is
# the proof the timeout hint was honored, not dropped.
stdout_contains '\[tool:call\] shell_execute' \
&& stdout_contains 'netclaw-timeout-eval-ok'
}
Expand Down
29 changes: 19 additions & 10 deletions feeds/skills/.system/files/netclaw-operations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: netclaw-operations
description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance."
metadata:
author: netclaw
version: "2.14.0"
version: "2.15.0"
---

# Netclaw Operations
Expand Down Expand Up @@ -310,15 +310,24 @@ session context on every turn.

## Tool argument validation

Tool argument names are validated strictly — unrecognized keys reject the call
before execution with a `did you mean '<canonical>'?` suggestion and the list
of valid argument names. Meta keys are exact-match: `_timeout_seconds` and
`_background` (a leading underscore, snake_case). `TimeoutSeconds`,
`timeout_seconds`, or `_timeoutSeconds` are rejected, never silently dropped.
Values must parse as their declared type: `_timeout_seconds: "1200ms"` or
`_background: "yes"` rejects the call instead of silently using defaults. When
a call is rejected this way the tool did NOT run — fix the argument and
re-issue once; do not retry the same shape.
Prefer the canonical argument names exactly as a tool declares them, and the
canonical meta keys `_rationale`, `_timeout_seconds`, `_background` (leading
underscore, snake_case). Recognition is spelling-tolerant so a near-miss is
consumed rather than dropped: declared params fold case/punctuation, and the
meta keys also accept the underscore-dropped/cased/shortened forms
(`TimeoutSeconds`, `timeout_seconds`, `Timeout` → the timeout hint; `Rationale`,
`Background` likewise). The supplied value is always *used* — never silently
defaulted.

Three things are still rejected loudly, and when rejected the tool did NOT run —
fix and re-issue once, do not retry the same shape:

- **Unknown keys** — a key that matches no parameter and no meta field rejects
with a `did you mean '<canonical>'?` suggestion and the list of valid names.
- **Invalid values** — a value that cannot parse as its type (`_timeout_seconds:
"1200ms"`, `_background: "yes"`) rejects instead of falling back to a default.
- **Ambiguous meta spelling** — supplying two keys that map to the same meta
field (e.g. both `_timeout_seconds` and `TimeoutSeconds`) rejects; send one.

## Large tool output

Expand Down
62 changes: 62 additions & 0 deletions src/Netclaw.Actors.Tests/Protocol/ChatMessageConverterTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,68 @@ private static byte[] SmallPng(int size = 16)
return data.ToArray();
}

[Fact]
public void FromAiMessage_with_interpreter_persists_schema_aware_meta()
{
var aiMsg = new AiChatMessage(AiChatRole.Assistant, new List<AIContent>
{
new FunctionCallContent("c1", "shell_execute", new Dictionary<string, object?>
{
["Command"] = "ls",
["TimeoutSeconds"] = 600 // ChatGPT-style near-miss
})
});

// With the schema-aware interpreter (what LlmSessionActor passes), the
// near-miss is stripped from persisted args and captured in MetaJson —
// so recorded history matches what the runtime actually executes.
var withInterp = ChatMessageConverter.FromAiMessage(aiMsg, interpretToolCall: tc =>
ToolCallMeta.ExtractFrom(tc.Arguments, ToolArgumentHelper.ResolveMetaField));
var call = Assert.Single(withInterp.ToolCalls);
Assert.DoesNotContain("TimeoutSeconds", call.ArgumentsJson);
Assert.NotNull(call.MetaJson);
Assert.Contains("600", call.MetaJson!);

// Without an interpreter (schema-blind default = exact), the near-miss is
// left raw and no meta is captured — the safe fallback for callers/replay.
var exact = ChatMessageConverter.FromAiMessage(aiMsg);
var exactCall = Assert.Single(exact.ToolCalls);
Assert.Contains("TimeoutSeconds", exactCall.ArgumentsJson);
Assert.Null(exactCall.MetaJson);
}

[Fact]
public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound()
{
// Persist a near-miss call: the meta key is stripped into MetaJson.
var persisted = ChatMessageConverter.FromAiMessage(
new AiChatMessage(AiChatRole.Assistant, new List<AIContent>
{
new FunctionCallContent("c1", "shell_execute", new Dictionary<string, object?>
{
["Command"] = "long-task",
["TimeoutSeconds"] = 1800
})
}),
interpretToolCall: tc => ToolCallMeta.ExtractFrom(tc.Arguments, ToolArgumentHelper.ResolveMetaField));
var stored = Assert.Single(persisted.ToolCalls);
Assert.DoesNotContain("TimeoutSeconds", stored.ArgumentsJson);

// Re-drive reconstruction re-injects the canonical key, so extraction
// reapplies the 1800s hint on re-dispatch (regression guard for the
// previously-dropped hint on post-approval re-drive).
var redriven = ChatMessageConverter.ToAiMessage(persisted, reinjectMeta: true);
var redrivenCall = redriven.Contents.OfType<FunctionCallContent>().Single();
Assert.True(redrivenCall.Arguments!.ContainsKey("_timeout_seconds"));
var (meta, _) = ToolCallMeta.ExtractFrom(redrivenCall.Arguments, ToolArgumentHelper.ResolveMetaField);
Assert.Equal(1800, meta?.TimeoutHintSeconds);

// Outbound provider history (the default) must NOT carry meta keys.
var outbound = ChatMessageConverter.ToAiMessage(persisted);
var outboundCall = outbound.Contents.OfType<FunctionCallContent>().Single();
Assert.False(outboundCall.Arguments?.ContainsKey("_timeout_seconds") ?? false);
}

private sealed class TempSessionDir : IDisposable
{
public string Path { get; } = System.IO.Path.Combine(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public void Extract_WithAllMetaFields_ReturnsMetaAndCleanArgs()
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.Equal("running tests", meta!.Rationale);
Expand All @@ -48,7 +48,7 @@ public void Extract_WithOnlyRationale_ReturnsMetaWithRationaleOnly()
};
var tc = new FunctionCallContent("call-1", "web_search", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.Equal("searching docs", meta!.Rationale);
Expand All @@ -65,7 +65,7 @@ public void Extract_WithNoMetaFields_ReturnsNullMeta()
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.Null(meta);
Assert.Same(tc, cleaned);
Expand All @@ -76,7 +76,7 @@ public void Extract_WithNullArguments_ReturnsNullMeta()
{
var tc = new FunctionCallContent("call-1", "shell_execute", null);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.Null(meta);
Assert.Same(tc, cleaned);
Expand All @@ -91,7 +91,7 @@ public void Extract_HandlesJsonElementValues()
args[prop.Name] = prop.Value;

var tc = new FunctionCallContent("call-1", "shell_execute", args);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc);
var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.Equal("from json", meta!.Rationale);
Expand All @@ -109,7 +109,7 @@ public void Extract_TimeoutSeconds_ZeroNotTreatedAsMeta()
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, _) = ToolCallMetaExtractor.Extract(tc);
var (meta, _) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

// Zero timeout is not meaningful — should not produce meta
Assert.Null(meta);
Expand All @@ -133,7 +133,7 @@ public void Extract_TimeoutAlone_DoesNotTriggerBackground()
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, _) = ToolCallMetaExtractor.Extract(tc);
var (meta, _) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.False(meta!.Background);
Expand All @@ -150,9 +150,118 @@ public void Extract_BackgroundFalse_DoesNotSetBackground()
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, _) = ToolCallMetaExtractor.Extract(tc);
var (meta, _) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.False(meta!.Background);
}

// ── ChatGPT-style meta naming (Qwen) is consumed, not dropped ──

[Fact]
public void Extract_MisnamedMetaFields_ConsumedAndStripped()
{
// The names Qwen emits: underscore dropped, capitalized, and the
// shortened "Timeout". All resolve onto the canonical fields and are
// removed from the args the tool binder sees.
var args = new Dictionary<string, object?>
{
["Command"] = "dotnet test",
["Rationale"] = "running tests",
["TimeoutSeconds"] = "1200",
["Background"] = true
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.Equal("running tests", meta!.Rationale);
Assert.Equal(1200, meta.TimeoutHintSeconds);
Assert.True(meta.Background);

var cleanArgs = (IDictionary<string, object?>)cleaned.Arguments!;
Assert.Contains("Command", cleanArgs);
Assert.DoesNotContain("Rationale", cleanArgs);
Assert.DoesNotContain("TimeoutSeconds", cleanArgs);
Assert.DoesNotContain("Background", cleanArgs);
}

[Fact]
public void Extract_BareTimeout_ResolvesToTimeoutHint()
{
var args = new Dictionary<string, object?>
{
["Command"] = "sleep 5",
["Timeout"] = 600
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc, ToolArgumentHelper.ResolveMetaField);

Assert.NotNull(meta);
Assert.Equal(600, meta!.TimeoutHintSeconds);
Assert.DoesNotContain("Timeout", (IDictionary<string, object?>)cleaned.Arguments!);
}

// ── Exact resolver (persistence / schema-blind default): near-misses are NOT meta ──
// The executor passes a schema-aware resolver (covered via the executor + MCP tests);
// the default exact resolver is what persistence and schema-blind callers use.

[Fact]
public void Extract_ExactResolver_DoesNotConsumeNearMissNames()
{
var args = new Dictionary<string, object?>
{
["Command"] = "ls",
["TimeoutSeconds"] = 1200,
["Timeout"] = 600
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc); // default = exact resolver

Assert.Null(meta);
Assert.Same(tc, cleaned); // near-miss keys retained, nothing stripped
}

[Fact]
public void Extract_ExactResolver_StillConsumesCanonicalMetaKeys()
{
var args = new Dictionary<string, object?>
{
["Command"] = "ls",
["_timeout_seconds"] = 300
};
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, cleaned) = ToolCallMetaExtractor.Extract(tc); // default = exact resolver

Assert.NotNull(meta);
Assert.Equal(300, meta!.TimeoutHintSeconds);
Assert.DoesNotContain("_timeout_seconds", (IDictionary<string, object?>)cleaned.Arguments!);
}

[Fact]
public void Extract_BindsEveryCanonicalMetaField()
{
// Guards ExtractFrom's `default:` throw against drift: every name in
// MetaFieldNames must have a matching extraction case, so a future meta
// field added without one fails here at CI time, not at runtime.
foreach (var name in ToolCallMeta.MetaFieldNames)
{
object? value = name switch
{
"_timeout_seconds" => 30,
"_background" => true,
_ => "because"
};
var args = new Dictionary<string, object?> { ["Command"] = "ls", [name] = value };
var tc = new FunctionCallContent("call-1", "shell_execute", args);

var (meta, _) = ToolCallMetaExtractor.Extract(tc);

Assert.NotNull(meta); // recognized + bound, no default-throw
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,12 @@ public async Task Spawn_agent_subagent_approval_uses_parent_authority_and_resume
"shell_execute",
new Dictionary<string, object?>
{
["Command"] = "git push origin main"
["Command"] = "git push origin main",
// Per-call timeout hint on the sub-agent path: the sub-agent
// loop must extract this via the shared executor seam and apply
// it to the tool context (it previously skipped extraction and
// silently dropped the hint).
["_timeout_seconds"] = 1800
})
];

Expand Down Expand Up @@ -348,6 +353,10 @@ await sessionManager.Ask<CommandAck>(new SendUserMessage
Assert.NotNull(_recordingShellTool);
Assert.True(_recordingShellTool!.WasCalled);
Assert.Equal(TrustAudience.Personal, _recordingShellTool.LastContext?.Audience);

// The sub-agent extracted the meta timeout hint and applied it to the
// tool context (regression guard for the previously-dropped hint).
Assert.Equal(1800, _recordingShellTool.LastContext?.RequestedTimeoutSeconds);
}

[Fact]
Expand Down
Loading
Loading