Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions evals/run-evals.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,27 @@ stdout_json_tool_call_arguments() {
"$STDOUT_FILE" 2>/dev/null
}

stdout_json_all_tool_calls_have_rationale() {
local minimum_calls="${1:-1}"
jq -e --argjson minimum_calls "$minimum_calls" '
(.toolCalls // []) as $calls
| ($calls | length) >= $minimum_calls
and all($calls[];
try (
(.argumentsJson | fromjson | ._rationale) as $rationale
| ($rationale | type) == "string"
and ($rationale | length) > 0
) catch false)
' "$STDOUT_FILE" >/dev/null 2>&1
}

stdout_json_tool_call_sequence_matches() {
local expected_json="$1"
jq -e --argjson expected "$expected_json" '
[(.toolCalls // [])[] | .toolName] == $expected
' "$STDOUT_FILE" >/dev/null 2>&1
}

stdout_skill_file_read_called() {
grep -aiE '^\[tool:call\] file_read\(' "$STDOUT_FILE" 2>/dev/null \
| grep -qi 'SKILL\.md'
Expand Down Expand Up @@ -1307,6 +1328,13 @@ assert_tool_local_repository_search() {
&& stdout_response_contains 'match.txt'
}

assert_tool_rationale_contract() {
stdout_json_envelope_valid \
&& stdout_json_tool_call_sequence_matches \
'["file_list","list_reminders","file_read","skill_load"]' \
&& stdout_json_all_tool_calls_have_rationale 4
}

assert_tool_timestamped_webhook() {
stdout_tool_called 'set_webhook' \
&& stdout_contains 'HmacTimestamped' \
Expand Down Expand Up @@ -2098,6 +2126,9 @@ run_all() {
run_case tool_local_repository_search "local repository search uses shell, not web_search" \
"Search recursively under /home/netclaw/.netclaw/workspaces/file-tool-selection/search-target for the exact text local-search-eval-token and tell me which file contains it."

run_case --json tool_rationale_contract "all calls retain rationales across two parallel tool iterations" \
"Use exactly two tool stages. First, call file_list on /home/netclaw/.netclaw/workspaces and list_reminders in one parallel batch. After both results return, call file_read on /home/netclaw/.netclaw/workspaces/netclaw-eval-largefile.txt for lines 1 through 3 and skill_load for netclaw-operations in one parallel batch. Use all four tools, then summarize the results."

run_case tool_timestamped_webhook "set_webhook called with Stripe timestamp verification" \
"Create a public inbound webhook route named stripe-events for Stripe. Use secret eval-whsec-123 and have it summarize each payment event."

Expand Down
7 changes: 6 additions & 1 deletion feeds/skills/.system/files/netclaw-operations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: netclaw-operations
description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance."
metadata:
author: netclaw
version: "2.52.0"
version: "2.53.0"
---

# Netclaw Operations
Expand Down Expand Up @@ -98,6 +98,11 @@ meta keys also accept the underscore-dropped/cased/shortened forms
`Background` likewise). The supplied value is always *used* — never silently
defaulted.

Every tool call requires a non-empty `_rationale` string. State the call intent
and reason in one sentence. Apply this rule to each parallel call and each later
tool iteration. If a correction reports a missing rationale, fix every call
before the retry.

Three things are still rejected loudly, and when rejected the tool did NOT run —
fix and re-issue once, do not retry the same shape:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ public void FromAiMessage_with_interpreter_persists_schema_aware_meta()
}

[Fact]
public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound()
public void ToAiMessage_replays_rationale_and_limits_execution_hints_to_redrive()
{
// Persist a near-miss call: the meta key is stripped into MetaJson.
var persisted = ChatMessageConverter.FromAiMessage(
Expand All @@ -646,6 +646,7 @@ public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound(
new FunctionCallContent("c1", "shell_execute", new Dictionary<string, object?>
{
["Command"] = "long-task",
["_rationale"] = "Run the long task for the user.",
["TimeoutSeconds"] = 1800
})
}),
Expand All @@ -659,12 +660,14 @@ public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound(
var redriven = ChatMessageConverter.ToAiMessage(persisted, reinjectMeta: true);
var redrivenCall = redriven.Contents.OfType<FunctionCallContent>().Single();
Assert.True(redrivenCall.Arguments!.ContainsKey("_timeout_seconds"));
Assert.Equal("Run the long task for the user.", redrivenCall.Arguments["_rationale"]);
var (meta, _) = ToolCallMeta.ExtractFrom(redrivenCall.Arguments, ToolArgumentHelper.ResolveMetaField);
Assert.Equal(1800, meta?.TimeoutHintSeconds);

// Outbound provider history (the default) must NOT carry meta keys.
// Outbound history keeps the required rationale. It omits execution hints.
var outbound = ChatMessageConverter.ToAiMessage(persisted);
var outboundCall = outbound.Contents.OfType<FunctionCallContent>().Single();
Assert.Equal("Run the long task for the user.", outboundCall.Arguments!["_rationale"]);
Assert.False(outboundCall.Arguments?.ContainsKey("_timeout_seconds") ?? false);
}

Expand Down
18 changes: 18 additions & 0 deletions src/Netclaw.Actors.Tests/Protocol/SerializationRoundTripTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,24 @@ public void TurnRecorded_round_trips_preserving_value_object_source_ids()
Assert.Equal(original.SourceBackgroundJobId, result.SourceBackgroundJobId);
}

[Fact]
public void Tool_result_failure_code_round_trips_through_the_transport_DTO()
{
var original = new ToolResultOutput
{
SessionId = new SessionId("test/wire"),
CallId = new ToolCallId("call-rejected"),
ToolName = new ToolName("search"),
Result = "The tool was not executed.",
FailureCode = "invalid_rationale"
};

var result = Assert.IsType<ToolResultOutput>(
SessionOutputDtoMapper.FromDto(SessionOutputDtoMapper.ToDto(original)));

Assert.Equal("invalid_rationale", result.FailureCode);
}

[Fact]
public void TurnRecorded_round_trips_with_null_source_ids()
{
Expand Down
122 changes: 122 additions & 0 deletions src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,128 @@ await sessionManager.Ask<CommandAck>(new SendUserMessage
await subscriber.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), cancellationToken: TestContext.Current.CancellationToken);
}

[Fact]
public async Task Repeated_invalid_rationales_disable_tools_after_three_iterations()
{
_fakeToolExecutor.RequireRationale = true;
for (var index = 1; index <= 3; index++)
{
_fakeChatClient.PlannedResponses.Enqueue(
[
new FunctionCallContent($"call-{index}", "load_tool",
new Dictionary<string, object?>
{
["Name"] = "browser_chrome_devtools/navigate_page"
})
]);
}

var sessionId = new SessionId("signalr/invalid-rationale-limit");
var sessionManager = ActorRegistry.Get<SessionManagerActorKey>();
var subscriber = CreateTestProbe("invalid-rationale-limit-sub");
await sessionManager.Ask<SessionJoined>(new JoinSession(subscriber)
{
SessionId = sessionId,
Filter = OutputFilter.Full
}, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
await subscriber.ExpectMsgAsync<SessionJoined>(
cancellationToken: TestContext.Current.CancellationToken);

await sessionManager.Ask<CommandAck>(new SendUserMessage
{
SessionId = sessionId,
Content = "Load the browser tool"
}, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);

for (var index = 1; index <= 3; index++)
{
await subscriber.ExpectMsgAsync<ToolCallOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
var result = await subscriber.ExpectMsgAsync<ToolResultOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
Assert.Equal("invalid_rationale", result.FailureCode);
}

await subscriber.ExpectMsgAsync<TextOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
await subscriber.ExpectMsgAsync<TurnCompleted>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
Assert.Equal(4, _fakeChatClient.CallCount);
Assert.Empty(_fakeChatClient.ReceivedToolNames[3]);
}

[Fact]
public async Task Successful_corrections_do_not_reset_the_invalid_rationale_limit()
{
_fakeToolExecutor.RequireRationale = true;
for (var index = 1; index <= 5; index++)
{
var arguments = new Dictionary<string, object?>
{
["Name"] = "browser_chrome_devtools/navigate_page"
};
if (index % 2 == 0)
arguments["_rationale"] = "Load the browser tool for this test.";

_fakeChatClient.PlannedResponses.Enqueue(
[
new FunctionCallContent($"call-{index}", "load_tool", arguments)
]);
}

var sessionId = new SessionId("signalr/alternating-invalid-rationale-limit");
var sessionManager = ActorRegistry.Get<SessionManagerActorKey>();
var subscriber = CreateTestProbe("alternating-invalid-rationale-limit-sub");
await sessionManager.Ask<SessionJoined>(new JoinSession(subscriber)
{
SessionId = sessionId,
Filter = OutputFilter.Full
}, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
await subscriber.ExpectMsgAsync<SessionJoined>(
cancellationToken: TestContext.Current.CancellationToken);

await sessionManager.Ask<CommandAck>(new SendUserMessage
{
SessionId = sessionId,
Content = "Load the browser tool"
}, TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);

for (var index = 1; index <= 5; index++)
{
await subscriber.ExpectMsgAsync<ToolCallOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
var result = await subscriber.ExpectMsgAsync<ToolResultOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
if (index % 2 == 0)
Assert.Null(result.FailureCode);
else
Assert.Equal("invalid_rationale", result.FailureCode);
}

await subscriber.ExpectMsgAsync<TextOutput>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
await subscriber.ExpectMsgAsync<TurnCompleted>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
Assert.Equal(6, _fakeChatClient.CallCount);
Assert.Empty(_fakeChatClient.ReceivedToolNames[5]);

var replayedValidCall = _fakeChatClient.ReceivedMessages[2]
.SelectMany(message => message.Contents)
.OfType<FunctionCallContent>()
.Single(call => call.CallId == "call-2");
Assert.Equal(
"Load the browser tool for this test.",
replayedValidCall.Arguments!["_rationale"]);
}

[Fact]
public async Task Discovered_tools_are_retained_then_expire_after_lease_window()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,35 @@ public Task<string> ExecuteAsync(FunctionCallContent toolCall, ToolExecutionCont
}
}

private sealed class RequiredRationaleExecutor : IToolExecutor
{
public int ExecutionCount;

public Task AuthorizeAsync(
FunctionCallContent toolCall,
ToolExecutionContext? context = null,
CancellationToken ct = default)
{
return Task.CompletedTask;
}

public ToolArgumentRejection? ValidateToolCall(FunctionCallContent toolCall)
=> ToolCallMetaExtractor.ValidateRequiredRationale(
toolCall.Arguments,
ToolCallMeta.ResolveExactMetaField) is { } error
? new ToolArgumentRejection(error, "invalid_rationale")
: null;

public Task<string> ExecuteAsync(
FunctionCallContent toolCall,
ToolExecutionContext? context = null,
CancellationToken ct = default)
{
ExecutionCount++;
return Task.FromResult($"executed:{toolCall.CallId}");
}
}

// ── Timeout hint is honored exactly (no clamp, no floor) ──

[Fact]
Expand Down Expand Up @@ -239,6 +268,47 @@ public async Task Malformed_metadata_returns_denial_without_execution()
Assert.Contains("'_background'", result.Content);
}

[Fact]
public async Task Missing_rationale_rejects_while_a_parallel_sibling_runs()
{
var executor = new RequiredRationaleExecutor();
var probe = CreateTestProbe("rationale-isolation");
var sessionId = new SessionId("D1/rationale-isolation");
var toolCalls = new List<FunctionCallContent>
{
new("call-valid", "shell_execute", new Dictionary<string, object?>
{
["Command"] = "echo valid",
["_rationale"] = "Run the valid sibling."
}),
new("call-invalid", "shell_execute", new Dictionary<string, object?>
{
["Command"] = "echo invalid"
})
};

var pipelineTask = new SessionToolPipelineTestFixture(executor, toolCalls, sessionId, probe.Ref)
.WithTurnContext(InteractiveTurnContext(sessionId))
.ExecuteAsync(TestContext.Current.CancellationToken);

var completed = await probe.ExpectMsgAsync<ToolExecutionCompleted>(
TimeSpan.FromSeconds(3),
cancellationToken: TestContext.Current.CancellationToken);
await pipelineTask.WaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);

Assert.Equal(1, executor.ExecutionCount);
Assert.Equal(2, completed.ToolResults.Count);
Assert.Contains(completed.ToolResults, result =>
result.ToolCallId == new ToolCallId("call-valid")
&& result.Content == "executed:call-valid");
Assert.Contains(completed.ToolResults, result =>
result.ToolCallId == new ToolCallId("call-invalid")
&& result.Content.Contains("'_rationale'", StringComparison.Ordinal)
&& result.Content.Contains("NOT executed", StringComparison.Ordinal));
Assert.Equal("invalid_rationale", completed.ToolFailureCodes["call-invalid"]);
Assert.False(completed.ToolFailureCodes.ContainsKey("call-valid"));
}

[Fact]
public async Task Non_integral_json_timeout_rejects_without_uncaught_throw()
{
Expand Down
Loading
Loading