diff --git a/evals/run-evals.sh b/evals/run-evals.sh index c7a2030f03..08525c5df2 100755 --- a/evals/run-evals.sh +++ b/evals/run-evals.sh @@ -1036,6 +1036,27 @@ stdout_json_tool_call_arguments() { "$STDOUT_FILE" 2>/dev/null } +stdout_json_all_tool_calls_have_rationale() { + local minimum_calls="${1:-1}" + jq -e --argjson minimum_calls "$minimum_calls" ' + (.toolCalls // []) as $calls + | ($calls | length) >= $minimum_calls + and all($calls[]; + try ( + (.argumentsJson | fromjson | ._rationale) as $rationale + | ($rationale | type) == "string" + and ($rationale | length) > 0 + ) catch false) + ' "$STDOUT_FILE" >/dev/null 2>&1 +} + +stdout_json_tool_call_sequence_matches() { + local expected_json="$1" + jq -e --argjson expected "$expected_json" ' + [(.toolCalls // [])[] | .toolName] == $expected + ' "$STDOUT_FILE" >/dev/null 2>&1 +} + stdout_skill_file_read_called() { grep -aiE '^\[tool:call\] file_read\(' "$STDOUT_FILE" 2>/dev/null \ | grep -qi 'SKILL\.md' @@ -1307,6 +1328,13 @@ assert_tool_local_repository_search() { && stdout_response_contains 'match.txt' } +assert_tool_rationale_contract() { + stdout_json_envelope_valid \ + && stdout_json_tool_call_sequence_matches \ + '["file_list","list_reminders","file_read","skill_load"]' \ + && stdout_json_all_tool_calls_have_rationale 4 +} + assert_tool_timestamped_webhook() { stdout_tool_called 'set_webhook' \ && stdout_contains 'HmacTimestamped' \ @@ -2166,6 +2194,9 @@ run_all() { run_case tool_local_repository_search "local repository search uses shell, not web_search" \ "Search recursively under /home/netclaw/.netclaw/workspaces/file-tool-selection/search-target for the exact text local-search-eval-token and tell me which file contains it." + run_case --json tool_rationale_contract "all calls retain rationales across two parallel tool iterations" \ + "Use exactly two tool stages. First, call file_list on /home/netclaw/.netclaw/workspaces and list_reminders in one parallel batch. After both results return, call file_read on /home/netclaw/.netclaw/workspaces/netclaw-eval-largefile.txt for lines 1 through 3 and skill_load for netclaw-operations in one parallel batch. Use all four tools, then summarize the results." + run_case tool_timestamped_webhook "set_webhook called with Stripe timestamp verification" \ "Create a public inbound webhook route named stripe-events for Stripe. Use secret eval-whsec-123 and have it summarize each payment event." diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 8cd6bfb03c..8667f09e62 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.52.0" + version: "2.53.0" --- # Netclaw Operations @@ -98,6 +98,11 @@ meta keys also accept the underscore-dropped/cased/shortened forms `Background` likewise). The supplied value is always *used* — never silently defaulted. +Every tool call requires a non-empty `_rationale` string. State the call intent +and reason in one sentence. Apply this rule to each parallel call and each later +tool iteration. If a correction reports a missing rationale, fix every call +before the retry. + Three things are still rejected loudly, and when rejected the tool did NOT run — fix and re-issue once, do not retry the same shape: diff --git a/src/Netclaw.Actors.Tests/Protocol/ChatMessageConverterTests.cs b/src/Netclaw.Actors.Tests/Protocol/ChatMessageConverterTests.cs index fe9f788684..2b536a6aaf 100644 --- a/src/Netclaw.Actors.Tests/Protocol/ChatMessageConverterTests.cs +++ b/src/Netclaw.Actors.Tests/Protocol/ChatMessageConverterTests.cs @@ -637,7 +637,7 @@ public void FromAiMessage_with_interpreter_persists_schema_aware_meta() } [Fact] - public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound() + public void ToAiMessage_replays_rationale_and_limits_execution_hints_to_redrive() { // Persist a near-miss call: the meta key is stripped into MetaJson. var persisted = ChatMessageConverter.FromAiMessage( @@ -646,6 +646,7 @@ public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound( new FunctionCallContent("c1", "shell_execute", new Dictionary { ["Command"] = "long-task", + ["_rationale"] = "Run the long task for the user.", ["TimeoutSeconds"] = 1800 }) }), @@ -659,12 +660,14 @@ public void ToAiMessage_reinjectMeta_restores_hint_for_redrive_but_not_outbound( var redriven = ChatMessageConverter.ToAiMessage(persisted, reinjectMeta: true); var redrivenCall = redriven.Contents.OfType().Single(); Assert.True(redrivenCall.Arguments!.ContainsKey("_timeout_seconds")); + Assert.Equal("Run the long task for the user.", redrivenCall.Arguments["_rationale"]); var (meta, _) = ToolCallMeta.ExtractFrom(redrivenCall.Arguments, ToolArgumentHelper.ResolveMetaField); Assert.Equal(1800, meta?.TimeoutHintSeconds); - // Outbound provider history (the default) must NOT carry meta keys. + // Outbound history keeps the required rationale. It omits execution hints. var outbound = ChatMessageConverter.ToAiMessage(persisted); var outboundCall = outbound.Contents.OfType().Single(); + Assert.Equal("Run the long task for the user.", outboundCall.Arguments!["_rationale"]); Assert.False(outboundCall.Arguments?.ContainsKey("_timeout_seconds") ?? false); } diff --git a/src/Netclaw.Actors.Tests/Protocol/SerializationRoundTripTests.cs b/src/Netclaw.Actors.Tests/Protocol/SerializationRoundTripTests.cs index 7618331ffd..6bae55b5f6 100644 --- a/src/Netclaw.Actors.Tests/Protocol/SerializationRoundTripTests.cs +++ b/src/Netclaw.Actors.Tests/Protocol/SerializationRoundTripTests.cs @@ -151,6 +151,24 @@ public void TurnRecorded_round_trips_preserving_value_object_source_ids() Assert.Equal(original.SourceBackgroundJobId, result.SourceBackgroundJobId); } + [Fact] + public void Tool_result_failure_code_round_trips_through_the_transport_DTO() + { + var original = new ToolResultOutput + { + SessionId = new SessionId("test/wire"), + CallId = new ToolCallId("call-rejected"), + ToolName = new ToolName("search"), + Result = "The tool was not executed.", + FailureCode = "invalid_rationale" + }; + + var result = Assert.IsType( + SessionOutputDtoMapper.FromDto(SessionOutputDtoMapper.ToDto(original))); + + Assert.Equal("invalid_rationale", result.FailureCode); + } + [Fact] public void TurnRecorded_round_trips_with_null_source_ids() { diff --git a/src/Netclaw.Actors.Tests/Sessions/Pipelines/MetaValidationAndNoticeTests.cs b/src/Netclaw.Actors.Tests/Sessions/Pipelines/MetaValidationAndNoticeTests.cs index 8bc9f190c6..3ded3ac745 100644 --- a/src/Netclaw.Actors.Tests/Sessions/Pipelines/MetaValidationAndNoticeTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/Pipelines/MetaValidationAndNoticeTests.cs @@ -98,6 +98,35 @@ public Task ExecuteAsync(FunctionCallContent toolCall, ToolExecutionCont } } + private sealed class RequiredRationaleExecutor : IToolExecutor + { + public int ExecutionCount; + + public Task AuthorizeAsync( + FunctionCallContent toolCall, + ToolExecutionContext? context = null, + CancellationToken ct = default) + { + return Task.CompletedTask; + } + + public ToolArgumentRejection? ValidateToolCall(FunctionCallContent toolCall) + => ToolCallMetaExtractor.ValidateRequiredRationale( + toolCall.Arguments, + ToolCallMeta.ResolveExactMetaField) is { } error + ? new ToolArgumentRejection(error, "invalid_rationale") + : null; + + public Task ExecuteAsync( + FunctionCallContent toolCall, + ToolExecutionContext? context = null, + CancellationToken ct = default) + { + ExecutionCount++; + return Task.FromResult($"executed:{toolCall.CallId}"); + } + } + // ── Timeout hint is honored exactly (no clamp, no floor) ── [Fact] @@ -239,6 +268,47 @@ public async Task Malformed_metadata_returns_denial_without_execution() Assert.Contains("'_background'", result.Content); } + [Fact] + public async Task Missing_rationale_rejects_while_a_parallel_sibling_runs() + { + var executor = new RequiredRationaleExecutor(); + var probe = CreateTestProbe("rationale-isolation"); + var sessionId = new SessionId("D1/rationale-isolation"); + var toolCalls = new List + { + new("call-valid", "shell_execute", new Dictionary + { + ["Command"] = "echo valid", + ["_rationale"] = "Run the valid sibling." + }), + new("call-invalid", "shell_execute", new Dictionary + { + ["Command"] = "echo invalid" + }) + }; + + var pipelineTask = new SessionToolPipelineTestFixture(executor, toolCalls, sessionId, probe.Ref) + .WithTurnContext(InteractiveTurnContext(sessionId)) + .ExecuteAsync(TestContext.Current.CancellationToken); + + var completed = await probe.ExpectMsgAsync( + TimeSpan.FromSeconds(3), + cancellationToken: TestContext.Current.CancellationToken); + await pipelineTask.WaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + Assert.Equal(1, executor.ExecutionCount); + Assert.Equal(2, completed.ToolResults.Count); + Assert.Contains(completed.ToolResults, result => + result.ToolCallId == new ToolCallId("call-valid") + && result.Content == "executed:call-valid"); + Assert.Contains(completed.ToolResults, result => + result.ToolCallId == new ToolCallId("call-invalid") + && result.Content.Contains("'_rationale'", StringComparison.Ordinal) + && result.Content.Contains("NOT executed", StringComparison.Ordinal)); + Assert.Equal("invalid_rationale", completed.ToolFailureCodes["call-invalid"]); + Assert.False(completed.ToolFailureCodes.ContainsKey("call-valid")); + } + [Fact] public async Task Non_integral_json_timeout_rejects_without_uncaught_throw() { diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index f23d7d412a..bdc6e1212d 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -34,6 +34,18 @@ public class SubAgentSpawnIntegrationTests : LlmSessionTestBase private RecordingContextTool? _recordingFileReadTool; private RecordingContextTool? _recordingShellTool; + private static FunctionCallContent CreateToolCall( + string callId, + string name, + IDictionary arguments) + { + var callArguments = new Dictionary(arguments, StringComparer.Ordinal) + { + ["_rationale"] = "Verify the sub-agent session behavior." + }; + return new FunctionCallContent(callId, name, callArguments); + } + public SubAgentSpawnIntegrationTests(ITestOutputHelper output) : base(output) { } @@ -205,7 +217,7 @@ public async Task Spawn_agent_runs_under_session_and_emits_subagent_events() { _clientProvider.Main.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( "call-spawn", "spawn_agent", new Dictionary @@ -295,7 +307,7 @@ public async Task Spawn_agent_subagent_approval_uses_parent_authority_and_resume _clientProvider.Main.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( parentCallId, "spawn_agent", new Dictionary @@ -306,7 +318,7 @@ public async Task Spawn_agent_subagent_approval_uses_parent_authority_and_resume ]; _clientProvider.Compaction.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( childCallId, "shell_execute", new Dictionary @@ -389,7 +401,7 @@ public async Task Spawn_agent_subagent_approval_expires_after_parent_session_rec { _clientProvider.Main.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( "call-spawn-shell-expire", "spawn_agent", new Dictionary @@ -400,7 +412,7 @@ public async Task Spawn_agent_subagent_approval_expires_after_parent_session_rec ]; _clientProvider.Compaction.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( "call-subagent-shell-expire", "shell_execute", new Dictionary @@ -708,7 +720,7 @@ public async Task Routed_slash_ignores_skill_allowed_tools_for_runtime_authoriza { _clientProvider.Compaction.ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( "call-read", "file_read", new Dictionary diff --git a/src/Netclaw.Actors.Tests/Sessions/ToolBatchHistoryWedgeTests.cs b/src/Netclaw.Actors.Tests/Sessions/ToolBatchHistoryWedgeTests.cs index e6aab24471..77f665c542 100644 --- a/src/Netclaw.Actors.Tests/Sessions/ToolBatchHistoryWedgeTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/ToolBatchHistoryWedgeTests.cs @@ -82,15 +82,22 @@ public async Task Partial_failure_of_parallel_tool_batch_leaves_history_well_for var ct = TestContext.Current.CancellationToken; // Turn 1: two parallel tool calls. call-A executes normally; call-B - // throws in InterpretToolCall, which escapes to ToolExecutionFailed -> - // FailCurrentTurn. call-A is recorded before the failure (Task.WhenAll - // invariant), so the batch fails with A answered and B unanswered. + // throws during the execution pipeline preflight. The pipeline reports + // ToolExecutionFailed after call-A reports its streamed result. _fakeChatClient.ToolCallsOnFirstCall = [ new FunctionCallContent("call-A", "web_search", - new Dictionary { ["query"] = "a" }), + new Dictionary + { + ["query"] = "a", + ["_rationale"] = "Search for the first test result." + }), new FunctionCallContent("call-B", "web_search", - new Dictionary { ["query"] = "b" }), + new Dictionary + { + ["query"] = "b", + ["_rationale"] = "Search for the second test result." + }), ]; _executor.FailInterpretForCallIds.Add("call-B"); diff --git a/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs index f5359ea3e5..e4ae4c19a1 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SpawnAgentStreamingTests.cs @@ -98,7 +98,8 @@ public async Task Spawn_agent_streams_activity_through_executor_dispatch_to_watc new Dictionary { ["agent"] = "summarizer", - ["task"] = "Summarize the project." + ["task"] = "Summarize the project.", + ["_rationale"] = "Verify streamed sub-agent activity." }); // Drain the stream the way the production pipeline drains a self-monitoring @@ -191,7 +192,8 @@ public async Task Spawn_agent_self_monitoring_survives_quiet_window_after_first_ new Dictionary { ["agent"] = "summarizer", - ["task"] = "Summarize the project." + ["task"] = "Summarize the project.", + ["_rationale"] = "Verify self-monitored sub-agent activity." }); // spawn_agent is self-monitoring, so the parent drains it with no watchdog at diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs index 18c4b4a8e8..6a5d691fc2 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs @@ -32,6 +32,21 @@ public class SubAgentActorTests : TestKit private static readonly TimeSpan ApprovalAskTimeout = TimeSpan.FromSeconds(30); public static bool IsPosix => !OperatingSystem.IsWindows(); + private static FunctionCallContent CreateToolCall(string callId, string name) + => CreateToolCall(callId, name, new Dictionary()); + + private static FunctionCallContent CreateToolCall( + string callId, + string name, + IDictionary arguments) + { + var callArguments = new Dictionary(arguments, StringComparer.Ordinal) + { + ["_rationale"] = "Verify the sub-agent behavior." + }; + return new FunctionCallContent(callId, name, callArguments); + } + public SubAgentActorTests(ITestOutputHelper output) : base(output: output) { } protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) @@ -290,7 +305,7 @@ public async Task Tool_call_executes_and_continues() { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-1", "greet", + CreateToolCall("call-1", "greet", new Dictionary { ["name"] = "World" }) ] }; @@ -321,7 +336,7 @@ public async Task Tool_model_input_image_is_attached_to_subagent_followup_call() onExecute: context => context.AddModelInputFile(imagePath, "diagram.png", "image/png")); var fakeClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-image", "load_image")] + ToolCallsOnFirstCall = [CreateToolCall("call-image", "load_image")] }; var definition = CreateDefinition([fakeTool]); var agent = Sys.ActorOf(SubAgentActor.CreateProps(definition, fakeClient, PermissivePolicy())); @@ -356,7 +371,7 @@ public async Task Tool_execution_inherits_parent_session_and_project_directories { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-context", "inspect_context") + CreateToolCall("call-context", "inspect_context") ] }; @@ -388,7 +403,7 @@ public async Task Tool_execution_with_no_parent_project_directory_passes_null_th var fakeTool = new FakeNetclawTool("inspect_context", "ok"); var fakeClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-no-project", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-no-project", "inspect_context")] }; var definition = CreateDefinition([fakeTool]); @@ -415,7 +430,7 @@ public async Task Tool_execution_inherits_parent_resolved_cwd_snapshot() var fakeTool = new FakeNetclawTool("inspect_context", "ok"); var fakeClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-cwd", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-cwd", "inspect_context")] }; var agent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition([fakeTool]), fakeClient, PermissivePolicy())); @@ -446,7 +461,7 @@ public async Task Tool_execution_with_null_parent_cwd_resolves_to_session_dir_or var fakeTool = new FakeNetclawTool("inspect_context", "ok"); var fakeClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-null-cwd", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-null-cwd", "inspect_context")] }; var agent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition([fakeTool]), fakeClient, PermissivePolicy())); @@ -477,7 +492,7 @@ public async Task Tool_execution_inherits_parent_cwd_when_child_has_no_project_o var fakeTool = new FakeNetclawTool("inspect_context", "ok"); var fakeClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-inherit-only", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-inherit-only", "inspect_context")] }; var agent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition([fakeTool]), fakeClient, PermissivePolicy())); @@ -504,7 +519,7 @@ public async Task Each_spawn_snapshots_its_own_parent_project_directory() var firstTool = new FakeNetclawTool("inspect_context", "ok"); var firstClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-1", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-1", "inspect_context")] }; var firstAgent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition([firstTool]), firstClient, PermissivePolicy())); @@ -522,7 +537,7 @@ public async Task Each_spawn_snapshots_its_own_parent_project_directory() var secondTool = new FakeNetclawTool("inspect_context", "ok"); var secondClient = new FakeChatClient { - ToolCallsOnFirstCall = [new FunctionCallContent("call-2", "inspect_context")] + ToolCallsOnFirstCall = [CreateToolCall("call-2", "inspect_context")] }; var secondAgent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition([secondTool]), secondClient, PermissivePolicy())); @@ -586,7 +601,7 @@ public async Task Session_scratch_context_does_not_authorize_headless_prompt_wor { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-approval", "shell_execute", + CreateToolCall("call-approval", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -624,7 +639,7 @@ public async Task Subagent_approval_request_carries_cwd_candidates_and_full_butt { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-cwd-prompt", "shell_execute", + CreateToolCall("call-cwd-prompt", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -769,7 +784,11 @@ public async Task Subagent_project_declaration_updates_child_prompt_before_uncha new FunctionCallContent( declarationCallId, SetWorkingDirectoryTool.ToolName, - new Dictionary { ["Path"] = worktree }), + new Dictionary + { + ["Path"] = worktree, + ["_rationale"] = "Declare the project directory before the next inspection." + }), ProjectScopeCall(retryCallId, worktree) ]); var approvalBridge = supportsApproval @@ -834,7 +853,11 @@ public async Task Subagent_rejects_control_characters_from_project_scope_result( new FunctionCallContent( "call-control-project", SetWorkingDirectoryTool.ToolName, - new Dictionary { ["Path"] = controlledDirectory }) + new Dictionary + { + ["Path"] = controlledDirectory, + ["_rationale"] = "Verify that the project scope rejects control characters." + }) ]); var promptProvider = new ProjectPromptProvider(controlledDirectory, projectGuidance); var actor = Sys.ActorOf(SubAgentActor.CreatePropsWithProjectInstructionProvider( @@ -953,11 +976,11 @@ public async Task Approve_once_does_not_leak_between_subagent_tool_calls() var policy = CreateApprovalRequiredPolicy(); var fakeClient = new SequencedToolCallChatClient( [ - new FunctionCallContent( + CreateToolCall( "call-approval-1", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }), - new FunctionCallContent( + CreateToolCall( "call-approval-2", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) @@ -994,7 +1017,7 @@ public async Task SubAgent_does_not_timeout_while_awaiting_human_approval() { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-slow-approval", "shell_execute", + CreateToolCall("call-slow-approval", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1038,7 +1061,7 @@ public async Task SubAgent_surfaces_approval_wait_and_resolution_to_parent_strea { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-activity-approval", "shell_execute", + CreateToolCall("call-activity-approval", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1092,7 +1115,7 @@ public async Task SubAgent_cancels_promptly_on_external_cancellation_during_appr { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-cancel", "shell_execute", + CreateToolCall("call-cancel", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1144,9 +1167,9 @@ public async Task SubAgent_parallel_tool_calls_each_awaiting_approval() { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-par-1", "shell_execute", + CreateToolCall("call-par-1", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }), - new FunctionCallContent("call-par-2", "shell_execute", + CreateToolCall("call-par-2", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1192,7 +1215,7 @@ public async Task Rejected_approval_returns_tool_result_without_executing_tool( { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-rejected", "shell_execute", + CreateToolCall("call-rejected", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1224,7 +1247,7 @@ public async Task External_stop_during_approval_wait_replies_once_and_cancels_wa { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-stop", "shell_execute", + CreateToolCall("call-stop", "shell_execute", new Dictionary { ["Command"] = "git push origin main" }) ] }; @@ -1416,14 +1439,16 @@ private static FunctionCallContent ScratchCall(string callId) => new(callId, ShellTool.ToolName, new Dictionary { ["Command"] = "gh api repos/example/project", - ["WorkingDirectory"] = "/tmp" + ["WorkingDirectory"] = "/tmp", + ["_rationale"] = "Inspect a disposable diagnostic artifact." }); private static FunctionCallContent ProjectScopeCall(string callId, string workingDirectory) => new(callId, ShellTool.ToolName, new Dictionary { ["Command"] = ProjectScopeCommand, - ["WorkingDirectory"] = workingDirectory + ["WorkingDirectory"] = workingDirectory, + ["_rationale"] = "Inspect the project metric sources." }); private static string ProjectScopeCommand => @@ -1514,7 +1539,7 @@ public async Task Max_iterations_forces_text_response() { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-loop", "looper") + CreateToolCall("call-loop", "looper") ], AlwaysReturnToolCalls = true }; @@ -1651,7 +1676,7 @@ public void Classify_distinguishes_keepalives_from_substantive_progress() // Tool-call content is substantive. var toolCall = StreamingResponseReader.Classify( - new ChatResponseUpdate { Role = ChatRole.Assistant, Contents = [new FunctionCallContent("call-1", "inspect_context")] }, + new ChatResponseUpdate { Role = ChatRole.Assistant, Contents = [CreateToolCall("call-1", "inspect_context")] }, anySubstantiveSeen: false); Assert.True(toolCall.HasSubstantiveContent); @@ -1726,7 +1751,7 @@ public async Task Tool_execution_uses_session_scope_for_mcp_invocation() { ToolCallsOnFirstCall = [ - new FunctionCallContent( + CreateToolCall( "call-1", "browser_playwright/navigate_page", new Dictionary { ["url"] = "https://example.com" }) @@ -1994,7 +2019,7 @@ public async Task Successful_first_party_edit_is_returned_as_confirmed_child_act { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-edit", "file_edit", + CreateToolCall("call-edit", "file_edit", new Dictionary { ["Path"] = "src/Calculator.cs" }) ] }; @@ -2025,7 +2050,7 @@ public async Task Denied_first_party_edit_is_not_returned_as_confirmed_child_act { ToolCallsOnFirstCall = [ - new FunctionCallContent("call-edit", "file_edit", + CreateToolCall("call-edit", "file_edit", new Dictionary { ["Path"] = "src/Calculator.cs" }) ] }; diff --git a/src/Netclaw.Actors.Tests/Tools/DispatchingToolExecutorTests.cs b/src/Netclaw.Actors.Tests/Tools/DispatchingToolExecutorTests.cs index b56b5d61af..717cce061d 100644 --- a/src/Netclaw.Actors.Tests/Tools/DispatchingToolExecutorTests.cs +++ b/src/Netclaw.Actors.Tests/Tools/DispatchingToolExecutorTests.cs @@ -27,6 +27,18 @@ public class DispatchingToolExecutorTests private readonly DispatchingToolExecutor _executor; private readonly DispatchingToolExecutor _restrictedExecutor; + private static FunctionCallContent CreateToolCall( + string callId, + string name, + IDictionary arguments) + { + var callArguments = new Dictionary(arguments, StringComparer.Ordinal) + { + ["_rationale"] = "Verify the executor behavior." + }; + return new FunctionCallContent(callId, name, callArguments); + } + public DispatchingToolExecutorTests() { var baseConfig = new ToolConfig(); @@ -93,7 +105,7 @@ public async Task Verbose_tool_output_over_budget_is_windowed_and_spilled() try { // shell_execute declares the small verbose budget (2000); echo > 2000 chars. - var toolCall = new FunctionCallContent("call-spill", "shell_execute", + var toolCall = CreateToolCall("call-spill", "shell_execute", ToolInput.Create("Command", $"echo {new string('x', 3000)}")); var context = TestToolExecutionContext.CreateBound("slack/thread-1", sessionDir, new TestToolExecutionContextOptions { @@ -123,7 +135,7 @@ public async Task Spilled_output_is_redacted_before_write() try { // Secret + padding so it both redacts and exceeds the shell budget → spills. - var toolCall = new FunctionCallContent("call-redact", "shell_execute", + var toolCall = CreateToolCall("call-redact", "shell_execute", ToolInput.Create("Command", $"echo API_KEY=supersecret123 {new string('x', 3000)}")); var context = TestToolExecutionContext.CreateBound("slack/thread-1", sessionDir, new TestToolExecutionContextOptions { @@ -147,7 +159,7 @@ public async Task Spilled_output_is_redacted_before_write() public async Task Small_output_is_redacted_without_spilling() { // Redaction happens centrally for every result, spill or not. - var toolCall = new FunctionCallContent("call-r", "shell_execute", + var toolCall = CreateToolCall("call-r", "shell_execute", ToolInput.Create("Command", "echo API_KEY=secret123")); var context = TestToolExecutionContext.CreateBound("signalr/thread-1", null, new TestToolExecutionContextOptions { @@ -174,7 +186,7 @@ public async Task File_read_preserves_secret_values_for_model() await File.WriteAllTextAsync(file, """{"secretKey": "real-secret-value", "name": "myapp"}""", CancellationToken.None); - var toolCall = new FunctionCallContent("call-secret", "file_read", + var toolCall = CreateToolCall("call-secret", "file_read", ToolInput.Create("Path", file)); var context = TestToolExecutionContext.CreateBound("slack/thread-1", sessionDir, new TestToolExecutionContextOptions { @@ -196,7 +208,7 @@ await File.WriteAllTextAsync(file, public async Task Shell_output_still_redacts_secrets() { // Shell output continues to be redacted — only file tools suppress it. - var toolCall = new FunctionCallContent("call-shell-secret", "shell_execute", + var toolCall = CreateToolCall("call-shell-secret", "shell_execute", ToolInput.Create("Command", "echo API_KEY=secret123")); var context = TestToolExecutionContext.CreateBound("signalr/thread-1", null, new TestToolExecutionContextOptions { @@ -224,7 +236,7 @@ public async Task File_read_spill_file_is_redacted_even_when_model_result_is_not var bigContent = $$"""{"secretKey": "real-secret-value", "data": "{{new string('x', 15000)}}"}"""; await File.WriteAllTextAsync(file, bigContent, CancellationToken.None); - var toolCall = new FunctionCallContent("call-spill-secret", "file_read", + var toolCall = CreateToolCall("call-spill-secret", "file_read", ToolInput.Create("Path", file)); var context = TestToolExecutionContext.CreateBound("slack/thread-1", sessionDir, new TestToolExecutionContextOptions { @@ -263,7 +275,7 @@ public async Task Content_tool_under_default_budget_not_spilled() // small file is returned whole with no spill. var file = Path.Combine(sessionDir, "note.txt"); await File.WriteAllTextAsync(file, "hello content", CancellationToken.None); - var toolCall = new FunctionCallContent("call-content", "file_read", + var toolCall = CreateToolCall("call-content", "file_read", ToolInput.Create("Path", file)); var context = TestToolExecutionContext.CreateBound("slack/thread-1", sessionDir, new TestToolExecutionContextOptions { @@ -285,7 +297,7 @@ public async Task Content_tool_under_default_budget_not_spilled() [Fact] public async Task Routes_shell_execute() { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-1", "shell_execute", ToolInput.Create("Command", "echo routed")); @@ -305,7 +317,7 @@ public async Task Routes_shell_execute() [Fact] public async Task Routes_file_read_missing_file() { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-2", "file_read", ToolInput.Create("Path", "/nonexistent/file.txt")); @@ -324,7 +336,7 @@ public async Task Routes_file_read_missing_file() [Fact] public async Task Shell_execute_is_denied_outside_personal_context() { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-deny", "shell_execute", ToolInput.Create("Command", "echo denied")); @@ -363,7 +375,7 @@ public async Task Shell_execute_is_denied_when_missing_from_personal_audience_pr commandPolicy, pathPolicy)); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-shell-profile-deny", "shell_execute", ToolInput.Create("Command", "echo denied")); @@ -402,7 +414,7 @@ public async Task Shell_execute_is_denied_when_shell_mode_is_off_even_in_persona commandPolicy, pathPolicy)); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-shell-off", "shell_execute", ToolInput.Create("Command", "echo denied")); @@ -427,7 +439,7 @@ public async Task Shell_execute_is_denied_when_shell_mode_is_off_even_in_persona [Fact] public async Task Shell_execute_is_allowed_in_personal_context() { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-allow", "shell_execute", ToolInput.Create("Command", "echo allowed")); @@ -451,7 +463,7 @@ public async Task Shell_execute_is_allowed_in_personal_context() public async Task Approval_exempt_shell_candidates_report_allow_reason(string command) { var executor = CreateApprovalGatedShellExecutor(); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-approval-exempt", "shell_execute", ToolInput.Create("Command", command)); @@ -473,7 +485,7 @@ public async Task Approval_exempt_shell_candidates_report_allow_reason(string co public async Task Shell_approval_without_extracted_candidates_fails_closed(string command) { var executor = CreateApprovalGatedShellExecutor(); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-no-approval-candidates", "shell_execute", ToolInput.Create("Command", command)); @@ -504,7 +516,7 @@ public async Task Shell_parser_rejection_fails_closed_without_execution() Assert.Empty(matcher.ExtractCandidates(new ToolName("shell_execute"), arguments)); var executor = CreateApprovalGatedShellExecutor(); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-parser-rejection", "shell_execute", arguments); @@ -553,7 +565,7 @@ public async Task Authorization_evaluation_preserves_partial_approval_matches() new ShellCommandPolicy(), new ToolPathPolicy([])), approvalService); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-partial-approval", "shell_execute", ToolInput.Create("Command", "git status && git push")); @@ -620,7 +632,7 @@ public async Task Authorization_evaluation_prompts_only_for_exact_unapproved_can new ShellCommandPolicy(), new ToolPathPolicy([])), approvalService); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-exact-partial-approval", "shell_execute", ToolInput.Create("Command", "git status && git push")); @@ -2117,7 +2129,7 @@ public async Task Authorization_evaluation_preserves_directory_for_duplicate_ver new ShellCommandPolicy(), new ToolPathPolicy([])), approvalService); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-duplicate-verb-scopes", "shell_execute", ToolInput.Create( @@ -2184,7 +2196,7 @@ public async Task Authorization_evaluation_denies_inconsistent_candidate_result( new ShellCommandPolicy(), new ToolPathPolicy([])), approvalService); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-inconsistent-partial-approval", "shell_execute", ToolInput.Create("Command", "git status && git push")); @@ -2300,7 +2312,7 @@ public async Task Authorization_evaluation_rejects_inconsistent_all_approved_res new ShellCommandPolicy(), new ToolPathPolicy([])), approvalService); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-inconsistent-all-approved", "shell_execute", ToolInput.Create("Command", "git status && git push")); @@ -2341,7 +2353,7 @@ public async Task Authorization_evaluation_logs_allow_reason_before_execution() new ShellCommandPolicy(), new ToolPathPolicy([])), logger: logger); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-authorization-telemetry", "telemetry_probe", ToolInput.Empty()); @@ -2374,7 +2386,7 @@ public async Task File_read_is_denied_outside_session_directory_in_public_contex try { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-file-read-deny", "file_read", ToolInput.Create("Path", filePath)); @@ -2405,7 +2417,7 @@ public async Task File_write_is_denied_outside_session_directory_in_team_context try { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-file-write-deny", "file_write", ToolInput.Create("Path", filePath, "Content", "blocked")); @@ -2436,7 +2448,7 @@ public async Task Routes_file_write() var filePath = Path.Combine(Path.GetTempPath(), $"netclaw-dispatch-{Guid.NewGuid():N}.txt"); try { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-3", "file_write", ToolInput.Create("Path", filePath, "Content", "dispatch test")); @@ -2463,7 +2475,7 @@ public async Task Routes_file_write() [Fact] public async Task Unknown_tool_returns_error_string() { - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-4", "unknown_tool", ToolInput.Create("arg", "value")); @@ -2563,7 +2575,7 @@ public async Task Mcp_tool_is_denied_when_server_not_allowed_for_audience() new ShellCommandPolicy(), new ToolPathPolicy([]))); - var toolCall = new FunctionCallContent("call-mcp-deny", "memorizer/search_memories", ToolInput.Empty()); + var toolCall = CreateToolCall("call-mcp-deny", "memorizer/search_memories", ToolInput.Empty()); var context = TestToolExecutionContext.CreateBound("slack/thread-1", null, new TestToolExecutionContextOptions { Audience = TrustAudience.Team, @@ -2610,7 +2622,7 @@ public async Task One_time_approval_allows_immediate_retry_only() pathPolicy), approvalService); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-approve-once", "shell_execute", // Use a non-side-effect verb (echo/printf/:/true/false @@ -2678,7 +2690,7 @@ public async Task One_time_approval_bypasses_policy_for_matching_shell_patterns( commandPolicy, pathPolicy)); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-approve-once-bypass", "shell_execute", ToolInput.Create("Command", "echo bypass")); @@ -2784,7 +2796,7 @@ public async Task One_time_approval_bypasses_policy_for_path_aware_file_patterns new ToolPathPolicy([]), fileApprovalMatcher: new FilePathApprovalMatcher(controlPlaneRoot))); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-file-approve-once-bypass", "file_write", ToolInput.Create("Path", targetPath, "Content", "approved once")); @@ -2807,7 +2819,7 @@ public async Task One_time_approval_bypasses_policy_for_path_aware_file_patterns Assert.Contains("Successfully wrote", retryResult, StringComparison.Ordinal); Assert.True(File.Exists(targetPath)); - var secondCall = new FunctionCallContent( + var secondCall = CreateToolCall( "call-file-approve-once-bypass-second", "file_write", ToolInput.Create("Path", secondPath, "Content", "different path")); @@ -2890,7 +2902,7 @@ await approvalService.RecordApprovalAsync( cwd: null, TestContext.Current.CancellationToken); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-filtered-once", "shell_execute", ToolInput.Create("Command", command)); @@ -2970,7 +2982,7 @@ public async Task Persistent_approval_hit_records_audit_context_without_promptin InteractiveApproval = TestToolExecutionContext.InteractiveApproval(true) }); - var call = new FunctionCallContent( + var call = CreateToolCall( "call-audit", "shell_execute", ToolInput.Create("Command", "git status")); @@ -3031,7 +3043,7 @@ public async Task Session_approval_allows_same_session_but_not_different_session new ToolPathPolicy([])), approvalService); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-session-approve", "shell_execute", // Non-side-effect verb so the approval flow under test @@ -3147,7 +3159,7 @@ public async Task Mcp_session_approval_recorded_under_canonical_name_authorizes_ // The LLM emits tool_use with the sanitized alias — mirror that // here. The registry's two-form lookup (introduced in PR #1134) // resolves it back to the same adapter. - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( "call-mcp-approve-session", sanitizedAlias, ToolInput.Empty()); @@ -3189,7 +3201,7 @@ await approvalService.RecordApprovalAsync( // Same call dispatched by the canonical name must also resolve // — the registry accepts both forms, so the gate should // authorize either way. - var canonicalToolCall = new FunctionCallContent( + var canonicalToolCall = CreateToolCall( "call-mcp-approve-session-canonical", canonicalName, ToolInput.Empty()); @@ -3232,7 +3244,7 @@ public async Task Background_job_control_does_not_contact_approval_service(bool "slack/thread-1", null, new TestToolExecutionContextOptions { Audience = TrustAudience.Personal }); - var toolCall = new FunctionCallContent( + var toolCall = CreateToolCall( $"call-job-{cancel}", CheckBackgroundJobTool.ToolName, ToolInput.Create("JobId", "abc123", "Cancel", cancel)); @@ -3318,7 +3330,9 @@ public async Task Authorization_only_and_each_execution_use_separate_authorizati ShellTool.ToolName, ToolInput.Create( "Command", - ShellEnvironment.Grammar == ShellGrammar.PowerShell ? "Get-Location" : "pwd")); + ShellEnvironment.Grammar == ShellGrammar.PowerShell ? "Get-Location" : "pwd", + "_rationale", + "Inspect the current working directory.")); await executor.AuthorizeAsync(call, context, TestContext.Current.CancellationToken); _ = await executor.ExecuteAsync(call, context, TestContext.Current.CancellationToken); @@ -3333,8 +3347,8 @@ public async Task Authorization_only_and_each_execution_use_separate_authorizati public async Task Auto_shell_without_command_returns_argument_error(bool includeNullCommand) { var arguments = includeNullCommand - ? ToolInput.Create("Command", null) - : ToolInput.Empty(); + ? ToolInput.Create("Command", null, "_rationale", "Probe argument error handling.") + : ToolInput.Create("_rationale", "Probe argument error handling."); var call = new FunctionCallContent( $"call-missing-command-{includeNullCommand}", ShellTool.ToolName, @@ -3354,7 +3368,7 @@ public async Task Auto_shell_stream_without_command_returns_argument_error() var call = new FunctionCallContent( "call-stream-missing-command", ShellTool.ToolName, - ToolInput.Empty()); + ToolInput.Create("_rationale", "Probe stream argument error handling.")); ToolCompletedUpdate? completed = null; await foreach (var update in _executor.ExecuteStreamAsync( @@ -3376,8 +3390,8 @@ public async Task Approved_shell_without_command_reaches_argument_error(bool inc { var executor = CreateApprovalGatedShellExecutor(); var arguments = includeNullCommand - ? ToolInput.Create("Command", null) - : ToolInput.Empty(); + ? ToolInput.Create("Command", null, "_rationale", "Probe argument error handling.") + : ToolInput.Create("_rationale", "Probe argument error handling."); var call = new FunctionCallContent( $"call-approved-missing-command-{includeNullCommand}", ShellTool.ToolName, @@ -3415,7 +3429,9 @@ public async Task Parallel_shell_calls_keep_their_authorized_analyses_isolated() "Command", TestShellEnvironment.PrintWorkingDirectoryCommand, "WorkingDirectory", - firstRoot)); + firstRoot, + "_rationale", + "Inspect the current working directory.")); var secondCall = new FunctionCallContent( "call-parallel-shell-second", ShellTool.ToolName, @@ -3423,7 +3439,9 @@ public async Task Parallel_shell_calls_keep_their_authorized_analyses_isolated() "Command", TestShellEnvironment.PrintWorkingDirectoryCommand, "WorkingDirectory", - secondRoot)); + secondRoot, + "_rationale", + "Inspect the current working directory.")); var results = await Task.WhenAll( _executor.ExecuteAsync(firstCall, context, TestContext.Current.CancellationToken), @@ -3454,7 +3472,9 @@ public async Task Stream_and_non_stream_shell_calls_use_explicit_authorized_anal "Command", TestShellEnvironment.PrintWorkingDirectoryCommand, "WorkingDirectory", - root)); + root, + "_rationale", + "Inspect the current working directory.")); var streamCall = new FunctionCallContent( "call-shell-analysis-stream", ShellTool.ToolName, @@ -3462,7 +3482,9 @@ public async Task Stream_and_non_stream_shell_calls_use_explicit_authorized_anal "Command", TestShellEnvironment.PrintWorkingDirectoryCommand, "WorkingDirectory", - root)); + root, + "_rationale", + "Inspect the current working directory.")); var nonStreamResult = await _executor.ExecuteAsync( nonStreamCall, @@ -3648,6 +3670,26 @@ private static ShellApprovalMatchResult CreateMalformedActorResult( return new ShellApprovalMatchResult(store, Array.AsReadOnly(matches)); } + [Fact] + public async Task Missing_rationale_rejects_before_the_approval_service() + { + var executor = CreateApprovalGatedShellExecutor(); + var context = CreateInteractivePersonalContext("signalr/rationale-rejection"); + var toolCall = new FunctionCallContent( + "call-missing-rationale", + "shell_execute", + ToolInput.Create("Command", "echo should-not-run")); + + var result = await executor.ExecuteAsync( + toolCall, + context, + TestContext.Current.CancellationToken); + + Assert.Contains("'_rationale'", result); + Assert.Contains("NOT executed", result); + Assert.DoesNotContain("should-not-run", result); + } + private static ToolExecutionContext CreateInteractivePersonalContext(string sessionId) => TestToolExecutionContext.CreateBound( sessionId, diff --git a/src/Netclaw.Actors.Tests/Tools/MessyCommandOneTimeApprovalTests.cs b/src/Netclaw.Actors.Tests/Tools/MessyCommandOneTimeApprovalTests.cs index fe206dedd6..6ffb13ec96 100644 --- a/src/Netclaw.Actors.Tests/Tools/MessyCommandOneTimeApprovalTests.cs +++ b/src/Netclaw.Actors.Tests/Tools/MessyCommandOneTimeApprovalTests.cs @@ -98,7 +98,9 @@ public async Task ApprovedOnce_on_messy_command_satisfies_one_time_bypass() "shell_execute", ToolInput.Create( "Command", - "for i in $(printf '1 2 3'); do echo \"$i\"; done")); + "for i in $(printf '1 2 3'); do echo \"$i\"; done", + "_rationale", + "Verify one-time approval for a complex command.")); var context = TestToolExecutionContext.CreateBound("signalr/thread-1", null, new TestToolExecutionContextOptions { diff --git a/src/Netclaw.Actors.Tests/Tools/ToolArgumentValidatorTests.cs b/src/Netclaw.Actors.Tests/Tools/ToolArgumentValidatorTests.cs index b35cd7a267..9199f8df0d 100644 --- a/src/Netclaw.Actors.Tests/Tools/ToolArgumentValidatorTests.cs +++ b/src/Netclaw.Actors.Tests/Tools/ToolArgumentValidatorTests.cs @@ -66,7 +66,12 @@ private async Task ExecuteShellAsync(IDictionary args) Directory.CreateDirectory(sessionDir); try { - var toolCall = new FunctionCallContent("call-1", "shell_execute", args); + var callArgs = new Dictionary(args, StringComparer.Ordinal); + if (!callArgs.Keys.Any(key => + string.Equals(ToolArgumentHelper.ResolveMetaField(key), "_rationale", StringComparison.Ordinal))) + callArgs["_rationale"] = "Validate the tool argument contract."; + + var toolCall = new FunctionCallContent("call-1", "shell_execute", callArgs); return await _executor.ExecuteAsync( toolCall, PersonalContext(sessionDir), TestContext.Current.CancellationToken); } @@ -76,6 +81,42 @@ private async Task ExecuteShellAsync(IDictionary args) } } + [Fact] + public void Missing_rationale_rejects_before_execution() + { + var rejection = _executor.ValidateToolCall(new FunctionCallContent( + "call-missing-rationale", + "shell_execute", + new Dictionary { ["Command"] = "echo should-not-run" })); + + Assert.NotNull(rejection); + Assert.Equal("invalid_rationale", rejection!.DenyReason); + Assert.Contains("'_rationale'", rejection.Message); + Assert.Contains("non-empty string", rejection.Message); + Assert.Contains("NOT executed", rejection.Message); + } + + [Fact] + public void Blank_null_and_non_string_rationales_reject() + { + object?[] invalidValues = [null, " ", 42, false]; + + foreach (var invalidValue in invalidValues) + { + var rejection = _executor.ValidateToolCall(new FunctionCallContent( + "call-invalid-rationale", + "shell_execute", + new Dictionary + { + ["Command"] = "echo should-not-run", + ["_rationale"] = invalidValue + })); + + Assert.NotNull(rejection); + Assert.Equal("invalid_rationale", rejection!.DenyReason); + } + } + [Fact] public async Task TimeoutSeconds_accepted_and_consumed_as_meta_field() { @@ -294,7 +335,8 @@ public async Task Mcp_tools_exempt_from_native_validation() result = await executor.ExecuteAsync( new FunctionCallContent("call-mcp", "memorizer/store", new Dictionary { - ["TotallyUnknownKey"] = "value" + ["TotallyUnknownKey"] = "value", + ["_rationale"] = "Verify the MCP validation boundary." }), TestToolExecutionContext.CreateUnbound(), ct: TestContext.Current.CancellationToken); @@ -345,7 +387,12 @@ public void Mcp_conflicting_meta_spellings_rejected_as_ambiguous() public void InterpretToolCall_valid_extracts_meta_and_strips_keys() { var interp = _executor.InterpretToolCall(new FunctionCallContent("c", "shell_execute", - new Dictionary { ["Command"] = "echo hi", ["TimeoutSeconds"] = 300 })); + new Dictionary + { + ["Command"] = "echo hi", + ["TimeoutSeconds"] = 300, + ["_rationale"] = "Verify meta extraction." + })); Assert.Null(interp.Rejection); Assert.Equal(300, interp.Meta?.TimeoutHintSeconds); diff --git a/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs b/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs index 2ec8fd9404..8a98ee1353 100644 --- a/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs +++ b/src/Netclaw.Actors/Protocol/ChatMessageConverter.cs @@ -36,12 +36,13 @@ public static class ChatMessageConverter /// lookup. /// /// - /// Re-drive only. When true, the per-call meta hints persisted in + /// When true, the execution hints persisted in /// SerializableToolCall.MetaJson (stripped from ArgumentsJson at /// persistence) are re-injected as their canonical keys into the reconstructed - /// tool-call arguments, so a re-dispatched call reapplies the timeout/background/ - /// rationale instead of silently falling back to defaults. Must stay false for - /// outbound provider history — the model must never receive meta keys. + /// tool-call arguments. A re-dispatched call then reapplies the timeout and + /// background hints instead of silently using defaults. The converter always + /// restores the rationale because provider history must show the required field. + /// Keep this value false for outbound provider history. /// /// /// When set, media references whose modality is not in this mask are dropped @@ -89,19 +90,18 @@ public static AiChatMessage ToAiMessage( args = JsonSerializer.Deserialize>(tc.ArgumentsJson); } - // Re-drive only: re-inject the persisted meta hints (stripped from - // ArgumentsJson at persistence) as canonical keys, so the re-dispatched - // call's extraction reapplies the timeout/background/rationale. Outbound - // provider history leaves them out — the model must not see meta keys. - if (reinjectMeta && ToolCallMeta.Parse(tc.MetaJson) is { } meta) + if (ToolCallMeta.Parse(tc.MetaJson) is { } meta) { args ??= new Dictionary(StringComparer.Ordinal); if (meta.Rationale is not null) args["_rationale"] = meta.Rationale; - if (meta.TimeoutHintSeconds is { } timeoutSeconds) - args["_timeout_seconds"] = timeoutSeconds; - if (meta.Background) - args["_background"] = true; + if (reinjectMeta) + { + if (meta.TimeoutHintSeconds is { } timeoutSeconds) + args["_timeout_seconds"] = timeoutSeconds; + if (meta.Background) + args["_background"] = true; + } } var wireName = toolNameResolver?.Invoke(tc.Name.Value) ?? tc.Name.Value; diff --git a/src/Netclaw.Actors/Protocol/SessionOutputDto.cs b/src/Netclaw.Actors/Protocol/SessionOutputDto.cs index f42cf69940..cd932466a1 100644 --- a/src/Netclaw.Actors/Protocol/SessionOutputDto.cs +++ b/src/Netclaw.Actors/Protocol/SessionOutputDto.cs @@ -66,6 +66,7 @@ public sealed record SessionOutputDto public string? ToolName { get; init; } public string? ArgumentsJson { get; init; } public string? Result { get; init; } + public string? ToolFailureCode { get; init; } // Usage public long? InputTokens { get; init; } diff --git a/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs b/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs index 0a60f7565a..930eb2a2a1 100644 --- a/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs +++ b/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs @@ -67,7 +67,8 @@ public static class SessionOutputDtoMapper TimestampMs = msg.TimestampMs, CallId = msg.CallId.Value, ToolName = msg.ToolName.Value, - Result = msg.Result + Result = msg.Result, + ToolFailureCode = msg.FailureCode }, UsageOutput msg => new SessionOutputDto @@ -252,7 +253,8 @@ public static SessionOutput FromDto(SessionOutputDto dto) TimestampMs = dto.TimestampMs, CallId = new Netclaw.Tools.ToolCallId(dto.CallId ?? string.Empty), ToolName = new Netclaw.Tools.ToolName(dto.ToolName ?? "unknown"), - Result = dto.Result ?? string.Empty + Result = dto.Result ?? string.Empty, + FailureCode = dto.ToolFailureCode }, SessionOutputTypes.Usage => new UsageOutput { diff --git a/src/Netclaw.Actors/Sessions/LlmMessages.cs b/src/Netclaw.Actors/Sessions/LlmMessages.cs index d68c9115b2..1a345f2356 100644 --- a/src/Netclaw.Actors/Sessions/LlmMessages.cs +++ b/src/Netclaw.Actors/Sessions/LlmMessages.cs @@ -80,6 +80,7 @@ internal sealed record ToolExecutionCompleted : INoSerializationVerificationNeed public List AcceptedSubAgentFindings { get; init; } = []; public List StartedBackgroundJobs { get; init; } = []; public List ScratchCorrectionChanges { get; init; } = []; + public Dictionary ToolFailureCodes { get; init; } = new(StringComparer.Ordinal); } internal sealed record ToolExecutionSingleCompleted(ToolCallResult Result) : INoSerializationVerificationNeeded; diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index ac804cbbdc..580a7abbfb 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -924,7 +924,8 @@ private void HandleToolExecutionCompleted(ToolExecutionCompleted msg) SessionId = _sessionId, CallId = toolCallId, ToolName = new ToolName(result.Name ?? "unknown"), - Result = result.Content ?? string.Empty + Result = result.Content ?? string.Empty, + FailureCode = msg.ToolFailureCodes.GetValueOrDefault(toolCallId.Value) }, OutputFilter.ToolCalls); } @@ -4725,7 +4726,8 @@ private void ProcessToolCallResult(Pipelines.ToolCallResult result) SessionId = _sessionId, CallId = toolCallId, ToolName = new ToolName(toolMessage.Name ?? "unknown"), - Result = toolMessage.Content ?? string.Empty + Result = toolMessage.Content ?? string.Empty, + FailureCode = result.FailureCode }, OutputFilter.ToolCalls); var updatedContext = WorkingContextUpdater.UpdateFromToolResults( diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs index c5f5b5b8ba..e1304f5423 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs @@ -37,7 +37,8 @@ internal sealed record ToolCallResult( IReadOnlyList CompletedSubAgentRuns, IReadOnlyList AcceptedSubAgentFindings, Jobs.ActiveJobInfo? StartedBackgroundJob = null, - SessionScratchCorrectionChange? ScratchCorrectionChange = null); + SessionScratchCorrectionChange? ScratchCorrectionChange = null, + string? FailureCode = null); internal abstract record SessionScratchCorrectionChange { @@ -392,7 +393,16 @@ public async Task ExecuteAsync(SessionToolBatch batch) CompletedSubAgentRuns = [.. results.SelectMany(r => r.CompletedSubAgentRuns)], AcceptedSubAgentFindings = [.. results.SelectMany(r => r.AcceptedSubAgentFindings)], StartedBackgroundJobs = [.. results.Where(r => r.StartedBackgroundJob is not null).Select(r => r.StartedBackgroundJob!)], - ScratchCorrectionChanges = [.. results.Where(r => r.ScratchCorrectionChange is not null).Select(r => r.ScratchCorrectionChange!)] + ScratchCorrectionChanges = [.. results.Where(r => r.ScratchCorrectionChange is not null).Select(r => r.ScratchCorrectionChange!)], + ToolFailureCodes = results + .Where(result => result.FailureCode is not null) + .ToDictionary( + result => result.Message.ToolCallId is { } callId + ? callId.Value + : throw new InvalidOperationException("A failed tool result requires a call identity."), + result => result.FailureCode + ?? throw new InvalidOperationException("A failed tool result requires a failure code."), + StringComparer.Ordinal) }); } catch (TimeoutException ex) @@ -425,6 +435,8 @@ private async Task ExecuteSingleToolAsync( string? sessionScratchDenialDirectory, ModelInputBatchBudget modelInputBudget) { + var originalToolCall = tc; + // Single execution-preflight seam, shared with the sub-agent path via // IToolExecutor.InterpretToolCall: validate the ORIGINAL arguments (parse // sentinel, invalid/ambiguous meta values, unrecognized keys) and, on @@ -438,7 +450,7 @@ private async Task ExecuteSingleToolAsync( Content = rejection.Message, ToolCallId = new ToolCallId(tc.CallId), Name = tc.Name - }, [], [], [], []); + }, [], [], [], [], FailureCode: rejection.DenyReason); } var meta = interpretation.Meta; @@ -648,7 +660,7 @@ private async Task ExecuteSingleToolAsync( } resultText = await ExecuteToolAttemptAsync( - _executor, tc, context, timeout, _timeProvider, batch.CancellationToken); + _executor, originalToolCall, context, timeout, _timeProvider, batch.CancellationToken); sw.Stop(); } @@ -788,7 +800,7 @@ ToolAgentCorrection.SessionScratchSuggested scratchCorrection } resultText = await ExecuteToolAttemptAsync( - _executor, tc, context, timeout, _timeProvider, batch.CancellationToken); + _executor, originalToolCall, context, timeout, _timeProvider, batch.CancellationToken); sw.Stop(); } diff --git a/src/Netclaw.Actors/Sessions/Pipelines/ToolCallMetaExtractor.cs b/src/Netclaw.Actors/Sessions/Pipelines/ToolCallMetaExtractor.cs index d9fa4d64ce..56f6b10e2a 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/ToolCallMetaExtractor.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/ToolCallMetaExtractor.cs @@ -15,6 +15,10 @@ namespace Netclaw.Actors.Sessions.Pipelines; /// internal static class ToolCallMetaExtractor { + internal const string RequiredRationaleError = + "Error: Required meta argument '_rationale' must be a non-empty string. " + + "Supply one sentence that states the tool call intent. The tool was NOT executed."; + /// /// Maps a key to its canonical meta field (schema-aware for the executor, /// exact for persistence). Defaults to exact. See @@ -91,4 +95,31 @@ public static (ToolCallMeta? Meta, FunctionCallContent CleanedToolCall) Extract( return valueError; } + + /// + /// Requires a usable rationale for a new tool execution. + /// Transcript extraction remains tolerant of old calls without this field. + /// + public static string? ValidateRequiredRationale( + IDictionary? arguments, Func resolveMeta) + { + if (arguments is null || arguments.Count == 0) + return RequiredRationaleError; + + foreach (var kvp in arguments) + { + if (!string.Equals(resolveMeta(kvp.Key), "_rationale", StringComparison.Ordinal)) + continue; + + return kvp.Value switch + { + string value when !string.IsNullOrWhiteSpace(value) => null, + JsonElement { ValueKind: JsonValueKind.String } value + when !string.IsNullOrWhiteSpace(value.GetString()) => null, + _ => RequiredRationaleError + }; + } + + return RequiredRationaleError; + } } diff --git a/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs b/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs index 8fa0e46124..d8fcf46c1d 100644 --- a/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs +++ b/src/Netclaw.Actors/Sessions/SessionProtocol.Outputs.cs @@ -88,6 +88,9 @@ public sealed record ToolResultOutput : SessionOutput public required ToolName ToolName { get; init; } public required string Result { get; init; } + + /// The stable preflight failure code, or null after execution. + public string? FailureCode { get; init; } } /// diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index 7a112e2ce3..b28ae37088 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -1336,7 +1336,7 @@ executor is ISessionScratchRetryAwareExecutor scratchAwareExecutor } try { - var result = await executor.ExecuteAsync(cleanedTc, toolContext, ct); + var result = await executor.ExecuteAsync(tc, toolContext, ct); return BuildToolResult( cleanedTc, result, @@ -1435,7 +1435,7 @@ ToolAgentCorrection.SessionScratchSuggested scratchCorrection // across parallel tool calls or later iterations. var retryContext = CreatePerToolExecutionContext(executionContext, meta); retryContext.Approval.SeedOneTimeApproval(tc.Name, OneTimeApprovalKeys.Create(ctx)); - var result = await executor.ExecuteAsync(cleanedTc, retryContext, ct); + var result = await executor.ExecuteAsync(tc, retryContext, ct); return BuildToolResult( cleanedTc, result, diff --git a/src/Netclaw.Actors/Tools/DispatchingToolExecutor.cs b/src/Netclaw.Actors/Tools/DispatchingToolExecutor.cs index 500cf8ab48..ff3089cfea 100644 --- a/src/Netclaw.Actors/Tools/DispatchingToolExecutor.cs +++ b/src/Netclaw.Actors/Tools/DispatchingToolExecutor.cs @@ -86,6 +86,9 @@ public ToolCallInterpretation InterpretToolCall(FunctionCallContent toolCall) if (ValidateArguments(toolCall.Arguments, resolveMeta) is { } rejection) return rejection; + if (ToolCallMetaExtractor.ValidateRequiredRationale(toolCall.Arguments, resolveMeta) is { } rationaleError) + return new ToolArgumentRejection(rationaleError, "invalid_rationale"); + if (registered is not McpToolAdapter && ToolArgumentValidator.ValidateArgumentKeys(registered, toolCall.Arguments) is { } keyError) return new ToolArgumentRejection(keyError, "unrecognized_argument"); @@ -141,12 +144,10 @@ public async Task ExecuteAsync(FunctionCallContent toolCall, ToolExecuti return $"Unknown tool: {toolCall.Name}"; } - // Pre-dispatch validation runs before authorization so a doomed call - // never raises an approval prompt. This is the shared seam: callers that - // bypass the session pipeline (sub-agents, direct callers) get the same - // protection here. The pipeline preflights via ValidateToolCall too, so - // for that path this is a cheap idempotent re-check. - if (ValidateToolCall(toolCall) is { } rejection) + // Interpret the original call before authorization. This keeps required + // metadata available for validation and removes it before tool dispatch. + var interpretation = InterpretToolCall(toolCall); + if (interpretation.Rejection is { } rejection) { _logger.LogWarning( "Rejected tool call ({Reason}): {ToolName} — {Error}", @@ -154,6 +155,8 @@ public async Task ExecuteAsync(FunctionCallContent toolCall, ToolExecuti return rejection.Message; } + toolCall = interpretation.Cleaned; + var authorized = await GetAuthorizedToolAsync(toolCall, context, ct); var tool = authorized.Tool; @@ -228,8 +231,9 @@ public async IAsyncEnumerable ExecuteStreamAsync( yield break; } - // Same pre-authorization validation as the non-streaming path. - if (ValidateToolCall(toolCall) is { } rejection) + // Use the same atomic validation and extraction as the non-streaming path. + var interpretation = InterpretToolCall(toolCall); + if (interpretation.Rejection is { } rejection) { _logger.LogWarning( "Rejected tool call ({Reason}): {ToolName} — {Error}", @@ -238,6 +242,8 @@ public async IAsyncEnumerable ExecuteStreamAsync( yield break; } + toolCall = interpretation.Cleaned; + // Authorization throws (ToolApprovalRequiredException / ToolAccessDeniedException) // before the first item is produced; the tool-execution pipeline handles // those exactly as it does for the non-streaming path. diff --git a/src/Netclaw.Configuration.Tests/FileSystemPromptProviderAudienceTests.cs b/src/Netclaw.Configuration.Tests/FileSystemPromptProviderAudienceTests.cs index 39a08ccd17..1ea7843597 100644 --- a/src/Netclaw.Configuration.Tests/FileSystemPromptProviderAudienceTests.cs +++ b/src/Netclaw.Configuration.Tests/FileSystemPromptProviderAudienceTests.cs @@ -35,6 +35,18 @@ public void Dispose() _dir.Dispose(); } + [Theory] + [InlineData(TrustAudience.Public)] + [InlineData(TrustAudience.Team)] + [InlineData(TrustAudience.Personal)] + public void Every_audience_receives_the_tool_rationale_contract(TrustAudience audience) + { + var prompt = _provider.GetSystemPrompt(audience); + + Assert.Contains("Every tool call must include a non-empty `_rationale` string.", prompt); + Assert.Contains("Apply this rule to each parallel call", prompt); + } + [Fact] public void Public_audience_gets_stripped_agents_without_team_only_content() { diff --git a/src/Netclaw.Configuration/Resources/AGENTS.md b/src/Netclaw.Configuration/Resources/AGENTS.md index ff1c3c6ee2..986669cc98 100644 --- a/src/Netclaw.Configuration/Resources/AGENTS.md +++ b/src/Netclaw.Configuration/Resources/AGENTS.md @@ -26,6 +26,13 @@ - Use built-in `web_search` for external discovery and `web_fetch` for page retrieval. - Do not use shell HTTP clients for external search or retrieval. +## Tool Call Contract + +- Every tool call must include a non-empty `_rationale` string. +- State the call intent and reason in one sentence. +- Apply this rule to each parallel call and each later tool iteration. +- If a correction reports a missing rationale, fix every call before the retry. + ## Declaring Project Scope (load-bearing for approvals) Path arguments give the approval gate an exact candidate scope. They do not diff --git a/src/Netclaw.Configuration/Resources/AGENTS.public.md b/src/Netclaw.Configuration/Resources/AGENTS.public.md index 5aeae0df66..2e75c5fb4a 100644 --- a/src/Netclaw.Configuration/Resources/AGENTS.public.md +++ b/src/Netclaw.Configuration/Resources/AGENTS.public.md @@ -14,6 +14,13 @@ - When one approach fails, try alternatives immediately. Do not report failure without attempting at least one fallback. +## Tool Call Contract + +- Every tool call must include a non-empty `_rationale` string. +- State the call intent and reason in one sentence. +- Apply this rule to each parallel call and each later tool iteration. +- If a correction reports a missing rationale, fix every call before the retry. + ## Grounding Rules - Never state runtime facts (versions, status, availability) without checking with a tool.