-
Notifications
You must be signed in to change notification settings - Fork 0
fix: NeedsClarification for habit-flavored titles without frequency #167
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
056f335
9965381
ca5fece
6823bf4
731c382
0d4a8fb
2c082c6
b93b9d3
192e0b7
d34b57f
31bc074
919245a
b44176c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,13 @@ | ||
| using System.Text.Json; | ||
| using System.Text.Json.Nodes; | ||
| using System.Text.Json.Serialization; | ||
| using FluentValidation; | ||
| using Microsoft.AspNetCore.Authorization; | ||
| using Microsoft.AspNetCore.Mvc; | ||
| using Orbit.Api.Extensions; | ||
| using Orbit.Api.RateLimiting; | ||
| using Orbit.Application.Chat.Models; | ||
| using Orbit.Application.Common; | ||
| using Orbit.Domain.Interfaces; | ||
| using Orbit.Domain.Models; | ||
|
|
||
|
|
@@ -15,9 +20,11 @@ public class AiController( | |
| IAgentCatalogService catalogService, | ||
| IAgentPolicyEvaluator policyEvaluator, | ||
| IPendingAgentOperationStore pendingOperationStore, | ||
| IPendingClarificationStore pendingClarificationStore, | ||
| IAgentStepUpService stepUpService, | ||
| IAgentAuditService auditService, | ||
| IAgentOperationExecutor operationExecutor) : ControllerBase | ||
| IAgentOperationExecutor operationExecutor, | ||
| IValidator<ResolveClarificationRequest> resolveClarificationValidator) : ControllerBase | ||
| { | ||
| [HttpGet("capabilities")] | ||
| public async Task<IActionResult> GetCapabilitiesMetadata(CancellationToken cancellationToken) | ||
|
|
@@ -294,4 +301,196 @@ await auditService.RecordAsync(new AgentAuditEntry( | |
|
|
||
| return Ok(result); | ||
| } | ||
|
|
||
| [HttpPost("clarifications/{operationId:guid}/resolve")] | ||
| [DistributedRateLimit("ai-resolve")] | ||
| public async Task<IActionResult> ResolveClarification( | ||
|
thomasluizon marked this conversation as resolved.
|
||
| Guid operationId, | ||
| [FromBody] ResolveClarificationRequest body, | ||
| CancellationToken cancellationToken) | ||
| { | ||
| // Clarification cards are a UI-only flow — API-key clients can't render or tap them. | ||
| // Mirrors the guard on ExecutePendingOperation and the step-up endpoints. | ||
| var authMethod = HttpContext.User.GetAgentAuthMethod(); | ||
| if (authMethod == AgentAuthMethod.ApiKey) | ||
| return Forbid(); | ||
|
|
||
| var userId = HttpContext.GetUserId(); | ||
|
|
||
| var validation = await resolveClarificationValidator.ValidateAsync(body, cancellationToken); | ||
| if (!validation.IsValid) | ||
| { | ||
| var firstError = validation.Errors[0]; | ||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| AgentPolicyDecisionStatus.Denied, | ||
| AgentOperationStatus.Failed, | ||
| $"invalid_clarification_value:{firstError.PropertyName}", | ||
| cancellationToken); | ||
| return BadRequest(new { error = firstError.ErrorMessage }); | ||
| } | ||
|
|
||
| var pending = await pendingClarificationStore.GetForResolutionAsync(operationId, userId, cancellationToken); | ||
| if (pending is null) | ||
| { | ||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| AgentPolicyDecisionStatus.Denied, | ||
| AgentOperationStatus.Failed, | ||
| "clarification_not_found", | ||
| cancellationToken); | ||
| return NotFound(new { error = ErrorMessages.ClarificationNotFound }); | ||
| } | ||
|
|
||
| // The patch must be one of the server-offered quick-action values. This is a | ||
| // defense-in-depth check: prevents a malicious client from hand-crafting a patch | ||
| // that overrides fields the contract never said could be changed. | ||
| if (!pending.AllowedValues.Contains(body.Value, StringComparer.Ordinal)) | ||
|
thomasluizon marked this conversation as resolved.
|
||
| { | ||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| AgentPolicyDecisionStatus.Denied, | ||
| AgentOperationStatus.Failed, | ||
| "clarification_value_not_offered", | ||
| cancellationToken); | ||
| return BadRequest(new { error = ErrorMessages.ClarificationValueNotOffered }); | ||
| } | ||
|
|
||
| JsonElement mergedArgs; | ||
| try | ||
| { | ||
| mergedArgs = MergeClarificationValue(pending.PartialArgumentsJson, body.Value); | ||
| } | ||
| catch (JsonException) | ||
| { | ||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| AgentPolicyDecisionStatus.Denied, | ||
| AgentOperationStatus.Failed, | ||
| "invalid_clarification_value", | ||
| cancellationToken); | ||
| return BadRequest(new { error = ErrorMessages.ClarificationValueNotJsonObject }); | ||
| } | ||
|
|
||
| // Atomic one-shot claim: if this returns false, either another concurrent request | ||
| // already marked the row resolved OR the row expired in the (typically sub-ms) | ||
| // window between Get and MarkResolved. Bail before re-invoking. | ||
| // | ||
| // The clarification is intentionally consumed BEFORE ExecuteAsync runs. If the | ||
| // executor throws or the tool returns Failed/Denied, the clarification is gone — | ||
| // the user must re-initiate the request via chat. This is acceptable because: | ||
| // (a) the alternative (un-claim on failure) reopens TOCTOU races on retry, | ||
| // (b) tool Failed/Denied is surfaced in the response so the client can prompt | ||
| // the user appropriately, | ||
| // (c) re-asking in chat is a natural recovery path the user already understands. | ||
| var claimed = await pendingClarificationStore.MarkResolvedAsync(operationId, userId, cancellationToken); | ||
| if (!claimed) | ||
| { | ||
| var auditError = pending.ExpiresAtUtc <= DateTime.UtcNow | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
In practice this race is negligibly rare (two requests resolve the same clarification within 1 ms of the TTL boundary), so it doesn't affect correctness — only the audit label. Worth noting if audit data is ever used for metrics. |
||
| ? "clarification_expired_mid_flight" | ||
| : "clarification_already_resolved"; | ||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| AgentPolicyDecisionStatus.Denied, | ||
| AgentOperationStatus.Failed, | ||
| auditError, | ||
| cancellationToken); | ||
| return Conflict(new { error = ErrorMessages.ClarificationAlreadyResolved }); | ||
|
thomasluizon marked this conversation as resolved.
|
||
| } | ||
|
|
||
| var result = await operationExecutor.ExecuteAsync(new AgentExecuteOperationRequest( | ||
|
thomasluizon marked this conversation as resolved.
|
||
| userId, | ||
| pending.ToolName, | ||
| mergedArgs, | ||
| AgentExecutionSurface.Chat, | ||
| authMethod, | ||
| HttpContext.User.GetGrantedAgentScopes(), | ||
| HttpContext.User.IsReadOnlyCredential(), | ||
| ConfirmationToken: null, | ||
| HttpContext.TraceIdentifier), cancellationToken); | ||
|
|
||
| await RecordResolveAuditAsync( | ||
| userId, | ||
| authMethod, | ||
| operationId, | ||
| result.Operation.Status == AgentOperationStatus.Succeeded | ||
| ? AgentPolicyDecisionStatus.Allowed | ||
| : AgentPolicyDecisionStatus.Denied, | ||
| result.Operation.Status, | ||
| result.Operation.PolicyReason, | ||
| cancellationToken, | ||
| targetName: result.Operation.TargetName); | ||
|
|
||
| return Ok(result); | ||
|
thomasluizon marked this conversation as resolved.
thomasluizon marked this conversation as resolved.
thomasluizon marked this conversation as resolved.
|
||
| } | ||
|
|
||
| private Task RecordResolveAuditAsync( | ||
| Guid userId, | ||
| AgentAuthMethod authMethod, | ||
| Guid operationId, | ||
| AgentPolicyDecisionStatus policyDecision, | ||
| AgentOperationStatus outcome, | ||
| string? error, | ||
| CancellationToken cancellationToken, | ||
| string? targetName = null) | ||
| { | ||
| return auditService.RecordAsync(new AgentAuditEntry( | ||
| userId, | ||
| AgentCapabilityIds.ChatInteract, | ||
| nameof(ResolveClarification), | ||
| AgentExecutionSurface.Chat, | ||
| authMethod, | ||
| AgentRiskClass.Low, | ||
| policyDecision, | ||
| outcome, | ||
| HttpContext.TraceIdentifier, | ||
| "Resolve clarification", | ||
| TargetId: operationId.ToString(), | ||
| TargetName: targetName, | ||
| Error: error), cancellationToken); | ||
| } | ||
|
|
||
| private static JsonElement MergeClarificationValue(string baseJson, string value) | ||
| { | ||
| // Fail closed if the stored args aren't a JSON object — silently coercing to {} | ||
| // would drop the original tool arguments and replay the tool with only the patch. | ||
| if (JsonNode.Parse(baseJson) is not JsonObject baseNode) | ||
| throw new JsonException("Stored partial arguments are not a JSON object."); | ||
|
|
||
|
thomasluizon marked this conversation as resolved.
|
||
| if (!string.IsNullOrWhiteSpace(value)) | ||
| { | ||
| if (JsonNode.Parse(value) is not JsonObject patchNode) | ||
| throw new JsonException("Clarification value must be a JSON object."); | ||
|
|
||
| DeepMerge(baseNode, patchNode); | ||
| } | ||
|
thomasluizon marked this conversation as resolved.
|
||
|
|
||
| return JsonDocument.Parse(baseNode.ToJsonString()).RootElement.Clone(); | ||
| } | ||
|
|
||
| // Deep merge: nested JsonObjects recurse instead of clobbering. | ||
| private static void DeepMerge(JsonObject target, JsonObject patch) | ||
|
thomasluizon marked this conversation as resolved.
|
||
| { | ||
| foreach (var kvp in patch.ToList()) | ||
| { | ||
| if (target[kvp.Key] is JsonObject targetChild && kvp.Value is JsonObject patchChild) | ||
| { | ||
| DeepMerge(targetChild, patchChild); | ||
| } | ||
| else | ||
| { | ||
| target[kvp.Key] = kvp.Value?.DeepClone(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -3,6 +3,7 @@ | |||||||||||||||||||||||||
| using MediatR; | ||||||||||||||||||||||||||
| using Microsoft.EntityFrameworkCore; | ||||||||||||||||||||||||||
| using Microsoft.Extensions.Logging; | ||||||||||||||||||||||||||
| using Orbit.Application.Chat.Models; | ||||||||||||||||||||||||||
| using Orbit.Application.Chat.Tools; | ||||||||||||||||||||||||||
| using Orbit.Application.Common; | ||||||||||||||||||||||||||
| using Orbit.Domain.Common; | ||||||||||||||||||||||||||
|
|
@@ -42,9 +43,10 @@ public record ActionResult( | |||||||||||||||||||||||||
| string? EntityName = null, | ||||||||||||||||||||||||||
| string? Error = null, | ||||||||||||||||||||||||||
| string? Field = null, | ||||||||||||||||||||||||||
| IReadOnlyList<AiAction>? SuggestedSubHabits = null); | ||||||||||||||||||||||||||
| IReadOnlyList<AiAction>? SuggestedSubHabits = null, | ||||||||||||||||||||||||||
| ClarificationRequest? ClarificationRequest = null); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| public enum ActionStatus { Success, Failed, Suggestion } | ||||||||||||||||||||||||||
| public enum ActionStatus { Success, Failed, Suggestion, NeedsClarification } | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| /// <summary> | ||||||||||||||||||||||||||
| /// Groups AI-related dependencies to reduce constructor parameter count (S107). | ||||||||||||||||||||||||||
|
|
@@ -76,7 +78,8 @@ public record ChatExecutionDependencies( | |||||||||||||||||||||||||
| IPayGateService PayGateService, | ||||||||||||||||||||||||||
| IUnitOfWork UnitOfWork, | ||||||||||||||||||||||||||
| IServiceScopeFactory ServiceScopeFactory, | ||||||||||||||||||||||||||
| IAgentOperationExecutor OperationExecutor); | ||||||||||||||||||||||||||
| IAgentOperationExecutor OperationExecutor, | ||||||||||||||||||||||||||
| IPendingClarificationStore PendingClarificationStore); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| public partial class ProcessUserChatCommandHandler( | ||||||||||||||||||||||||||
| ChatDataDependencies data, | ||||||||||||||||||||||||||
|
|
@@ -363,11 +366,70 @@ public async Task<Result<ChatResponse>> Handle( | |||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| var operationResult = executionResponse.Operation; | ||||||||||||||||||||||||||
| var toolResult = BuildToolCallResult(call, operationResult); | ||||||||||||||||||||||||||
| var isClarification = operationResult.Payload is NeedsClarificationPayload; | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| if (operationResult.Status == AgentOperationStatus.Succeeded) | ||||||||||||||||||||||||||
| if (operationResult.Status == AgentOperationStatus.Succeeded && !isClarification) | ||||||||||||||||||||||||||
| LogToolSucceeded(logger, call.Name, operationResult.TargetName); | ||||||||||||||||||||||||||
| else if (operationResult.Status is AgentOperationStatus.Failed or AgentOperationStatus.Denied) | ||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||
| LogToolFailed(logger, call.Name, operationResult.PolicyReason); | ||||||||||||||||||||||||||
|
thomasluizon marked this conversation as resolved.
thomasluizon marked this conversation as resolved.
|
||||||||||||||||||||||||||
| if (isClarification) | ||||||||||||||||||||||||||
| LogClarificationDroppedOnFailedTool(logger, call.Name, operationResult.PolicyReason); | ||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| if (operationResult.Status == AgentOperationStatus.Succeeded | ||||||||||||||||||||||||||
| && operationResult.Payload is NeedsClarificationPayload payload) | ||||||||||||||||||||||||||
|
thomasluizon marked this conversation as resolved.
|
||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||
| // Serialize null as an empty array literal — JsonSerializer.Serialize(null) | ||||||||||||||||||||||||||
| // returns the string "null" which bypasses PendingClarification.Create's | ||||||||||||||||||||||||||
| // "[]" default and would break ExtractQuickActionValues on read. | ||||||||||||||||||||||||||
| var quickActionsJson = payload.QuickActions is null | ||||||||||||||||||||||||||
| ? "[]" | ||||||||||||||||||||||||||
| : JsonSerializer.Serialize(payload.QuickActions); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| // Cap the stashed args so a runaway tool argument can't bloat the table. | ||||||||||||||||||||||||||
| // 16 KB covers realistic create_habit calls (a few sub_habits with checklists | ||||||||||||||||||||||||||
| // and descriptions); larger blows past expected payloads. | ||||||||||||||||||||||||||
| var partialArgsJson = call.Args.GetRawText(); | ||||||||||||||||||||||||||
| if (partialArgsJson.Length > AppConstants.MaxClarificationArgsLength) | ||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||
| LogClarificationArgsTooLarge(logger, call.Name, partialArgsJson.Length); | ||||||||||||||||||||||||||
| return ( | ||||||||||||||||||||||||||
| toolResult, | ||||||||||||||||||||||||||
| new ActionResult( | ||||||||||||||||||||||||||
| ToolNameToPascalCase(call.Name), | ||||||||||||||||||||||||||
| ActionStatus.Failed, | ||||||||||||||||||||||||||
| Error: "Tool arguments exceeded the clarification stash limit."), | ||||||||||||||||||||||||||
| operationResult, | ||||||||||||||||||||||||||
| executionResponse.PolicyDenial, | ||||||||||||||||||||||||||
| executionResponse.PendingOperation); | ||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| var stashedId = await execution.PendingClarificationStore.CreateAsync( | ||||||||||||||||||||||||||
| request.UserId, | ||||||||||||||||||||||||||
| call.Name, | ||||||||||||||||||||||||||
| partialArgsJson, | ||||||||||||||||||||||||||
| payload.MissingArgumentKey, | ||||||||||||||||||||||||||
| payload.Question, | ||||||||||||||||||||||||||
| quickActionsJson, | ||||||||||||||||||||||||||
| cancellationToken); | ||||||||||||||||||||||||||
| LogClarificationRequested(logger, call.Name, stashedId, payload.MissingArgumentKey); | ||||||||||||||||||||||||||
| var clarification = new ClarificationRequest( | ||||||||||||||||||||||||||
| payload.Question, | ||||||||||||||||||||||||||
| stashedId, | ||||||||||||||||||||||||||
| payload.MissingArgumentKey, | ||||||||||||||||||||||||||
| payload.QuickActions ?? Array.Empty<QuickAction>()); | ||||||||||||||||||||||||||
| return ( | ||||||||||||||||||||||||||
| toolResult, | ||||||||||||||||||||||||||
| new ActionResult( | ||||||||||||||||||||||||||
| ToolNameToPascalCase(call.Name), | ||||||||||||||||||||||||||
| ActionStatus.NeedsClarification, | ||||||||||||||||||||||||||
| EntityName: call.Name, | ||||||||||||||||||||||||||
| ClarificationRequest: clarification), | ||||||||||||||||||||||||||
| operationResult, | ||||||||||||||||||||||||||
| executionResponse.PolicyDenial, | ||||||||||||||||||||||||||
| executionResponse.PendingOperation); | ||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| return operationResult.Status switch | ||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||
|
|
@@ -760,6 +822,15 @@ private static async Task IncrementAiMessageCountAsync( | |||||||||||||||||||||||||
| [LoggerMessage(EventId = 22, Level = LogLevel.Warning, Message = "Background message counter increment failed")] | ||||||||||||||||||||||||||
| private static partial void LogBackgroundMessageCounterFailed(ILogger logger, Exception ex); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| [LoggerMessage(EventId = 24, Level = LogLevel.Information, Message = "Tool {Name} requested clarification (operationId={OperationId}, missing={MissingKey})")] | ||||||||||||||||||||||||||
| private static partial void LogClarificationRequested(ILogger logger, string name, Guid operationId, string missingKey); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| [LoggerMessage(EventId = 25, Level = LogLevel.Warning, Message = "Tool {Name} emitted a clarification payload on a Failed/Denied result and it was dropped: {Reason}")] | ||||||||||||||||||||||||||
| private static partial void LogClarificationDroppedOnFailedTool(ILogger logger, string name, string? reason); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| [LoggerMessage(EventId = 26, Level = LogLevel.Warning, Message = "Tool {Name} requested clarification with oversized partial args ({Length} chars) — dropped without stashing")] | ||||||||||||||||||||||||||
| private static partial void LogClarificationArgsTooLarge(ILogger logger, string name, int length); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| [LoggerMessage(EventId = 23, Level = LogLevel.Warning, Message = "Background post-response work failed")] | ||||||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. EventId 23 is declared here but EventIds 24, 25, and 26 are declared on lines 825–831 above it. The numeric sequence in the file reads 22 → 24 → 25 → 26 → 23, which makes it look like EventId 23 is missing when scanning the file. Swap the declaration order so the sequence reads 22 → 23 → 24 → 25 → 26, or at minimum add a comment explaining the ordering.
Suggested change
|
||||||||||||||||||||||||||
| private static partial void LogBackgroundPostResponseFailed(ILogger logger, Exception ex); | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| namespace Orbit.Application.Chat.Models; | ||
|
|
||
| /// <summary> | ||
| /// External-facing "I need to ask the user a question" payload. The frontend renders | ||
| /// quick-action buttons; tapping one POSTs the chosen <c>QuickAction.Value</c> to | ||
| /// <c>POST /api/ai/clarifications/{OperationId}/resolve</c>, which merges the value | ||
| /// into the partial arguments stash and re-invokes the original tool deterministically. | ||
| /// <para> | ||
| /// Tools do not construct this directly — they return a <see cref="NeedsClarificationPayload"/> | ||
| /// and the chat handler attaches the store-minted <c>OperationId</c> when building the | ||
| /// outbound <c>ActionResult</c>. | ||
| /// </para> | ||
| /// </summary> | ||
| public record ClarificationRequest( | ||
| string Question, | ||
| Guid OperationId, | ||
| string MissingArgumentKey, | ||
| IReadOnlyList<QuickAction> QuickActions); | ||
|
thomasluizon marked this conversation as resolved.
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| namespace Orbit.Application.Chat.Models; | ||
|
|
||
| /// <summary> | ||
| /// Internal payload returned by a tool that wants to ask the user a question instead | ||
| /// of executing. The chat handler stashes this server-side, mints an <c>OperationId</c> | ||
| /// from the store, and constructs the external-facing <see cref="ClarificationRequest"/> | ||
| /// before surfacing it to the frontend. | ||
| /// <para> | ||
| /// Tools should not attempt to populate an <c>OperationId</c> themselves — the handler | ||
| /// owns that field. Keeping the tool's output free of the id avoids a <c>Guid.Empty</c> | ||
| /// sentinel leaking into the contract. | ||
| /// </para> | ||
| /// </summary> | ||
| public record NeedsClarificationPayload( | ||
| string Question, | ||
| string MissingArgumentKey, | ||
| IReadOnlyList<QuickAction>? QuickActions); |
Uh oh!
There was an error while loading. Please reload this page.