diff --git a/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs b/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs index 07e8d1a7..6791a0ce 100644 --- a/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs +++ b/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs @@ -13,4 +13,5 @@ public static class SystemToolNames public const string ReadTabularData = "read_tabular_data"; public const string GenerateImage = "generate_image"; public const string GenerateChart = "generate_chart"; + public const string InspectImage = "inspect_image"; } diff --git a/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md b/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md index 854f76bf..acf4653d 100644 --- a/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md +++ b/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md @@ -74,3 +74,4 @@ description: Initial standalone release notes for the CrestApps.Core repository. - adds `CrestApps.Core.PostgreSQL` and `CrestApps.Core.AI.PostgreSQL` packages providing a lightweight PostgreSQL + pgvector vector search backend as an alternative to Elasticsearch and Azure AI Search, registers the same keyed services (`ISearchIndexManager`, `ISearchDocumentManager`, `IDataSourceContentManager`, `IDataSourceDocumentReader`, `IODataFilterTranslator`) under the `"PostgreSQL"` provider name, supports `AddAIDocuments()`, `AddAIDataSources()`, and `AddAIMemory()` builder extensions, and integrates into both MVC and Blazor sample hosts - fixes hosted document and data-source indexing flows so background workers create a scoped service provider before resolving scoped indexing services, preventing upload-triggered failures and similar nightly alignment lifetime issues - standardizes Azure AI Search configuration on top-level `AuthenticationType`, `ApiKey`, `IdentityClientId`, and `IndexPrefix` settings under `CrestApps:AzureAISearch`, and refreshes the sample host / docs examples to list the full supported option set in one place +- replaces per-turn raw image byte injection with an analyze-once-at-upload strategy: `IImageAnalysisService` calls a vision model to extract caption, OCR text, and detected entities when images are uploaded, stores the results as `AIDocumentChunk` records searchable via `read_document` and `search_documents`, adds `inspect_image` as an on-demand tool for pixel-level inspection when the text analysis is insufficient, removes `BuildVisionUserContentsAsync` from `DocumentOrchestrationHandler` so image bytes are never attached to every user message, and updates the document-availability prompt to guide the model toward text-based tools first diff --git a/src/CrestApps.Core.Docs/docs/core/ai-documents.md b/src/CrestApps.Core.Docs/docs/core/ai-documents.md index 92abb357..2f3d98a7 100644 --- a/src/CrestApps.Core.Docs/docs/core/ai-documents.md +++ b/src/CrestApps.Core.Docs/docs/core/ai-documents.md @@ -68,9 +68,11 @@ Users upload documents (PDFs, Word files, spreadsheets, text files) and expect t The document processing system handles this full pipeline from upload to retrieval, while the built-in document tools make the content available to the AI during orchestration. -When a chat deployment also supports the `Vision` purpose, chat interaction and chat session uploads can include supported image formats (`.bmp`, `.gif`, `.jpeg`, `.jpg`, `.png`, `.webp`) alongside standard document files. Those images are stored as `AIDocument` records and attached to the current user message as multimodal content instead of going through text extraction and chunk embedding. The shared document-availability prompt distinguishes image attachments from searchable documents so the model analyzes supported uploaded images directly instead of treating them like text-only document metadata. +When a chat deployment also supports the `Vision` purpose, chat interaction and chat session uploads can include supported image formats (`.bmp`, `.gif`, `.jpeg`, `.jpg`, `.png`, `.webp`) alongside standard document files. Those images are stored as `AIDocument` records, analyzed at upload time by `IImageAnalysisService` to extract a structured summary (caption, OCR text, detected entities), and the results are persisted as `AIDocumentChunk` records — exactly like text documents. This makes image content available through the same `read_document` and `search_documents` tools used for regular documents. -`DocumentOrchestrationHandler` still has to materialize image bytes before building `DataContent`, but it now reads directly into the target byte buffer and honors `ChatDocumentsOptions.MaxVisionInputBytesPerRequest` so a single request cannot pull an unbounded batch of uploaded images into memory. +For cases where the text analysis is insufficient (e.g., reading fine text, comparing visual elements, or understanding spatial layout), the `inspect_image` tool provides on-demand raw image inspection by sending the original bytes to a vision model in a one-shot call. This approach eliminates the cost of attaching raw image bytes to every chat request while preserving full visual inspection capability when needed. + +The `ChatDocumentsOptions.AnalyzeImagesAtUpload` setting controls whether analysis runs at upload time, and `MaxInspectImageCallsPerRequest` limits how many costly raw-image inspections the model can perform per turn. ### Creating a chat client to describe an image @@ -144,6 +146,7 @@ public sealed class ImageDescriptionService( │ • SearchDocumentsTool (vector RAG) │ │ • ReadDocumentTool (full text read) │ │ • ReadTabularDataTool (CSV/Excel) │ + │ • InspectImageTool (vision on-demand)│ └──────────────┬──────────────────────┘ ▼ ┌─────────────────────────────────────┐ diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs index 1328c6c3..61c2dbd1 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs @@ -26,12 +26,15 @@ public abstract class AIChatDocumentEndpointBase string referenceType, ChatDocumentsOptions documentOptions, IAIDocumentProcessingService documentProcessingService, + IImageAnalysisService imageAnalysisService, IEmbeddingGenerator> embeddingGenerator, IAIDocumentStore documentStore, IAIDocumentChunkStore chunkStore, IDocumentFileStore fileStore, TimeProvider timeProvider, bool allowVisionImages, + bool allowDocumentUploads, + string chatDeploymentName, ILogger logger, IStringLocalizer S) { @@ -49,7 +52,12 @@ public abstract class AIChatDocumentEndpointBase if (MediaTypeHelper.IsVisionImageExtension(extension) && !allowVisionImages) { - return (false, S["Image uploads require a vision-capable chat deployment."].Value, null); + return (false, S["Image uploads are not enabled or no vision deployment is configured."].Value, null); + } + + if (!MediaTypeHelper.IsVisionImageExtension(extension) && !allowDocumentUploads) + { + return (false, S["Document uploads are not enabled."].Value, null); } if (allowVisionImages && MediaTypeHelper.IsVisionImageExtension(extension) && @@ -58,7 +66,7 @@ public abstract class AIChatDocumentEndpointBase return (false, S["The uploaded image exceeds the maximum allowed size of {0} MB.", documentOptions.MaxVisionImageBytesPerFile / (1024 * 1024)].Value, null); } - if (!documentOptions.IsAllowedFileExtension(extension, allowVisionImages)) + if (!documentOptions.IsAllowedFileExtension(extension, allowVisionImages, allowDocumentUploads)) { return (false, S["File type '{0}' is not supported.", extension].Value, null); } @@ -74,7 +82,19 @@ public abstract class AIChatDocumentEndpointBase { if (allowVisionImages && MediaTypeHelper.IsVisionImageExtension(extension)) { - return await ProcessVisionImageAsync(file, referenceId, referenceType, documentStore, fileStore, timeProvider); + return await ProcessVisionImageAsync( + file, + referenceId, + referenceType, + documentOptions, + imageAnalysisService, + embeddingGenerator, + documentStore, + chunkStore, + fileStore, + timeProvider, + chatDeploymentName, + logger); } var result = await documentProcessingService.ProcessFileAsync(file, referenceId, referenceType, embeddingGenerator); @@ -89,15 +109,6 @@ public abstract class AIChatDocumentEndpointBase } } - /// - /// Determines whether the deployment supports vision uploads. - /// - /// The deployment. - protected static bool SupportsVisionUploads(AIDeployment deployment) - { - return deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; - } - /// /// Gets files. /// @@ -115,6 +126,17 @@ protected static IReadOnlyList GetFiles(IFormCollection form) return singleFile == null ? [] : [singleFile]; } + /// + /// Determines whether session document upload is enabled for the profile. + /// Returns true when either document uploads or image uploads are allowed. + /// + /// The profile. + protected static bool IsSessionUploadEnabled(AIProfile profile) + { + return profile.TryGet(out var metadata) + && (metadata.AllowSessionDocuments || metadata.AllowSessionImageUploads); + } + /// /// Determines whether session document upload enabled. /// @@ -227,9 +249,15 @@ protected static async Task InvokeRemovedHandlersAsync(IEnumerable> embeddingGenerator, IAIDocumentStore documentStore, + IAIDocumentChunkStore chunkStore, IDocumentFileStore fileStore, - TimeProvider timeProvider) + TimeProvider timeProvider, + string chatDeploymentName, + ILogger logger) { var now = timeProvider.GetUtcNow().UtcDateTime; var contentType = MediaTypeHelper.InferMediaType(Path.GetExtension(file.FileName), file.ContentType); @@ -269,11 +297,137 @@ protected static async Task InvokeRemovedHandlersAsync(IEnumerable chunks = []; + + if (documentOptions.AnalyzeImagesAtUpload && imageAnalysisService != null) + { + chunks = await AnalyzeAndStoreImageChunksAsync( + file, + document, + imageAnalysisService, + embeddingGenerator, + chunkStore, + chatDeploymentName, + logger); + } + return (true, null, new AIChatUploadedDocument { File = file, Document = document, DocumentInfo = documentInfo, + Chunks = chunks, }); } + + private static async Task> AnalyzeAndStoreImageChunksAsync( + IFormFile file, + AIDocument document, + IImageAnalysisService imageAnalysisService, + IEmbeddingGenerator> embeddingGenerator, + IAIDocumentChunkStore chunkStore, + string chatDeploymentName, + ILogger logger) + { + try + { + ImageAnalysisResult analysis; + + using (var stream = file.OpenReadStream()) + { + analysis = await imageAnalysisService.AnalyzeAsync( + stream, + document.ContentType, + document.FileName, + chatDeploymentName); + } + + if (!analysis.Success) + { + logger.LogWarning( + "Image analysis failed for document '{DocumentId}' ({FileName}): {Error}", + document.ItemId, + document.FileName, + analysis.Error); + + return []; + } + + var chunks = BuildImageAnalysisChunks(document, analysis); + + if (embeddingGenerator != null && chunks.Count > 0) + { + var textsToEmbed = chunks.Select(c => c.Content).ToList(); + var embeddings = await embeddingGenerator.GenerateAsync(textsToEmbed); + + for (var i = 0; i < chunks.Count && i < embeddings.Count; i++) + { + chunks[i].Embedding = embeddings[i].Vector.ToArray(); + } + } + + foreach (var chunk in chunks) + { + await chunkStore.CreateAsync(chunk); + } + + return chunks; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to analyze image '{FileName}' for document '{DocumentId}'.", document.FileName, document.ItemId); + + return []; + } + } + + private static List BuildImageAnalysisChunks(AIDocument document, ImageAnalysisResult analysis) + { + var chunks = new List(); + var index = 0; + + if (!string.IsNullOrWhiteSpace(analysis.Caption)) + { + chunks.Add(CreateChunk(document, $"[Image Caption]\n{analysis.Caption}", index++)); + } + + if (!string.IsNullOrWhiteSpace(analysis.Description)) + { + chunks.Add(CreateChunk(document, $"[Image Description]\n{analysis.Description}", index++)); + } + + if (!string.IsNullOrWhiteSpace(analysis.OcrText) && + !string.Equals(analysis.OcrText, "None", StringComparison.OrdinalIgnoreCase)) + { + chunks.Add(CreateChunk(document, $"[OCR Text]\n{analysis.OcrText}", index++)); + } + + if (!string.IsNullOrWhiteSpace(analysis.DetectedEntities) && + !string.Equals(analysis.DetectedEntities, "None", StringComparison.OrdinalIgnoreCase)) + { + chunks.Add(CreateChunk(document, $"[Detected Entities]\n{analysis.DetectedEntities}", index++)); + } + + // If no structured sections were produced, store the raw analysis as a single chunk. + if (chunks.Count == 0 && !string.IsNullOrWhiteSpace(analysis.RawAnalysis)) + { + chunks.Add(CreateChunk(document, analysis.RawAnalysis, index)); + } + + return chunks; + } + + private static AIDocumentChunk CreateChunk(AIDocument document, string content, int index) + { + return new AIDocumentChunk + { + ItemId = UniqueId.GenerateId(), + AIDocumentId = document.ItemId, + ReferenceId = document.ReferenceId, + ReferenceType = document.ReferenceType, + Content = content, + Index = index, + }; + } } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatInteractionDocument.cs b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatInteractionDocument.cs index 3407754f..9b5cf609 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatInteractionDocument.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatInteractionDocument.cs @@ -53,6 +53,7 @@ private sealed class UploadChatInteractionDocumentEndpoint : AIChatDocumentEndpo /// The chunk store. /// The file store. /// The document processing service. + /// The image analysis service. /// The authorization service. /// The event handlers. /// The document options. @@ -67,10 +68,12 @@ public static async Task HandleAsync( [FromServices] IAIDocumentChunkStore chunkStore, [FromServices] IDocumentFileStore fileStore, [FromServices] IAIDocumentProcessingService documentProcessingService, + [FromServices] IImageAnalysisService imageAnalysisService, [FromServices] TimeProvider timeProvider, [FromServices] IAuthorizationService authorizationService, [FromServices] IEnumerable eventHandlers, [FromServices] IOptions documentOptions, + [FromServices] IOptions interactionDocumentOptions, [FromServices] ILoggerFactory loggerFactory, [FromServices] IStringLocalizerFactory localizerFactory) { @@ -123,7 +126,10 @@ public static async Task HandleAsync( } var embeddingGenerator = embeddingDeployment == null ? null : await aiClientFactory.CreateEmbeddingGeneratorAsync(embeddingDeployment); - var allowVisionImages = SupportsVisionUploads(deployment); + var visionDeployment = await deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision); + var interactionDocOptions = interactionDocumentOptions.Value; + var allowVisionImages = interactionDocOptions.AllowImageUploads && visionDeployment != null; + var allowDocumentUploads = interactionDocOptions.AllowDocumentUploads; if (logger.IsEnabled(LogLevel.Information)) { logger.LogInformation("Created embedding generator for interaction '{InteractionId}': {HasEmbeddingGenerator}.", interaction.ItemId, embeddingGenerator != null); @@ -156,12 +162,15 @@ public static async Task HandleAsync( AIReferenceTypes.Document.ChatInteraction, documentOptions.Value, documentProcessingService, + imageAnalysisService, embeddingGenerator, documentStore, chunkStore, fileStore, timeProvider, allowVisionImages, + allowDocumentUploads, + visionDeployment?.Name, logger, S); if (!result.Success) diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatSessionDocument.cs b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatSessionDocument.cs index 310bdc4a..29b68ec6 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatSessionDocument.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatSessionDocument.cs @@ -55,6 +55,7 @@ private sealed class UploadChatSessionDocumentEndpoint : AIChatDocumentEndpointB /// The chunk store. /// The file store. /// The document processing service. + /// The image analysis service. /// The authorization service. /// The event handlers. /// The document options. @@ -70,6 +71,7 @@ public static async Task HandleAsync( [FromServices] IAIDocumentChunkStore chunkStore, [FromServices] IDocumentFileStore fileStore, [FromServices] IAIDocumentProcessingService documentProcessingService, + [FromServices] IImageAnalysisService imageAnalysisService, [FromServices] TimeProvider timeProvider, [FromServices] IAuthorizationService authorizationService, [FromServices] IEnumerable eventHandlers, @@ -112,9 +114,9 @@ public static async Task HandleAsync( return TypedResults.NotFound(); } - if (!IsSessionDocumentUploadEnabled(profile)) + if (!IsSessionUploadEnabled(profile)) { - return TypedResults.BadRequest("Session document uploads are not enabled for this AI profile."); + return TypedResults.BadRequest("Session document or image uploads are not enabled for this AI profile."); } session = await sessionManager.NewAsync(profile, new NewAIChatSessionContext()); @@ -129,9 +131,9 @@ public static async Task HandleAsync( return TypedResults.NotFound(); } - if (!IsSessionDocumentUploadEnabled(profile)) + if (!IsSessionUploadEnabled(profile)) { - return TypedResults.BadRequest("Session document uploads are not enabled for this AI profile."); + return TypedResults.BadRequest("Session document or image uploads are not enabled for this AI profile."); } var authorization = await authorizationService.AuthorizeAsync( @@ -146,7 +148,10 @@ public static async Task HandleAsync( var deployment = await ResolveSessionDeploymentAsync(profile, deploymentManager); var embeddingDeployment = await deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Embedding, clientName: deployment?.ClientName); var embeddingGenerator = embeddingDeployment == null ? null : await aiClientFactory.CreateEmbeddingGeneratorAsync(embeddingDeployment); - var allowVisionImages = SupportsVisionUploads(deployment); + + profile.TryGet(out var sessionDocMetadata); + var visionDeployment = await deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision); + var allowVisionImages = sessionDocMetadata?.AllowSessionImageUploads == true && visionDeployment != null; var logger = loggerFactory.CreateLogger("AIChatDocumentEndpoints"); var S = localizerFactory.Create(typeof(AIChatDocumentEndpointBase)); session.Documents ??= []; @@ -170,12 +175,15 @@ public static async Task HandleAsync( AIReferenceTypes.Document.ChatSession, documentOptions.Value, documentProcessingService, + imageAnalysisService, embeddingGenerator, documentStore, chunkStore, fileStore, timeProvider, allowVisionImages, + sessionDocMetadata?.AllowSessionDocuments == true, + visionDeployment?.Name, logger, S); if (!result.Success) diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs index a066135a..2af770bd 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs @@ -1,11 +1,9 @@ using CrestApps.Core.AI.Completions; -using CrestApps.Core.AI.Deployments; using CrestApps.Core.AI.Documents.Models; using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Orchestration; using CrestApps.Core.AI.Tooling; using CrestApps.Core.Templates.Services; -using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -26,38 +24,22 @@ namespace CrestApps.Core.AI.Documents.Handlers; public sealed class DocumentOrchestrationHandler : IOrchestrationContextBuilderHandler { private readonly AIToolDefinitionOptions _toolDefinitions; - private readonly ChatDocumentsOptions _documentOptions; private readonly ITemplateService _templateService; - private readonly IAIDocumentStore _documentStore; - private readonly IDocumentFileStore _fileStore; - private readonly IAIDeploymentManager _deploymentManager; private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// /// The tool definitions. - /// The document options. /// The template service. - /// The document store. - /// The document file store. - /// The deployment manager. /// The logger. public DocumentOrchestrationHandler( IOptions toolDefinitions, - IOptions documentOptions, ITemplateService templateService, - IAIDocumentStore documentStore, - IDocumentFileStore fileStore, - IAIDeploymentManager deploymentManager, ILogger logger) { _toolDefinitions = toolDefinitions.Value; - _documentOptions = documentOptions.Value; _templateService = templateService; - _documentStore = documentStore; - _fileStore = fileStore; - _deploymentManager = deploymentManager; _logger = logger; } @@ -158,14 +140,14 @@ sessionObj is AIChatSession session && return; } - var visionContentResult = await BuildVisionUserContentsAsync(context, userSuppliedDocuments, cancellationToken); - + // Separate vision images from regular documents so the template can + // provide appropriate tool usage guidance for each category. var searchableUserSuppliedDocuments = userSuppliedDocuments? .Where(document => !IsVisionDocument(document)) .ToArray(); var visionUserSuppliedDocuments = userSuppliedDocuments? - .Where(document => IsVisionDocument(document) && visionContentResult.IncludedDocumentIds.Contains(document.DocumentId)) + .Where(IsVisionDocument) .ToArray(); context.OrchestrationContext.Documents ??= []; @@ -209,11 +191,6 @@ sessionObj is AIChatSession session && context.OrchestrationContext.SystemMessageBuilder.AppendLine(); context.OrchestrationContext.SystemMessageBuilder.Append(header); } - - if (visionContentResult.Contents.Count > 0) - { - context.OrchestrationContext.Properties[OrchestrationPropertyKeys.VisionUserContents] = visionContentResult.Contents; - } } private static AIDataSourceRagMetadata GetRagMetadata(object resource) @@ -233,96 +210,6 @@ private static AIDataSourceRagMetadata GetRagMetadata(object resource) return null; } - private async Task BuildVisionUserContentsAsync( - OrchestrationContextBuiltContext context, - IEnumerable userSuppliedDocuments, - CancellationToken cancellationToken) - { - if (userSuppliedDocuments?.Any() != true) - { - return VisionUserContentResult.Empty; - } - - var deployment = await ResolveChatDeploymentAsync(context, cancellationToken); - - if (deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) != true) - { - return VisionUserContentResult.Empty; - } - - var session = context.OrchestrationContext.CompletionContext?.AdditionalProperties is not null - && context.OrchestrationContext.CompletionContext.AdditionalProperties.TryGetValue(AICompletionContextKeys.Session, out var sessionObject) - ? sessionObject as AIChatSession - : null; - - var reference = GetVisionDocumentReference(context.Resource, session); - - if (reference == null) - { - return VisionUserContentResult.Empty; - } - - var visionDocuments = await _documentStore.GetDocumentsAsync(reference.Value.ReferenceId, reference.Value.ReferenceType); - var documentIds = new HashSet( - userSuppliedDocuments - .Where(document => MediaTypeHelper.IsVisionImageMediaType(document.ContentType) || MediaTypeHelper.IsVisionImageExtension(Path.GetExtension(document.FileName))) - .Select(document => document.DocumentId), - StringComparer.OrdinalIgnoreCase); - - if (documentIds.Count == 0) - { - return VisionUserContentResult.Empty; - } - - var remainingBytes = _documentOptions.MaxVisionInputBytesPerRequest > 0 - ? _documentOptions.MaxVisionInputBytesPerRequest - : long.MaxValue; - - var contents = new List(); - var includedDocumentIds = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var document in visionDocuments.Where(document => documentIds.Contains(document.ItemId))) - { - if (string.IsNullOrWhiteSpace(document.StoredFilePath)) - { - continue; - } - - if (ShouldSkipVisionDocument(document, remainingBytes)) - { - continue; - } - - await using var stream = await _fileStore.GetFileAsync(document.StoredFilePath); - - if (stream == null) - { - continue; - } - - var data = await ReadVisionDocumentBytesAsync(document, stream, cancellationToken); - - if (data == null || data.Length == 0) - { - continue; - } - - contents.Add(new DataContent(data, document.ContentType ?? MediaTypeHelper.InferMediaType(Path.GetExtension(document.FileName)))); - includedDocumentIds.Add(document.ItemId); - remainingBytes -= data.Length; - } - - return new VisionUserContentResult(contents, includedDocumentIds); - } - - private async Task ResolveChatDeploymentAsync(OrchestrationContextBuiltContext context, CancellationToken cancellationToken) - { - return await _deploymentManager.ResolveOrDefaultAsync( - AIDeploymentPurpose.Chat, - deploymentName: context.OrchestrationContext.CompletionContext?.ChatDeploymentName, - cancellationToken: cancellationToken); - } - private static bool IsVisionDocument(ChatDocumentInfo document) { if (document == null) @@ -337,109 +224,4 @@ private static bool IsVisionDocument(ChatDocumentInfo document) return MediaTypeHelper.IsVisionImageExtension(Path.GetExtension(document.FileName)); } - - private static (string ReferenceId, string ReferenceType)? GetVisionDocumentReference(object resource, AIChatSession session) - { - return resource switch - { - ChatInteraction interaction => (interaction.ItemId, AIReferenceTypes.Document.ChatInteraction), - AIProfile when session != null => (session.SessionId, AIReferenceTypes.Document.ChatSession), - _ => null, - }; - } - - private bool ShouldSkipVisionDocument(AIDocument document, long remainingBytes) - { - if (document.FileSize <= 0) - { - _logger.LogWarning( - "Skipping vision document '{DocumentId}' because its file size metadata is missing or invalid.", - document.ItemId); - - return true; - } - - if (document.FileSize > int.MaxValue) - { - _logger.LogWarning( - "Skipping vision document '{DocumentId}' because its size ({FileSize} bytes) exceeds the supported in-memory limit.", - document.ItemId, - document.FileSize); - - return true; - } - - if (_documentOptions.MaxVisionImageBytesPerFile > 0 && document.FileSize > _documentOptions.MaxVisionImageBytesPerFile) - { - _logger.LogWarning( - "Skipping vision document '{DocumentId}' because its size ({FileSize} bytes) exceeds the per-file limit of {MaxBytesPerFile} bytes.", - document.ItemId, - document.FileSize, - _documentOptions.MaxVisionImageBytesPerFile); - - return true; - } - - if (document.FileSize > remainingBytes) - { - _logger.LogWarning( - "Skipping vision document '{DocumentId}' because it would exceed the configured multimodal image budget of {MaxBytes} bytes for a single request.", - document.ItemId, - _documentOptions.MaxVisionInputBytesPerRequest); - - return true; - } - - return false; - } - - private static async Task ReadVisionDocumentBytesAsync( - AIDocument document, - Stream stream, - CancellationToken cancellationToken) - { - var data = GC.AllocateUninitializedArray((int)document.FileSize); - var totalRead = 0; - - while (totalRead < data.Length) - { - var bytesRead = await stream.ReadAsync(data.AsMemory(totalRead), cancellationToken); - - if (bytesRead == 0) - { - break; - } - - totalRead += bytesRead; - } - - if (totalRead == 0) - { - return null; - } - - if (totalRead == data.Length) - { - return data; - } - - return data[..totalRead]; - } - - private sealed class VisionUserContentResult - { - public static readonly VisionUserContentResult Empty = new([], new HashSet(StringComparer.OrdinalIgnoreCase)); - - public VisionUserContentResult( - IReadOnlyList contents, - IReadOnlySet includedDocumentIds) - { - Contents = contents; - IncludedDocumentIds = includedDocumentIds; - } - - public IReadOnlyList Contents { get; } - - public IReadOnlySet IncludedDocumentIds { get; } - } } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/AIProfileSessionDocumentsMetadata.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/AIProfileSessionDocumentsMetadata.cs index e0d124cf..9053d26e 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/AIProfileSessionDocumentsMetadata.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/AIProfileSessionDocumentsMetadata.cs @@ -13,4 +13,11 @@ public sealed class AIProfileSessionDocumentsMetadata /// to chat sessions that use this profile. /// public bool AllowSessionDocuments { get; set; } + + /// + /// Gets or sets whether users are allowed to upload image files + /// to chat sessions that use this profile. When enabled, image uploads + /// are processed using the global vision deployment. + /// + public bool AllowSessionImageUploads { get; set; } } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptions.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptions.cs index 7b8c667e..854ec75d 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptions.cs @@ -37,6 +37,20 @@ public sealed class ChatDocumentsOptions /// public long MaxVisionImageBytesPerFile { get; set; } = DefaultMaxVisionImageBytesPerFile; + /// + /// Gets or sets a value indicating whether images should be analyzed at upload time + /// using a vision model to extract caption, OCR text, and detected entities. + /// When enabled, analysis results are stored as document chunks so the model can + /// access image information via text-based tools instead of raw byte injection. + /// + public bool AnalyzeImagesAtUpload { get; set; } = true; + + /// + /// Gets or sets the maximum number of inspect_image tool invocations + /// allowed per chat request. This limits the cost of on-demand raw image inspection. + /// + public int MaxInspectImageCallsPerRequest { get; set; } = 2; + internal void Add(string extension, bool embeddable = true) { ArgumentException.ThrowIfNullOrWhiteSpace(extension); diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs index c52bd00a..b496ca89 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs @@ -10,9 +10,10 @@ public static class ChatDocumentsOptionsExtensions /// /// The options. /// Whether to include supported vision image extensions. - public static string GetAllowedFileExtensionsAcceptValue(this ChatDocumentsOptions options, bool includeVisionImages = false) + /// Whether to include document extensions. + public static string GetAllowedFileExtensionsAcceptValue(this ChatDocumentsOptions options, bool includeVisionImages = false, bool includeDocuments = true) { - return BuildAcceptValue(GetAllowedFileExtensions(options, includeVisionImages)); + return BuildAcceptValue(GetAllowedFileExtensions(options, includeVisionImages, includeDocuments)); } /// @@ -20,9 +21,10 @@ public static string GetAllowedFileExtensionsAcceptValue(this ChatDocumentsOptio /// /// The options. /// Whether to include supported vision image extensions. - public static string GetAllowedFileExtensionsDisplayValue(this ChatDocumentsOptions options, bool includeVisionImages = false) + /// Whether to include document extensions. + public static string GetAllowedFileExtensionsDisplayValue(this ChatDocumentsOptions options, bool includeVisionImages = false, bool includeDocuments = true) { - return BuildDisplayValue(GetAllowedFileExtensions(options, includeVisionImages)); + return BuildDisplayValue(GetAllowedFileExtensions(options, includeVisionImages, includeDocuments)); } /// @@ -48,13 +50,22 @@ public static string GetEmbeddableFileExtensionsDisplayValue(this ChatDocumentsO /// /// The options. /// Whether to include supported vision image extensions. - public static IReadOnlyList GetAllowedFileExtensions(this ChatDocumentsOptions options, bool includeVisionImages = false) + /// Whether to include document extensions. + public static IReadOnlyList GetAllowedFileExtensions(this ChatDocumentsOptions options, bool includeVisionImages = false, bool includeDocuments = true) { - var extensions = OrderExtensions(options?.AllowedFileExtensions); + IEnumerable extensions = []; - return includeVisionImages - ? OrderExtensions(extensions.Concat(MediaTypeHelper.VisionImageExtensions)) - : extensions; + if (includeDocuments) + { + extensions = OrderExtensions(options?.AllowedFileExtensions); + } + + if (includeVisionImages) + { + extensions = extensions.Concat(MediaTypeHelper.VisionImageExtensions); + } + + return OrderExtensions(extensions); } /// @@ -63,14 +74,15 @@ public static IReadOnlyList GetAllowedFileExtensions(this ChatDocumentsO /// The options. /// The file extension. /// Whether to include supported vision image extensions. - public static bool IsAllowedFileExtension(this ChatDocumentsOptions options, string extension, bool includeVisionImages = false) + /// Whether to include document extensions. + public static bool IsAllowedFileExtension(this ChatDocumentsOptions options, string extension, bool includeVisionImages = false, bool includeDocuments = true) { if (string.IsNullOrWhiteSpace(extension)) { return false; } - return GetAllowedFileExtensions(options, includeVisionImages) + return GetAllowedFileExtensions(options, includeVisionImages, includeDocuments) .Contains(extension.StartsWith('.') ? extension : '.' + extension, StringComparer.OrdinalIgnoreCase); } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/ImageAnalysisResult.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/ImageAnalysisResult.cs new file mode 100644 index 00000000..30eb068e --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/ImageAnalysisResult.cs @@ -0,0 +1,82 @@ +namespace CrestApps.Core.AI.Documents.Models; + +/// +/// Represents the structured result of analyzing an uploaded image using a vision model. +/// +public sealed class ImageAnalysisResult +{ + /// + /// Gets or sets a concise 1–2 sentence description of the image content. + /// + public string Caption { get; set; } + + /// + /// Gets or sets a detailed multi-sentence description of the image, + /// covering composition, colors, layout, and context. + /// + public string Description { get; set; } + + /// + /// Gets or sets any readable text detected in the image via OCR. + /// + public string OcrText { get; set; } + + /// + /// Gets or sets a description of notable detected entities such as objects, people, charts, or UI elements. + /// + public string DetectedEntities { get; set; } + + /// + /// Gets or sets the full raw analysis response from the vision model. + /// + public string RawAnalysis { get; set; } + + /// + /// Gets or sets a value indicating whether the analysis completed successfully. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the error message if the analysis failed. + /// + public string Error { get; set; } + + /// + /// Creates a successful analysis result. + /// + /// The image caption. + /// The detailed image description. + /// The OCR-extracted text. + /// The detected entities description. + /// The full raw analysis response. + public static ImageAnalysisResult Succeeded( + string caption, + string description, + string ocrText, + string detectedEntities, + string rawAnalysis) + { + return new ImageAnalysisResult + { + Success = true, + Caption = caption, + Description = description, + OcrText = ocrText, + DetectedEntities = detectedEntities, + RawAnalysis = rawAnalysis, + }; + } + + /// + /// Creates a failed analysis result. + /// + /// The error message describing the failure. + public static ImageAnalysisResult Failed(string error) + { + return new ImageAnalysisResult + { + Success = false, + Error = error, + }; + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentOptions.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentOptions.cs index 2090cd48..759c9e24 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentOptions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentOptions.cs @@ -20,4 +20,16 @@ public sealed class InteractionDocumentOptions /// Gets or sets how retrieved document matches are added to AI context. /// public DocumentRetrievalMode RetrievalMode { get; set; } = DocumentRetrievalMode.Chunk; + + /// + /// Gets or sets whether users are allowed to upload document files in chat interactions. + /// Default is . + /// + public bool AllowDocumentUploads { get; set; } = true; + + /// + /// Gets or sets whether users are allowed to upload image files in chat interactions. + /// When enabled, image uploads are processed using the global vision deployment. + /// + public bool AllowImageUploads { get; set; } } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentSettings.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentSettings.cs index 79cf04b8..ac45522e 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentSettings.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/InteractionDocumentSettings.cs @@ -20,4 +20,16 @@ public sealed class InteractionDocumentSettings /// Gets or sets how retrieved document matches are added to AI context. /// public DocumentRetrievalMode RetrievalMode { get; set; } = DocumentRetrievalMode.Chunk; + + /// + /// Gets or sets whether users are allowed to upload document files in chat interactions. + /// Default is . + /// + public bool AllowDocumentUploads { get; set; } = true; + + /// + /// Gets or sets whether users are allowed to upload image files in chat interactions. + /// When enabled, image uploads are processed using the global vision deployment. + /// + public bool AllowImageUploads { get; set; } } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs b/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs index 7b1b416b..72bfc70f 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs @@ -73,6 +73,7 @@ public static IServiceCollection AddCoreAIDocumentProcessing(this IServiceCollec }); services.TryAddScoped(); + services.TryAddScoped(); services.TryAddScoped(); services.TryAddScoped(); services.TryAddSingleton(); @@ -108,6 +109,11 @@ public static IServiceCollection AddCoreAIDocumentProcessing(this IServiceCollec .WithDescription("Reads and parses tabular data (CSV, TSV, Excel) from a document.") .WithPurpose(AIToolPurposes.DocumentProcessing); + services.AddCoreAITool(InspectImageTool.TheName) + .WithTitle("Inspect Image") + .WithDescription("Performs detailed visual inspection of an uploaded image when text summaries are insufficient.") + .WithPurpose(AIToolPurposes.DocumentProcessing); + return services; } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultImageAnalysisService.cs b/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultImageAnalysisService.cs new file mode 100644 index 00000000..fa64503a --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultImageAnalysisService.cs @@ -0,0 +1,202 @@ +using System.Text.Json; +using CrestApps.Core.AI.Clients; +using CrestApps.Core.AI.Deployments; +using CrestApps.Core.AI.Documents.Models; +using CrestApps.Core.AI.Models; +using CrestApps.Core.Support.Json; +using CrestApps.Core.Templates.Services; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Documents.Services; + +/// +/// Default implementation of that sends the image +/// to a vision-capable chat model and parses the structured JSON analysis response. +/// +public sealed class DefaultImageAnalysisService : IImageAnalysisService +{ + private readonly IAIDeploymentManager _deploymentManager; + private readonly IAIClientFactory _clientFactory; + private readonly ITemplateService _templateService; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The deployment manager for resolving vision models. + /// The client factory for creating chat clients. + /// The template service for rendering the analysis prompt. + /// The logger. + public DefaultImageAnalysisService( + IAIDeploymentManager deploymentManager, + IAIClientFactory clientFactory, + ITemplateService templateService, + ILogger logger) + { + _deploymentManager = deploymentManager; + _clientFactory = clientFactory; + _templateService = templateService; + _logger = logger; + } + + /// + public async Task AnalyzeAsync( + Stream imageStream, + string contentType, + string fileName, + string chatDeploymentName = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(imageStream); + ArgumentException.ThrowIfNullOrWhiteSpace(contentType); + ArgumentException.ThrowIfNullOrWhiteSpace(fileName); + + try + { + var deployment = await ResolveVisionDeploymentAsync(chatDeploymentName, cancellationToken); + + if (deployment == null) + { + _logger.LogWarning("No vision-capable deployment available for image analysis of '{FileName}'.", fileName); + + return ImageAnalysisResult.Failed("No vision-capable deployment is available for image analysis."); + } + + var chatClient = await _clientFactory.CreateChatClientAsync(deployment); + + var messages = new List(); + + var systemPrompt = await _templateService.RenderAsync(AITemplateIds.ImageAnalysis, cancellationToken: cancellationToken); + + if (!string.IsNullOrWhiteSpace(systemPrompt)) + { + messages.Add(new(ChatRole.System, systemPrompt)); + } + + var imageBytes = await ReadStreamBytesAsync(imageStream, cancellationToken); + + var userContents = new List + { + new TextContent($"Analyze this image: \"{fileName}\""), + new DataContent(imageBytes, contentType), + }; + + messages.Add(new(ChatRole.User, userContents)); + + var response = await chatClient.GetResponseAsync(messages, cancellationToken: cancellationToken); + + var rawText = response?.Text; + + if (string.IsNullOrWhiteSpace(rawText)) + { + _logger.LogWarning("Vision model returned empty response for image '{FileName}'.", fileName); + + return ImageAnalysisResult.Failed("The vision model returned an empty response."); + } + + return ParseJsonAnalysisResponse(rawText); + } + catch (Exception ex) + { + _logger.LogError(ex, "Image analysis failed for '{FileName}'.", fileName); + + return ImageAnalysisResult.Failed($"Image analysis failed: {ex.Message}"); + } + } + + private async Task ResolveVisionDeploymentAsync( + string chatDeploymentName, + CancellationToken cancellationToken) + { + // Prioritize the global vision deployment. + var visionDeployment = await _deploymentManager.ResolveOrDefaultAsync( + AIDeploymentPurpose.Vision, + cancellationToken: cancellationToken); + + if (visionDeployment != null) + { + return visionDeployment; + } + + // Fall back to the specified chat deployment if it supports vision. + if (!string.IsNullOrWhiteSpace(chatDeploymentName)) + { + var chatDeployment = await _deploymentManager.ResolveOrDefaultAsync( + AIDeploymentPurpose.Chat, + deploymentName: chatDeploymentName, + cancellationToken: cancellationToken); + + if (chatDeployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true) + { + return chatDeployment; + } + } + + return null; + } + + private ImageAnalysisResult ParseJsonAnalysisResponse(string rawText) + { + var json = JsonExtractor.ExtractJsonObject(rawText); + + if (json == null) + { + _logger.LogWarning("Vision model response did not contain a valid JSON object. Falling back to raw text."); + + return ImageAnalysisResult.Succeeded( + caption: rawText.Length > 200 ? rawText[..200] : rawText, + description: rawText, + ocrText: string.Empty, + detectedEntities: string.Empty, + rawAnalysis: rawText); + } + + try + { + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + var caption = GetStringProperty(root, "caption"); + var description = GetStringProperty(root, "description"); + var ocrText = GetStringProperty(root, "ocr_text"); + var detectedEntities = GetStringProperty(root, "detected_entities"); + + return ImageAnalysisResult.Succeeded(caption, description, ocrText, detectedEntities, rawText); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse vision model JSON response. Falling back to raw text."); + + return ImageAnalysisResult.Succeeded( + caption: rawText.Length > 200 ? rawText[..200] : rawText, + description: rawText, + ocrText: string.Empty, + detectedEntities: string.Empty, + rawAnalysis: rawText); + } + } + + private static string GetStringProperty(JsonElement root, string propertyName) + { + if (root.TryGetProperty(propertyName, out var element) && element.ValueKind == JsonValueKind.String) + { + return element.GetString() ?? string.Empty; + } + + return string.Empty; + } + + private static async Task ReadStreamBytesAsync(Stream stream, CancellationToken cancellationToken) + { + if (stream is MemoryStream ms) + { + return ms.ToArray(); + } + + using var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream, cancellationToken); + + return memoryStream.ToArray(); + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Services/IImageAnalysisService.cs b/src/Primitives/CrestApps.Core.AI.Documents/Services/IImageAnalysisService.cs new file mode 100644 index 00000000..446d568d --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Services/IImageAnalysisService.cs @@ -0,0 +1,29 @@ +using CrestApps.Core.AI.Documents.Models; + +namespace CrestApps.Core.AI.Documents.Services; + +/// +/// Defines a service that analyzes images using a vision-capable AI model +/// and returns structured results (caption, OCR text, detected entities). +/// +public interface IImageAnalysisService +{ + /// + /// Analyzes an image stream using a vision model and returns a structured result. + /// + /// The image data stream. + /// The MIME type of the image (e.g., "image/png"). + /// The original file name of the image. + /// + /// The optional deployment name to use for the vision call. + /// If , the default vision-capable deployment is resolved. + /// + /// The token to monitor for cancellation requests. + /// An containing the structured analysis or an error. + Task AnalyzeAsync( + Stream imageStream, + string contentType, + string fileName, + string chatDeploymentName = null, + CancellationToken cancellationToken = default); +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tools/InspectImageTool.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tools/InspectImageTool.cs new file mode 100644 index 00000000..a38b6820 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tools/InspectImageTool.cs @@ -0,0 +1,256 @@ +using System.Text.Json; +using CrestApps.Core.AI.Clients; +using CrestApps.Core.AI.Deployments; +using CrestApps.Core.AI.Documents.Models; +using CrestApps.Core.AI.Extensions; +using CrestApps.Core.AI.Models; +using CrestApps.Core.AI.Orchestration; +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace CrestApps.Core.AI.Documents.Tools; + +/// +/// System tool that performs on-demand visual inspection of an uploaded image. +/// The model calls this tool when text-based summaries are insufficient and +/// raw pixel-level understanding is required. +/// +public sealed class InspectImageTool : AIFunction +{ + public const string TheName = SystemToolNames.InspectImage; + + private static readonly JsonElement _jsonSchema = JsonSerializer.Deserialize( + """ + { + "type": "object", + "properties": { + "document_id": { + "type": "string", + "description": "The unique identifier of the image document to inspect." + }, + "question": { + "type": "string", + "description": "An optional specific question about the image to focus the inspection on." + } + }, + "required": ["document_id"], + "additionalProperties": false + } + """); + + /// + /// Gets the name. + /// + public override string Name => TheName; + + /// + /// Gets the description. + /// + public override string Description => "Performs a detailed visual inspection of an uploaded image. Use this when the text summary from read_document is insufficient and you need pixel-level analysis such as reading fine text, comparing visual elements, or understanding spatial layout."; + + /// + /// Gets the json Schema. + /// + public override JsonElement JsonSchema => _jsonSchema; + + /// + /// Gets the additional Properties. + /// + public override IReadOnlyDictionary AdditionalProperties { get; } = + new Dictionary() + { + ["Strict"] = false, + }; + + /// + /// Invokes the image inspection logic. + /// + /// The arguments. + /// The cancellation token. + protected override async ValueTask InvokeCoreAsync( + AIFunctionArguments arguments, + CancellationToken cancellationToken) + { + var logger = arguments.Services.GetRequiredService>(); + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("AI tool '{ToolName}' invoked.", Name); + } + + if (!arguments.TryGetFirstString("document_id", out var documentId)) + { + logger.LogWarning("AI tool '{ToolName}' missing required argument 'document_id'.", Name); + + return "Unable to find a 'document_id' argument in the arguments parameter."; + } + + arguments.TryGetFirstString("question", out var question); + + var executionContext = AIInvocationScope.Current?.ToolExecutionContext; + + if (executionContext is null) + { + logger.LogWarning("AI tool '{ToolName}' failed: execution context is missing.", Name); + + return "Image inspection requires an active execution context."; + } + + var documentStore = arguments.Services.GetService(); + + if (documentStore is null) + { + logger.LogWarning("AI tool '{ToolName}' failed: document store is not available.", Name); + + return "Document store is not available."; + } + + var document = await ResolveDocumentAsync(documentStore, documentId, executionContext, cancellationToken); + + if (document == null) + { + logger.LogWarning("AI tool '{ToolName}' failed: document '{DocumentId}' was not found in this session.", Name, documentId); + + return $"Image document with ID '{documentId}' was not found in this session."; + } + + if (!IsVisionImage(document)) + { + logger.LogWarning("AI tool '{ToolName}' failed: document '{DocumentId}' is not a vision image.", Name, documentId); + + return $"Document '{documentId}' is not an image. Use 'read_document' for non-image documents."; + } + + var fileStore = arguments.Services.GetService(); + + if (fileStore is null || string.IsNullOrWhiteSpace(document.StoredFilePath)) + { + logger.LogWarning("AI tool '{ToolName}' failed: file store is not available or file path is missing.", Name); + + return "Image file is not available for inspection."; + } + + var options = arguments.Services.GetRequiredService>().Value; + + if (options.MaxVisionImageBytesPerFile > 0 && document.FileSize > options.MaxVisionImageBytesPerFile) + { + logger.LogWarning( + "AI tool '{ToolName}' failed: image '{DocumentId}' size ({FileSize} bytes) exceeds per-file limit of {MaxBytes} bytes.", + Name, + documentId, + document.FileSize, + options.MaxVisionImageBytesPerFile); + + return $"Image is too large for inspection ({document.FileSize} bytes exceeds the {options.MaxVisionImageBytesPerFile} byte limit)."; + } + + await using var stream = await fileStore.GetFileAsync(document.StoredFilePath); + + if (stream == null) + { + logger.LogWarning("AI tool '{ToolName}' failed: image file not found at path '{Path}'.", Name, document.StoredFilePath); + + return "Image file could not be retrieved from storage."; + } + + using var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream, cancellationToken); + var imageBytes = memoryStream.ToArray(); + + if (imageBytes.Length == 0) + { + return "Image file is empty."; + } + + var deploymentManager = arguments.Services.GetRequiredService(); + + var deployment = await deploymentManager.ResolveOrDefaultAsync( + AIDeploymentPurpose.Vision, + cancellationToken: cancellationToken); + + if (deployment == null) + { + logger.LogWarning("AI tool '{ToolName}' failed: no vision-capable deployment available.", Name); + + return "No vision-capable deployment is available for image inspection."; + } + + var clientFactory = arguments.Services.GetRequiredService(); + var chatClient = await clientFactory.CreateChatClientAsync(deployment); + + var contentType = document.ContentType ?? MediaTypeHelper.InferMediaType(Path.GetExtension(document.FileName)); + var userPrompt = string.IsNullOrWhiteSpace(question) + ? $"Describe this image (\"{document.FileName}\") in detail. Include any text, layout, colors, and notable elements." + : $"Regarding this image (\"{document.FileName}\"): {question}"; + + var userContents = new List + { + new TextContent(userPrompt), + new DataContent(imageBytes, contentType), + }; + + var messages = new List + { + new(ChatRole.System, "You are a precise image analysis assistant. Answer the user's question about the provided image accurately and concisely."), + new(ChatRole.User, userContents), + }; + + var response = await chatClient.GetResponseAsync(messages, cancellationToken: cancellationToken); + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("AI tool '{ToolName}' completed for document '{DocumentId}'.", Name, documentId); + } + + return response?.Text ?? "The vision model did not return a response."; + } + + private static async Task ResolveDocumentAsync( + IAIDocumentStore documentStore, + string documentId, + AIToolExecutionContext executionContext, + CancellationToken cancellationToken) + { + var document = await documentStore.FindByIdAsync(documentId, cancellationToken); + + if (document == null) + { + return null; + } + + if (executionContext.Resource is ChatInteraction interaction) + { + return document.ReferenceId == interaction.ItemId ? document : null; + } + + if (executionContext.Resource is AIProfile profile) + { + if (document.ReferenceId == profile.ItemId) + { + return document; + } + + if (AIInvocationScope.Current?.Items.TryGetValue(nameof(AIChatSession), out var sessionObj) == true && + sessionObj is AIChatSession session && + document.ReferenceId == session.SessionId) + { + return document; + } + } + + return null; + } + + private static bool IsVisionImage(AIDocument document) + { + if (MediaTypeHelper.IsVisionImageMediaType(document.ContentType)) + { + return true; + } + + return MediaTypeHelper.IsVisionImageExtension(Path.GetExtension(document.FileName)); + } +} diff --git a/src/Primitives/CrestApps.Core.AI/AITemplateIds.cs b/src/Primitives/CrestApps.Core.AI/AITemplateIds.cs index b256f4a2..8fb9d5cd 100644 --- a/src/Primitives/CrestApps.Core.AI/AITemplateIds.cs +++ b/src/Primitives/CrestApps.Core.AI/AITemplateIds.cs @@ -56,4 +56,6 @@ public static class AITemplateIds public const string AgentAvailability = "agent-availability"; public const string TabularBatchProcessing = "tabular-batch-processing"; + + public const string ImageAnalysis = "image-analysis"; } diff --git a/src/Primitives/CrestApps.Core.AI/Orchestration/DefaultOrchestrator.cs b/src/Primitives/CrestApps.Core.AI/Orchestration/DefaultOrchestrator.cs index e17382d3..9de61519 100644 --- a/src/Primitives/CrestApps.Core.AI/Orchestration/DefaultOrchestrator.cs +++ b/src/Primitives/CrestApps.Core.AI/Orchestration/DefaultOrchestrator.cs @@ -416,7 +416,7 @@ private List GetPlanningMessages(OrchestrationContext context) // Ensure the current user message is always included as the last message. if (messages.Count == 0 || messages[^1].Text != context.UserMessage) { - messages.Add(CreateCurrentUserMessage(context)); + messages.Add(new ChatMessage(ChatRole.User, context.UserMessage)); } return messages; @@ -425,7 +425,7 @@ private List GetPlanningMessages(OrchestrationContext context) private static List GetExecutionMessages(OrchestrationContext context) { var messages = context.ConversationHistory?.ToList() ?? []; - var currentUserMessage = CreateCurrentUserMessage(context); + var currentUserMessage = new ChatMessage(ChatRole.User, context.UserMessage); if (messages.Count == 0) { @@ -480,26 +480,6 @@ private static string BuildScoringContext(OrchestrationContext context) return sb.ToString(); } - private static ChatMessage CreateCurrentUserMessage(OrchestrationContext context) - { - if (!context.Properties.TryGetValue(OrchestrationPropertyKeys.VisionUserContents, out var value) || value is not IReadOnlyList visionContents || visionContents.Count == 0) - { - return new ChatMessage(ChatRole.User, context.UserMessage); - } - - var contents = new List - { - new TextContent(context.UserMessage), - }; - - contents.AddRange(visionContents); - - return new ChatMessage(ChatRole.User, context.UserMessage) - { - Contents = contents, - }; - } - /// /// Attempts to create a chat client using the utility deployment, /// falling back to the chat deployment if no utility deployment is configured. @@ -527,6 +507,6 @@ private async Task ResolveChatDeploymentAsync(OrchestrationContext return await _deploymentManager.ResolveOrDefaultAsync( AIDeploymentPurpose.Chat, deploymentName: context.CompletionContext?.ChatDeploymentName) - ?? throw new InvalidOperationException("Unable to resolve a chat deployment for the orchestration context."); + ?? throw new InvalidOperationException("Unable to resolve a chat deployment for the orchestration context."); } } diff --git a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md index 1d44e06e..8bd5f6d8 100644 --- a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md +++ b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md @@ -5,7 +5,7 @@ Parameters: - tools: array of AIToolDefinitionEntry objects for document processing tools available. - knowledgeBaseDocuments: array of profile-level ChatDocumentInfo objects that are hidden background knowledge. - userSuppliedDocuments: array of non-image session/user-level ChatDocumentInfo objects that are user-visible uploads/attachments. - - visionUserSuppliedDocuments: array of supported image session/user-level ChatDocumentInfo objects that are attached to the current user message as multimodal inputs. + - visionUserSuppliedDocuments: array of image session/user-level ChatDocumentInfo objects with text analysis available via document tools. IsListable: false Category: Documents --- @@ -18,10 +18,11 @@ Category: Documents {% assign hasKnowledgeBaseDocuments = knowledgeBaseDocuments.size > 0 %} {% if hasVisionUserSuppliedDocuments %} -The user has uploaded the following image attachments as supplementary context. -These supported image attachments are already attached to the current user message as multimodal inputs. -When the user asks what is shown in one of these images, inspect the image directly and answer from its visual content. -Do not say that you cannot view images or ask the user to upload the image again unless the image input is actually unavailable. +The user has uploaded the following image attachments. +Each image has been analyzed and its content (caption, OCR text, detected entities) is available as document text. +Use `read_document` with the image's document ID to retrieve the full analysis. +Use `search_documents` to find image content by semantic search. +Use `inspect_image` only when you need pixel-level detail that the text analysis does not provide (e.g., fine text, color comparison, spatial layout, or visual elements not captured in the summary). ### Available image attachments: {% for doc in visionUserSuppliedDocuments %} diff --git a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/image-analysis.md b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/image-analysis.md new file mode 100644 index 00000000..146b2721 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/image-analysis.md @@ -0,0 +1,24 @@ +--- +Title: Image Analysis +Description: Instructs the vision model to produce a structured JSON analysis of an uploaded image including caption, description, OCR text, and detected entities. +IsListable: false +Category: Documents +--- + +You are a precise image analysis system. Analyze the provided image and return a structured JSON response. + +[Rules] +1. Return ONLY valid JSON — no markdown code fences, no commentary, no text before or after the JSON object. +2. If a field has no applicable content, use an empty string for that field. +3. For ocr_text, preserve the original text layout as closely as possible. +4. For detected_entities, list the most prominent or relevant items, not every pixel. +5. Keep the caption concise (1–3 sentences). +6. The description should be a detailed multi-sentence explanation covering composition, colors, spatial layout, context, and notable visual relationships. + +[Output Schema] +{ + "caption": "A concise 1–3 sentence summary of what the image shows", + "description": "A detailed multi-sentence description covering composition, colors, layout, context, and visual relationships", + "ocr_text": "Any readable text found in the image, preserving layout where possible", + "detected_entities": "Notable objects, people, charts, diagrams, UI elements, icons, or structural components" +} diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Create.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Create.razor index c807d182..a7fd5014 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Create.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Create.razor @@ -799,9 +799,17 @@
Allow users to upload documents during chat sessions with this profile.
+
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
+
Attach Documents
-

Uploaded files will be indexed as hidden knowledge for this profile after you save it. Supported knowledge formats: @_supportedExtensionsDisplay

+

Uploaded files will be indexed as hidden knowledge for this profile after you save it.

@if (_model.AttachedDocuments.Count > 0) { @@ -859,7 +867,7 @@ Browse files
-
Supported knowledge formats: @_supportedExtensionsDisplay
+
Supported formats: @_supportedExtensionsDisplay
diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Edit.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Edit.razor index c8aeaa2b..0dea82d8 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Edit.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/AIProfiles/Edit.razor @@ -740,11 +740,19 @@ else if (_model != null)
Allow users to upload documents during chat sessions with this profile.
+
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
+
Attached Documents
-

These files are indexed as background knowledge for this profile and remain hidden from end users. Supported knowledge formats: @_supportedExtensionsDisplay

+

These files are indexed as background knowledge for this profile and remain hidden from end users.

@_model.AttachedDocuments.Count existing
@@ -799,7 +807,7 @@ else if (_model != null) Browse files
-
Supported knowledge formats: @_supportedExtensionsDisplay
+
Supported formats: @_supportedExtensionsDisplay
diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Create.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Create.razor index c80a48ac..050e5ecc 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Create.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Create.razor @@ -459,6 +459,14 @@
Allow users to upload documents during chat sessions with profiles created from this template.
+
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
+ diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Edit.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Edit.razor index 32b98f2c..a1a09a7d 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Edit.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AI/Templates/Edit.razor @@ -478,6 +478,14 @@
Allow users to upload documents during chat sessions with profiles created from this template.
+
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
+ diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AIChat/Chat.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AIChat/Chat.razor index dbbb210b..6a165f95 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AIChat/Chat.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/AIChat/Chat.razor @@ -2,7 +2,6 @@ @attribute [Authorize(Policy = "Admin")] @using CrestApps.Core.AI @using CrestApps.Core.AI.Chat -@using CrestApps.Core.AI.Deployments @using CrestApps.Core.AI.Documents.Models @using CrestApps.Core.AI.Models @using CrestApps.Core.AI.Profiles @@ -15,7 +14,6 @@ @inject IJSRuntime JS @inject SiteSettingsStore SiteSettings @inject IOptions ChatDocumentOptions -@inject IAIDeploymentManager DeploymentManager Chat - @(_profileDisplayText ?? "AI Chat") - Blazor AI Integration Sample @@ -80,7 +78,7 @@ else
-
+
@@ -132,7 +130,7 @@ else private string _supportedExtensionsAccept; private string _supportedExtensionsDisplay; private IReadOnlyList _existingDocuments = Array.Empty(); - private bool _supportsVisionUploads; + private bool _allowImageUploads; protected override async Task OnInitializedAsync() { @@ -188,9 +186,10 @@ else _ => ChatMode.TextInput, }; - _supportsVisionUploads = await SupportsVisionUploadsAsync(_profile); - _supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_supportsVisionUploads); - _supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_supportsVisionUploads); + _allowImageUploads = sessionDocMeta?.AllowSessionImageUploads == true + && !string.IsNullOrWhiteSpace(deploymentDefaults.DefaultVisionDeploymentName); + _supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_allowImageUploads, _allowSessionDocuments); + _supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_allowImageUploads, _allowSessionDocuments); } protected override async Task OnAfterRenderAsync(bool firstRender) @@ -217,7 +216,7 @@ else conversationButtonElementSelector = "#chat-conversation-btn", ttsVoiceName = _ttsVoiceName, textToSpeechEnabled = _textToSpeechEnabled, - sessionDocumentsEnabled = _allowSessionDocuments, + sessionDocumentsEnabled = _allowSessionDocuments || _allowImageUploads, existingDocuments = _existingDocuments, documentBarSelector = "#chat-document-bar", uploadDocumentUrl = "/ai/chat-sessions/upload-document", @@ -228,11 +227,4 @@ else } } - private async Task SupportsVisionUploadsAsync(AIProfile profile) - { - var deployment = await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Chat, deploymentName: profile.ChatDeploymentName) - ?? await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Utility, deploymentName: profile.UtilityDeploymentName); - - return deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; - } } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor index db3a178e..99facb1b 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor @@ -25,6 +25,7 @@ @inject ISearchIndexProfileStore IndexProfileStore @inject IDataProtectionProvider DataProtectionProvider @inject ClaudeClientService ClaudeClientService +@inject SiteSettingsChangedNotifier SettingsChangedNotifier AI Settings - Blazor AI Integration Sample @@ -370,6 +371,26 @@ else
Controls whether document retrieval returns matching chunks directly or expands results hierarchically by parent document.
+
+
+ + +
+
When enabled, users can upload document files in chat interactions. Documents are indexed and used as background knowledge.
+
+ +
+
+ + +
+
When enabled, users can upload images in chat interactions. Images will be analyzed using the configured vision deployment.
+ @if (string.IsNullOrWhiteSpace(_model.DefaultVisionDeploymentName)) + { +
No vision deployment is configured. Image uploads will not be processed until a vision deployment is selected in the Deployments section above.
+ } +
+
@@ -711,6 +732,8 @@ else DocumentIndexProfileName = documentSettings.IndexProfileName, DocumentTopN = documentSettings.TopN, DocumentRetrievalMode = documentSettings.RetrievalMode, + AllowInteractionImageUploads = documentSettings.AllowImageUploads, + AllowInteractionDocumentUploads = documentSettings.AllowDocumentUploads, DataSourceDefaultStrictness = dataSourceSettings.DefaultStrictness, DataSourceDefaultTopNDocuments = dataSourceSettings.DefaultTopNDocuments, McpServerAuthenticationType = mcpServerSettings.AuthenticationType, @@ -975,6 +998,8 @@ else IndexProfileName = _model.DocumentIndexProfileName?.Trim(), TopN = _model.DocumentTopN, RetrievalMode = _model.DocumentRetrievalMode, + AllowDocumentUploads = _model.AllowInteractionDocumentUploads, + AllowImageUploads = _model.AllowInteractionImageUploads, }); SiteSettings.Set(new AIDataSourceSettings @@ -1056,6 +1081,7 @@ else }); await SiteSettings.SaveChangesAsync(); + await SettingsChangedNotifier.NotifyAsync(); _successMessage = "Settings saved successfully."; } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Chat.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Chat.razor index ee02f6c1..14c53148 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Chat.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Chat.razor @@ -34,7 +34,6 @@ @inject IOptionsMonitor DataSourceOptions @inject IOptions OrchestratorOptionsAccessor @inject IOptions ToolOptionsAccessor -@inject IOptionsMonitor InteractionDocumentOptionsAccessor @inject IOptions ChatDocumentOptions @inject ISearchIndexProfileStore IndexProfileStore @inject IAIDocumentStore DocumentStore @@ -275,30 +274,40 @@ else } -
- -
- -
- -
-
Drag and drop files here
-
@(SupportsVisionUploads ? "or browse for documents and images to upload into this chat interaction" : "or browse for documents to upload into this chat interaction")
-
-
- + @if (_model.AllowDocumentUploads || _model.AllowImageUploads) + { + var uploadLabel = _model.AllowImageUploads && _model.AllowDocumentUploads + ? "Upload documents and images" + : _model.AllowImageUploads ? "Upload images" : "Upload documents"; + var uploadDescription = _model.AllowImageUploads && _model.AllowDocumentUploads + ? "or browse for documents and images to upload into this chat interaction" + : _model.AllowImageUploads ? "or browse for images to upload into this chat interaction" : "or browse for documents to upload into this chat interaction"; + +
+ +
+ +
+ +
+
Drag and drop files here
+
@uploadDescription
+
+
+ +
+
Supported formats: @SupportedExtensionsDisplay
-
Supported formats: @SupportedExtensionsDisplay
+
+
+
0%
+
+
-
-
-
0%
-
-
-
+ }
@if (_model.Documents.Any()) { @@ -678,10 +687,10 @@ else string.Equals(d.Name, d.ModelName, StringComparison.OrdinalIgnoreCase) ? d.Name : $"{d.Name} ({d.ModelName})", d.Name)) .ToList(); - model.DeploymentVisionSupport = deployments - .Where(d => d.Purpose.Supports(AIDeploymentPurpose.Chat)) - .ToDictionary(d => d.Name, d => d.Purpose.Supports(AIDeploymentPurpose.Vision), StringComparer.OrdinalIgnoreCase); - model.DefaultChatDeploymentSupportsVision = (await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Chat))?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; + var interactionDocSettings = SiteSettingsStore.Get(); + var visionDeployment = await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision); + model.AllowImageUploads = interactionDocSettings.AllowImageUploads && visionDeployment != null; + model.AllowDocumentUploads = interactionDocSettings.AllowDocumentUploads; var orchestratorOptions = OrchestratorOptionsAccessor.Value; var orchestrators = orchestratorOptions.GetOrchestratorDescriptors(); @@ -776,23 +785,9 @@ else .GroupBy(template => template.Category) .OrderBy(group => group.Key); - private bool SupportsVisionUploads => TrySupportsVision(_model?.ChatDeploymentName); - - private string SupportedExtensionsAccept => ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(SupportsVisionUploads); - - private string SupportedExtensionsDisplay => ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(SupportsVisionUploads); - - private bool TrySupportsVision(string deploymentName) - { - if (_model?.DeploymentVisionSupport != null && - !string.IsNullOrWhiteSpace(deploymentName) && - _model.DeploymentVisionSupport.TryGetValue(deploymentName, out var supportsVision)) - { - return supportsVision; - } + private string SupportedExtensionsAccept => ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_model?.AllowImageUploads == true, _model?.AllowDocumentUploads != false); - return _model?.DefaultChatDeploymentSupportsVision == true; - } + private string SupportedExtensionsDisplay => ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_model?.AllowImageUploads == true, _model?.AllowDocumentUploads != false); private bool HasFilteredPromptTemplates => FilteredPromptTemplateGroups.Any(); diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Create.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Create.razor index afbb61ee..b3f10979 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Create.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/ChatInteractions/Create.razor @@ -410,9 +410,29 @@ @if (_model.HasDocumentIndexConfiguration) { -
@(SupportsVisionUploads ? "Attach Documents and Images" : "Attach Documents")
+
+ Documents uploaded here are indexed into the @_model.DocumentIndexProfileName AI Documents index and can be used as background knowledge during chat. +
+ } + else + { +
+ Document knowledge-base search is not configured yet. Create an AI Documents index profile and select it under Settings → AI Settings → Documents before expecting uploaded documents to influence answers. +
+ } + + @if (_model.AllowDocumentUploads || _model.AllowImageUploads) + { + var uploadHeading = _model.AllowImageUploads && _model.AllowDocumentUploads + ? "Attach Documents and Images" + : _model.AllowImageUploads ? "Attach Images" : "Attach Documents"; + var uploadLabel = _model.AllowImageUploads && _model.AllowDocumentUploads + ? "Upload documents and images" + : _model.AllowImageUploads ? "Upload images" : "Upload documents"; + +
@uploadHeading
- +
Supported formats: @SupportedExtensionsDisplay
@@ -453,10 +473,10 @@ string.Equals(d.Name, d.ModelName, StringComparison.OrdinalIgnoreCase) ? d.Name : $"{d.Name} ({d.ModelName})", d.Name)) .ToList(); - _model.DeploymentVisionSupport = deployments - .Where(d => d.Purpose.Supports(AIDeploymentPurpose.Chat)) - .ToDictionary(d => d.Name, d => d.Purpose.Supports(AIDeploymentPurpose.Vision), StringComparer.OrdinalIgnoreCase); - _model.DefaultChatDeploymentSupportsVision = (await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Chat))?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; + var interactionDocSettings = SiteSettingsStore.Get(); + var visionDeployment = await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision); + _model.AllowImageUploads = interactionDocSettings.AllowImageUploads && visionDeployment != null; + _model.AllowDocumentUploads = interactionDocSettings.AllowDocumentUploads; var orchestratorOptions = OrchestratorOptionsAccessor.Value; var orchestrators = orchestratorOptions.GetOrchestratorDescriptors(); @@ -583,22 +603,9 @@ _selectedDocuments = e.GetMultipleFiles().ToList(); } - private bool SupportsVisionUploads => TrySupportsVision(_model.ChatDeploymentName); - - private string SupportedExtensionsAccept => ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(SupportsVisionUploads); - - private string SupportedExtensionsDisplay => ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(SupportsVisionUploads); - - private bool TrySupportsVision(string deploymentName) - { - if (!string.IsNullOrWhiteSpace(deploymentName) && - _model.DeploymentVisionSupport.TryGetValue(deploymentName, out var supportsVision)) - { - return supportsVision; - } + private string SupportedExtensionsAccept => ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_model.AllowImageUploads, _model.AllowDocumentUploads); - return _model.DefaultChatDeploymentSupportsVision; - } + private string SupportedExtensionsDisplay => ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_model.AllowImageUploads, _model.AllowDocumentUploads); private async Task HandleSubmitAsync() { diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Shared/ChatWidget.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Shared/ChatWidget.razor index 663d63dc..27a90887 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Shared/ChatWidget.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Shared/ChatWidget.razor @@ -1,16 +1,18 @@ @using CrestApps.Core.AI -@using CrestApps.Core.AI.Deployments @using CrestApps.Core.AI.Documents.Models @using CrestApps.Core.AI.Models @using CrestApps.Core.AI.Profiles @using CrestApps.Core.Blazor.Web.Models +@using CrestApps.Core.Blazor.Web.Services @using CrestApps.Core.Services @using Microsoft.Extensions.Options +@implements IDisposable @inject IAIProfileManager ProfileManager @inject IOptions ChatDocumentOptions @inject SiteSettingsStore SiteSettings @inject IJSRuntime JS -@inject IAIDeploymentManager DeploymentManager +@inject NavigationManager NavigationManager +@inject SiteSettingsChangedNotifier SettingsChangedNotifier @if (_profile != null) { @@ -89,21 +91,51 @@ private string _supportedExtensionsAccept; private string _supportedExtensionsDisplay; private bool _initialized; - private bool _supportsVisionUploads; + private bool _allowImageUploads; + private string _lastProfileId; protected override async Task OnInitializedAsync() + { + NavigationManager.LocationChanged += OnLocationChanged; + SettingsChangedNotifier.SettingsChanged += OnSettingsChangedAsync; + await RefreshWidgetStateAsync(); + } + + private async void OnLocationChanged(object sender, Microsoft.AspNetCore.Components.Routing.LocationChangedEventArgs e) + { + await RefreshWidgetStateAsync(); + await InvokeAsync(StateHasChanged); + } + + private async Task OnSettingsChangedAsync() + { + await RefreshWidgetStateAsync(); + await InvokeAsync(StateHasChanged); + } + + private async Task RefreshWidgetStateAsync() { var adminWidgetSettings = SiteSettings.Get(); var deploymentDefaults = SiteSettings.Get(); if (string.IsNullOrWhiteSpace(adminWidgetSettings.ProfileId)) { + _profile = null; + return; } + if (!string.Equals(_lastProfileId, adminWidgetSettings.ProfileId, StringComparison.Ordinal)) + { + _lastProfileId = adminWidgetSettings.ProfileId; + _initialized = false; + } + var selectedProfile = await ProfileManager.FindByIdAsync(adminWidgetSettings.ProfileId); if (selectedProfile == null || selectedProfile.Type != AIProfileType.Chat) { + _profile = null; + return; } @@ -137,9 +169,10 @@ _metricsEnabled = selectedProfile.TryGet(out var analyticsMetadata) && analyticsMetadata.EnableSessionMetrics; _allowSessionDocuments = selectedProfile.TryGet(out var sessionDocMeta) && sessionDocMeta.AllowSessionDocuments; - _supportsVisionUploads = await SupportsVisionUploadsAsync(selectedProfile); - _supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_supportsVisionUploads); - _supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_supportsVisionUploads); + _allowImageUploads = sessionDocMeta?.AllowSessionImageUploads == true + && !string.IsNullOrWhiteSpace(deploymentDefaults.DefaultVisionDeploymentName); + _supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(_allowImageUploads, _allowSessionDocuments); + _supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(_allowImageUploads, _allowSessionDocuments); var widgetStyle = string.IsNullOrWhiteSpace(adminWidgetSettings.PrimaryColor) ? null @@ -149,7 +182,7 @@ protected override async Task OnAfterRenderAsync(bool firstRender) { - if (firstRender && _profile != null && !_initialized) + if (_profile != null && !_initialized) { _initialized = true; @@ -176,7 +209,7 @@ chatMode = _chatMode.ToString(), ttsVoiceName = _ttsVoiceName, enableSessionMetrics = _metricsEnabled, - allowSessionDocuments = _allowSessionDocuments, + allowSessionDocuments = _allowSessionDocuments || _allowImageUploads, }, chatConfig = new { @@ -193,7 +226,7 @@ micButtonElementSelector = "#widget-mic-btn", conversationButtonElementSelector = "#widget-conversation-btn", ttsVoiceName = _ttsVoiceName, - sessionDocumentsEnabled = _allowSessionDocuments, + sessionDocumentsEnabled = _allowSessionDocuments || _allowImageUploads, existingDocuments = Array.Empty(), documentBarSelector = "#widget-chat-document-bar", uploadDocumentUrl = "/ai/chat-sessions/upload-document", @@ -217,11 +250,10 @@ } } - private async Task SupportsVisionUploadsAsync(AIProfile profile) + public void Dispose() { - var deployment = await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Chat, deploymentName: profile.ChatDeploymentName) - ?? await DeploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Utility, deploymentName: profile.UtilityDeploymentName); - - return deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; + NavigationManager.LocationChanged -= OnLocationChanged; + SettingsChangedNotifier.SettingsChanged -= OnSettingsChangedAsync; } + } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Program.cs b/src/Startup/CrestApps.Core.Blazor.Web/Program.cs index 52d17629..150522ed 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Program.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/Program.cs @@ -177,6 +177,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); +builder.Services.AddScoped(); var app = builder.Build(); diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Services/SiteSettingsChangedNotifier.cs b/src/Startup/CrestApps.Core.Blazor.Web/Services/SiteSettingsChangedNotifier.cs new file mode 100644 index 00000000..05bed2a3 --- /dev/null +++ b/src/Startup/CrestApps.Core.Blazor.Web/Services/SiteSettingsChangedNotifier.cs @@ -0,0 +1,25 @@ +namespace CrestApps.Core.Blazor.Web.Services; + +/// +/// A scoped service that notifies subscribers when site settings have changed. +/// In Blazor Interactive Server, scoped services live for the duration of the circuit, +/// enabling cross-component communication within the same user session. +/// +public sealed class SiteSettingsChangedNotifier +{ + /// + /// Raised when site settings are saved. + /// + public event Func? SettingsChanged; + + /// + /// Notifies all subscribers that site settings have changed. + /// + public async Task NotifyAsync() + { + if (SettingsChanged != null) + { + await SettingsChanged.Invoke(); + } + } +} diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIProfileViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIProfileViewModel.cs index e49e4e17..6ff14129 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIProfileViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIProfileViewModel.cs @@ -122,6 +122,8 @@ public sealed class AIProfileViewModel public bool AllowSessionDocuments { get; set; } + public bool AllowSessionImageUploads { get; set; } + public bool HasDocumentIndexConfiguration { get; set; } public string DocumentIndexProfileName { get; set; } @@ -330,6 +332,7 @@ public static AIProfileViewModel FromProfile(AIProfile profile) if (profile.TryGet(out var sessionDocMetadata)) { vm.AllowSessionDocuments = sessionDocMetadata.AllowSessionDocuments; + vm.AllowSessionImageUploads = sessionDocMetadata.AllowSessionImageUploads; } if (profile.TryGet(out var analyticsMetadata)) @@ -496,6 +499,7 @@ public void ApplyTo(AIProfile profile) profile.Alter(metadata => { metadata.AllowSessionDocuments = AllowSessionDocuments; + metadata.AllowSessionImageUploads = AllowSessionImageUploads; }); profile.AlterSettings(s => diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AITemplateViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AITemplateViewModel.cs index f844cb7c..e56d9b97 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AITemplateViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AITemplateViewModel.cs @@ -104,6 +104,8 @@ public sealed class AITemplateViewModel // Documents. public bool AllowSessionDocuments { get; set; } + public bool AllowSessionImageUploads { get; set; } + public int? DocumentTopN { get; set; } public DocumentRetrievalMode? DocumentRetrievalMode { get; set; } @@ -266,6 +268,7 @@ public static AITemplateViewModel FromTemplate(AIProfileTemplate template) if (template.TryGet(out var sessionDocMetadata)) { model.AllowSessionDocuments = sessionDocMetadata.AllowSessionDocuments; + model.AllowSessionImageUploads = sessionDocMetadata.AllowSessionImageUploads; } if (template.TryGet(out var docMetadata)) @@ -463,6 +466,7 @@ public void ApplyTo(AIProfileTemplate template) template.Put(new AIProfileSessionDocumentsMetadata { AllowSessionDocuments = AllowSessionDocuments, + AllowSessionImageUploads = AllowSessionImageUploads, }); template.Put(new DocumentsMetadata diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionChatViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionChatViewModel.cs index 04d78cb9..efa08339 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionChatViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionChatViewModel.cs @@ -93,6 +93,6 @@ public sealed class ChatInteractionChatViewModel public List Orchestrators { get; set; } = []; public List CopilotAvailableModels { get; set; } = []; public List AnthropicAvailableModels { get; set; } = []; - public Dictionary DeploymentVisionSupport { get; set; } = []; - public bool DefaultChatDeploymentSupportsVision { get; set; } + public bool AllowImageUploads { get; set; } + public bool AllowDocumentUploads { get; set; } = true; } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionViewModel.cs index 0ac7458a..d7c89a9b 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/ChatInteractionViewModel.cs @@ -79,8 +79,8 @@ public sealed class ChatInteractionViewModel public List Orchestrators { get; set; } = []; public List CopilotAvailableModels { get; set; } = []; public List AnthropicAvailableModels { get; set; } = []; - public Dictionary DeploymentVisionSupport { get; set; } = []; - public bool DefaultChatDeploymentSupportsVision { get; set; } + public bool AllowImageUploads { get; set; } + public bool AllowDocumentUploads { get; set; } = true; } public sealed class SelectOption diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs index 8b0326bc..4462892a 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs @@ -36,6 +36,10 @@ public sealed class SettingsViewModel public DocumentRetrievalMode DocumentRetrievalMode { get; set; } = DocumentRetrievalMode.Chunk; + public bool AllowInteractionImageUploads { get; set; } + + public bool AllowInteractionDocumentUploads { get; set; } = true; + public int DataSourceDefaultStrictness { get; set; } = AIDataSourceSettings.MinStrictness; public int DataSourceDefaultTopNDocuments { get; set; } = AIDataSourceSettings.MinTopNDocuments; diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AIProfileViewModel.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AIProfileViewModel.cs index 59689731..0d483b8c 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AIProfileViewModel.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AIProfileViewModel.cs @@ -113,6 +113,8 @@ public sealed class AIProfileViewModel public bool AllowSessionDocuments { get; set; } + public bool AllowSessionImageUploads { get; set; } + public bool HasDocumentIndexConfiguration { get; set; } public string DocumentIndexProfileName { get; set; } @@ -316,6 +318,7 @@ public static AIProfileViewModel FromProfile(AIProfile profile) if (profile.TryGet(out var sessionDocMetadata)) { vm.AllowSessionDocuments = sessionDocMetadata.AllowSessionDocuments; + vm.AllowSessionImageUploads = sessionDocMetadata.AllowSessionImageUploads; } if (profile.TryGet(out var analyticsMetadata)) @@ -484,6 +487,7 @@ public void ApplyTo(AIProfile profile) profile.Alter(metadata => { metadata.AllowSessionDocuments = AllowSessionDocuments; + metadata.AllowSessionImageUploads = AllowSessionImageUploads; }); profile.AlterSettings(s => diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AITemplateViewModel.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AITemplateViewModel.cs index bd320f18..91e6965e 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AITemplateViewModel.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/ViewModels/AITemplateViewModel.cs @@ -96,6 +96,8 @@ public sealed class AITemplateViewModel // Documents. public bool AllowSessionDocuments { get; set; } + public bool AllowSessionImageUploads { get; set; } + public int? DocumentTopN { get; set; } public DocumentRetrievalMode? DocumentRetrievalMode { get; set; } public bool HasDocumentIndexConfiguration { get; set; } @@ -250,6 +252,7 @@ public static AITemplateViewModel FromTemplate(AIProfileTemplate template) if (template.TryGet(out var sessionDocMetadata)) { model.AllowSessionDocuments = sessionDocMetadata.AllowSessionDocuments; + model.AllowSessionImageUploads = sessionDocMetadata.AllowSessionImageUploads; } if (template.TryGet(out var docMetadata)) @@ -447,6 +450,7 @@ public void ApplyTo(AIProfileTemplate template) template.Put(new AIProfileSessionDocumentsMetadata { AllowSessionDocuments = AllowSessionDocuments, + AllowSessionImageUploads = AllowSessionImageUploads, }); template.Put(new DocumentsMetadata diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Create.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Create.cshtml index ce9b9c56..1b7250c6 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Create.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Create.cshtml @@ -588,9 +588,17 @@
Allow users to upload documents during chat sessions with this profile.
+ +
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
Attach Documents
-

Uploaded files will be indexed as hidden knowledge for this profile after you save it. Supported knowledge formats: @embeddableExtensionsDisplay

+

Uploaded files will be indexed as hidden knowledge for this profile after you save it.

@if (Model.AttachedDocuments.Count > 0) {
@@ -626,7 +634,7 @@ Browse files
-
Supported knowledge formats: @embeddableExtensionsDisplay
+
Supported formats: @embeddableExtensionsDisplay
diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Edit.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Edit.cshtml index 2d329bfa..66e953ab 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Edit.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AIProfile/Edit.cshtml @@ -644,11 +644,19 @@
Allow users to upload documents during chat sessions with this profile.
+
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
+
Attached Documents
-

These files are indexed as background knowledge for this profile and remain hidden from end users. Supported knowledge formats: @embeddableExtensionsDisplay

+

These files are indexed as background knowledge for this profile and remain hidden from end users.

@Model.AttachedDocuments.Count existing
@@ -683,7 +691,7 @@ Browse files
-
Supported knowledge formats: @embeddableExtensionsDisplay
+
Supported formats: @embeddableExtensionsDisplay
diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Create.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Create.cshtml index 8e91c41b..21ebdd45 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Create.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Create.cshtml @@ -588,6 +588,14 @@
Allow users to upload documents during chat sessions with profiles created from this template.
+ +
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Edit.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Edit.cshtml index f4ce4e8b..14820490 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Edit.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AI/Views/AITemplate/Edit.cshtml @@ -578,6 +578,14 @@
Allow users to upload documents during chat sessions with profiles created from this template.
+ +
+
+ + +
+
Allow users to upload images during chat sessions. Requires a vision deployment to be configured in the global AI settings.
+
diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Controllers/AIChatController.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Controllers/AIChatController.cs index 357c74c7..6dad2f1d 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Controllers/AIChatController.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Controllers/AIChatController.cs @@ -1,6 +1,7 @@ using CrestApps.Core.AI; using CrestApps.Core.AI.Chat; using CrestApps.Core.AI.Deployments; +using CrestApps.Core.AI.Documents.Models; using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Profiles; using Microsoft.AspNetCore.Authorization; @@ -54,7 +55,7 @@ public async Task Chat(string sessionId) ViewData["Session"] = session; ViewData["Prompts"] = prompts; - ViewData["SupportsVisionUploads"] = await SupportsVisionUploadsAsync(profile); + ViewData["AllowImageUploads"] = await AllowSessionImageUploadsAsync(profile); return View(profile); } @@ -152,11 +153,15 @@ public async Task Test(string profileId) return View(profile); } - private async Task SupportsVisionUploadsAsync(AIProfile profile) + private async Task AllowSessionImageUploadsAsync(AIProfile profile) { - var deployment = await _deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Chat, deploymentName: profile.ChatDeploymentName) - ?? await _deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Utility, deploymentName: profile.UtilityDeploymentName); + if (!profile.TryGet(out var metadata) || !metadata.AllowSessionImageUploads) + { + return false; + } + + var visionDeployment = await _deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision); - return deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true; + return visionDeployment != null; } } diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Views/AIChat/Chat.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Views/AIChat/Chat.cshtml index 6386e5b6..d29a1db1 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Views/AIChat/Chat.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/AIChat/Views/AIChat/Chat.cshtml @@ -22,10 +22,10 @@ var placeholderMessage = !string.IsNullOrWhiteSpace(Model.WelcomeMessage) ? Model.WelcomeMessage : "What do you want to know?"; - var supportsVisionUploads = ViewData["SupportsVisionUploads"] as bool? == true; - var supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(supportsVisionUploads); - var supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(supportsVisionUploads); + var allowImageUploads = ViewData["AllowImageUploads"] as bool? == true; var allowSessionDocuments = Model.TryGet(out var sessionDocumentsMetadata) && sessionDocumentsMetadata.AllowSessionDocuments; + var supportedExtensionsAccept = ChatDocumentOptions.Value.GetAllowedFileExtensionsAcceptValue(allowImageUploads, allowSessionDocuments); + var supportedExtensionsDisplay = ChatDocumentOptions.Value.GetAllowedFileExtensionsDisplayValue(allowImageUploads, allowSessionDocuments); var metricsEnabled = Model.TryGet(out var analyticsMetadataVal) && analyticsMetadataVal.EnableSessionMetrics; var existingDocuments = session?.Documents ?? []; var defaultDeploymentSettings = SiteSettings.Get(); @@ -195,7 +195,7 @@
-
+