Improve Vision Support to Reduce Costs and Increase Flexibility

src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs

@@ -13,4 +13,5 @@ public static class SystemToolNames
     public const string ReadTabularData = "read_tabular_data";
     public const string GenerateImage = "generate_image";
     public const string GenerateChart = "generate_chart";
+    public const string InspectImage = "inspect_image";
 }

src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md

@@ -74,3 +74,4 @@ description: Initial standalone release notes for the CrestApps.Core repository.
 - adds `CrestApps.Core.PostgreSQL` and `CrestApps.Core.AI.PostgreSQL` packages providing a lightweight PostgreSQL + pgvector vector search backend as an alternative to Elasticsearch and Azure AI Search, registers the same keyed services (`ISearchIndexManager`, `ISearchDocumentManager`, `IDataSourceContentManager`, `IDataSourceDocumentReader`, `IODataFilterTranslator`) under the `"PostgreSQL"` provider name, supports `AddAIDocuments()`, `AddAIDataSources()`, and `AddAIMemory()` builder extensions, and integrates into both MVC and Blazor sample hosts
 - fixes hosted document and data-source indexing flows so background workers create a scoped service provider before resolving scoped indexing services, preventing upload-triggered failures and similar nightly alignment lifetime issues
 - standardizes Azure AI Search configuration on top-level `AuthenticationType`, `ApiKey`, `IdentityClientId`, and `IndexPrefix` settings under `CrestApps:AzureAISearch`, and refreshes the sample host / docs examples to list the full supported option set in one place
+- replaces per-turn raw image byte injection with an analyze-once-at-upload strategy: `IImageAnalysisService` calls a vision model to extract caption, OCR text, and detected entities when images are uploaded, stores the results as `AIDocumentChunk` records searchable via `read_document` and `search_documents`, adds `inspect_image` as an on-demand tool for pixel-level inspection when the text analysis is insufficient, removes `BuildVisionUserContentsAsync` from `DocumentOrchestrationHandler` so image bytes are never attached to every user message, and updates the document-availability prompt to guide the model toward text-based tools first

src/CrestApps.Core.Docs/docs/core/ai-documents.md

@@ -68,9 +68,11 @@ Users upload documents (PDFs, Word files, spreadsheets, text files) and expect t
 
 The document processing system handles this full pipeline from upload to retrieval, while the built-in document tools make the content available to the AI during orchestration.
 
-When a chat deployment also supports the `Vision` purpose, chat interaction and chat session uploads can include supported image formats (`.bmp`, `.gif`, `.jpeg`, `.jpg`, `.png`, `.webp`) alongside standard document files. Those images are stored as `AIDocument` records and attached to the current user message as multimodal content instead of going through text extraction and chunk embedding. The shared document-availability prompt distinguishes image attachments from searchable documents so the model analyzes supported uploaded images directly instead of treating them like text-only document metadata.
+When a chat deployment also supports the `Vision` purpose, chat interaction and chat session uploads can include supported image formats (`.bmp`, `.gif`, `.jpeg`, `.jpg`, `.png`, `.webp`) alongside standard document files. Those images are stored as `AIDocument` records, analyzed at upload time by `IImageAnalysisService` to extract a structured summary (caption, OCR text, detected entities), and the results are persisted as `AIDocumentChunk` records — exactly like text documents. This makes image content available through the same `read_document` and `search_documents` tools used for regular documents.
 
-`DocumentOrchestrationHandler` still has to materialize image bytes before building `DataContent`, but it now reads directly into the target byte buffer and honors `ChatDocumentsOptions.MaxVisionInputBytesPerRequest` so a single request cannot pull an unbounded batch of uploaded images into memory.
+For cases where the text analysis is insufficient (e.g., reading fine text, comparing visual elements, or understanding spatial layout), the `inspect_image` tool provides on-demand raw image inspection by sending the original bytes to a vision model in a one-shot call. This approach eliminates the cost of attaching raw image bytes to every chat request while preserving full visual inspection capability when needed.
+
+The `ChatDocumentsOptions.AnalyzeImagesAtUpload` setting controls whether analysis runs at upload time, and `MaxInspectImageCallsPerRequest` limits how many costly raw-image inspections the model can perform per turn.
 
 ### Creating a chat client to describe an image
 
@@ -144,6 +146,7 @@ public sealed class ImageDescriptionService(
        │  • SearchDocumentsTool (vector RAG)  │
        │  • ReadDocumentTool (full text read) │
        │  • ReadTabularDataTool (CSV/Excel)   │
+       │  • InspectImageTool (vision on-demand)│
        └──────────────┬──────────────────────┘
                       ▼
        ┌─────────────────────────────────────┐

src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs

@@ -26,12 +26,15 @@ public abstract class AIChatDocumentEndpointBase
         string referenceType,
         ChatDocumentsOptions documentOptions,
         IAIDocumentProcessingService documentProcessingService,
+        IImageAnalysisService imageAnalysisService,
         IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
         IAIDocumentStore documentStore,
         IAIDocumentChunkStore chunkStore,
         IDocumentFileStore fileStore,
         TimeProvider timeProvider,
         bool allowVisionImages,
+        bool allowDocumentUploads,
+        string chatDeploymentName,
         ILogger logger,
         IStringLocalizer S)
     {
@@ -49,7 +52,12 @@ public abstract class AIChatDocumentEndpointBase
 
         if (MediaTypeHelper.IsVisionImageExtension(extension) && !allowVisionImages)
         {
-            return (false, S["Image uploads require a vision-capable chat deployment."].Value, null);
+            return (false, S["Image uploads are not enabled or no vision deployment is configured."].Value, null);
+        }
+
+        if (!MediaTypeHelper.IsVisionImageExtension(extension) && !allowDocumentUploads)
+        {
+            return (false, S["Document uploads are not enabled."].Value, null);
         }
 
         if (allowVisionImages && MediaTypeHelper.IsVisionImageExtension(extension) &&
@@ -58,7 +66,7 @@ public abstract class AIChatDocumentEndpointBase
             return (false, S["The uploaded image exceeds the maximum allowed size of {0} MB.", documentOptions.MaxVisionImageBytesPerFile / (1024 * 1024)].Value, null);
         }
 
-        if (!documentOptions.IsAllowedFileExtension(extension, allowVisionImages))
+        if (!documentOptions.IsAllowedFileExtension(extension, allowVisionImages, allowDocumentUploads))
         {
             return (false, S["File type '{0}' is not supported.", extension].Value, null);
         }
@@ -74,7 +82,19 @@ public abstract class AIChatDocumentEndpointBase
         {
             if (allowVisionImages && MediaTypeHelper.IsVisionImageExtension(extension))
             {
-                return await ProcessVisionImageAsync(file, referenceId, referenceType, documentStore, fileStore, timeProvider);
+                return await ProcessVisionImageAsync(
+                    file,
+                    referenceId,
+                    referenceType,
+                    documentOptions,
+                    imageAnalysisService,
+                    embeddingGenerator,
+                    documentStore,
+                    chunkStore,
+                    fileStore,
+                    timeProvider,
+                    chatDeploymentName,
+                    logger);
             }
 
             var result = await documentProcessingService.ProcessFileAsync(file, referenceId, referenceType, embeddingGenerator);
@@ -89,15 +109,6 @@ public abstract class AIChatDocumentEndpointBase
         }
     }
 
-    /// <summary>
-    /// Determines whether the deployment supports vision uploads.
-    /// </summary>
-    /// <param name="deployment">The deployment.</param>
-    protected static bool SupportsVisionUploads(AIDeployment deployment)
-    {
-        return deployment?.Purpose.Supports(AIDeploymentPurpose.Vision) == true;
-    }
-
     /// <summary>
     /// Gets files.
     /// </summary>
@@ -115,6 +126,17 @@ protected static IReadOnlyList<IFormFile> GetFiles(IFormCollection form)
         return singleFile == null ? [] : [singleFile];
     }
 
+    /// <summary>
+    /// Determines whether session document upload is enabled for the profile.
+    /// Returns <c>true</c> when either document uploads or image uploads are allowed.
+    /// </summary>
+    /// <param name="profile">The profile.</param>
+    protected static bool IsSessionUploadEnabled(AIProfile profile)
+    {
+        return profile.TryGet<AIProfileSessionDocumentsMetadata>(out var metadata)
+            && (metadata.AllowSessionDocuments || metadata.AllowSessionImageUploads);
+    }
+
     /// <summary>
     /// Determines whether session document upload enabled.
     /// </summary>
@@ -227,9 +249,15 @@ protected static async Task InvokeRemovedHandlersAsync(IEnumerable<IAIChatDocume
         IFormFile file,
         string referenceId,
         string referenceType,
+        ChatDocumentsOptions documentOptions,
+        IImageAnalysisService imageAnalysisService,
+        IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
         IAIDocumentStore documentStore,
+        IAIDocumentChunkStore chunkStore,
         IDocumentFileStore fileStore,
-        TimeProvider timeProvider)
+        TimeProvider timeProvider,
+        string chatDeploymentName,
+        ILogger logger)
     {
         var now = timeProvider.GetUtcNow().UtcDateTime;
         var contentType = MediaTypeHelper.InferMediaType(Path.GetExtension(file.FileName), file.ContentType);
@@ -269,11 +297,137 @@ protected static async Task InvokeRemovedHandlersAsync(IEnumerable<IAIChatDocume
 
         await documentStore.CreateAsync(document);
 
+        // Analyze the image at upload time to create searchable text chunks.
+        IReadOnlyList<AIDocumentChunk> chunks = [];
+
+        if (documentOptions.AnalyzeImagesAtUpload && imageAnalysisService != null)
+        {
+            chunks = await AnalyzeAndStoreImageChunksAsync(
+                file,
+                document,
+                imageAnalysisService,
+                embeddingGenerator,
+                chunkStore,
+                chatDeploymentName,
+                logger);
+        }
+
         return (true, null, new AIChatUploadedDocument
         {
             File = file,
             Document = document,
             DocumentInfo = documentInfo,
+            Chunks = chunks,
         });
     }
+
+    private static async Task<IReadOnlyList<AIDocumentChunk>> AnalyzeAndStoreImageChunksAsync(
+        IFormFile file,
+        AIDocument document,
+        IImageAnalysisService imageAnalysisService,
+        IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
+        IAIDocumentChunkStore chunkStore,
+        string chatDeploymentName,
+        ILogger logger)
+    {
+        try
+        {
+            ImageAnalysisResult analysis;
+
+            using (var stream = file.OpenReadStream())
+            {
+                analysis = await imageAnalysisService.AnalyzeAsync(
+                    stream,
+                    document.ContentType,
+                    document.FileName,
+                    chatDeploymentName);
+            }
+
+            if (!analysis.Success)
+            {
+                logger.LogWarning(
+                    "Image analysis failed for document '{DocumentId}' ({FileName}): {Error}",
+                    document.ItemId,
+                    document.FileName,
+                    analysis.Error);
+
+                return [];
+            }
+
+            var chunks = BuildImageAnalysisChunks(document, analysis);
+
+            if (embeddingGenerator != null && chunks.Count > 0)
+            {
+                var textsToEmbed = chunks.Select(c => c.Content).ToList();
+                var embeddings = await embeddingGenerator.GenerateAsync(textsToEmbed);
+
+                for (var i = 0; i < chunks.Count && i < embeddings.Count; i++)
+                {
+                    chunks[i].Embedding = embeddings[i].Vector.ToArray();
+                }
+            }
+
+            foreach (var chunk in chunks)
+            {
+                await chunkStore.CreateAsync(chunk);
+            }
+
+            return chunks;
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to analyze image '{FileName}' for document '{DocumentId}'.", document.FileName, document.ItemId);
+
+            return [];
+        }
+    }
+
+    private static List<AIDocumentChunk> BuildImageAnalysisChunks(AIDocument document, ImageAnalysisResult analysis)
+    {
+        var chunks = new List<AIDocumentChunk>();
+        var index = 0;
+
+        if (!string.IsNullOrWhiteSpace(analysis.Caption))
+        {
+            chunks.Add(CreateChunk(document, $"[Image Caption]\n{analysis.Caption}", index++));
+        }
+
+        if (!string.IsNullOrWhiteSpace(analysis.Description))
+        {
+            chunks.Add(CreateChunk(document, $"[Image Description]\n{analysis.Description}", index++));
+        }
+
+        if (!string.IsNullOrWhiteSpace(analysis.OcrText) &&
+            !string.Equals(analysis.OcrText, "None", StringComparison.OrdinalIgnoreCase))
+        {
+            chunks.Add(CreateChunk(document, $"[OCR Text]\n{analysis.OcrText}", index++));
+        }
+
+        if (!string.IsNullOrWhiteSpace(analysis.DetectedEntities) &&
+            !string.Equals(analysis.DetectedEntities, "None", StringComparison.OrdinalIgnoreCase))
+        {
+            chunks.Add(CreateChunk(document, $"[Detected Entities]\n{analysis.DetectedEntities}", index++));
+        }
+
+        // If no structured sections were produced, store the raw analysis as a single chunk.
+        if (chunks.Count == 0 && !string.IsNullOrWhiteSpace(analysis.RawAnalysis))
+        {
+            chunks.Add(CreateChunk(document, analysis.RawAnalysis, index));
+        }
+
+        return chunks;
+    }
+
+    private static AIDocumentChunk CreateChunk(AIDocument document, string content, int index)
+    {
+        return new AIDocumentChunk
+        {
+            ItemId = UniqueId.GenerateId(),
+            AIDocumentId = document.ItemId,
+            ReferenceId = document.ReferenceId,
+            ReferenceType = document.ReferenceType,
+            Content = content,
+            Index = index,
+        };
+    }
 }

src/Primitives/CrestApps.Core.AI.Documents/Endpoints/UploadChatInteractionDocument.cs

@@ -53,6 +53,7 @@ private sealed class UploadChatInteractionDocumentEndpoint : AIChatDocumentEndpo
         /// <param name="chunkStore">The chunk store.</param>
         /// <param name="fileStore">The file store.</param>
         /// <param name="documentProcessingService">The document processing service.</param>
+        /// <param name="imageAnalysisService">The image analysis service.</param>
         /// <param name="authorizationService">The authorization service.</param>
         /// <param name="eventHandlers">The event handlers.</param>
         /// <param name="documentOptions">The document options.</param>
@@ -67,10 +68,12 @@ public static async Task<IResult> HandleAsync(
             [FromServices] IAIDocumentChunkStore chunkStore,
             [FromServices] IDocumentFileStore fileStore,
             [FromServices] IAIDocumentProcessingService documentProcessingService,
+            [FromServices] IImageAnalysisService imageAnalysisService,
             [FromServices] TimeProvider timeProvider,
             [FromServices] IAuthorizationService authorizationService,
             [FromServices] IEnumerable<IAIChatDocumentEventHandler> eventHandlers,
             [FromServices] IOptions<ChatDocumentsOptions> documentOptions,
+            [FromServices] IOptions<InteractionDocumentOptions> interactionDocumentOptions,
             [FromServices] ILoggerFactory loggerFactory,
             [FromServices] IStringLocalizerFactory localizerFactory)
         {
@@ -123,7 +126,10 @@ public static async Task<IResult> HandleAsync(
             }
 
             var embeddingGenerator = embeddingDeployment == null ? null : await aiClientFactory.CreateEmbeddingGeneratorAsync(embeddingDeployment);
-            var allowVisionImages = SupportsVisionUploads(deployment);
+            var visionDeployment = await deploymentManager.ResolveOrDefaultAsync(AIDeploymentPurpose.Vision);
+            var interactionDocOptions = interactionDocumentOptions.Value;
+            var allowVisionImages = interactionDocOptions.AllowImageUploads && visionDeployment != null;
+            var allowDocumentUploads = interactionDocOptions.AllowDocumentUploads;
             if (logger.IsEnabled(LogLevel.Information))
             {
                 logger.LogInformation("Created embedding generator for interaction '{InteractionId}': {HasEmbeddingGenerator}.", interaction.ItemId, embeddingGenerator != null);
@@ -156,12 +162,15 @@ public static async Task<IResult> HandleAsync(
                     AIReferenceTypes.Document.ChatInteraction,
                     documentOptions.Value,
                     documentProcessingService,
+                    imageAnalysisService,
                     embeddingGenerator,
                     documentStore,
                     chunkStore,
                     fileStore,
                     timeProvider,
                     allowVisionImages,
+                    allowDocumentUploads,
+                    visionDeployment?.Name,
                     logger,
                     S);
                 if (!result.Success)