Improve Performance Bottlenecks When Importing Large Tabular Files

src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs

@@ -9,6 +9,7 @@ public static class SystemToolNames
     public const string ListDocuments = "list_documents";
     public const string ReadDocument = "read_document";
     public const string SearchDocuments = "search_documents";
+    public const string GetDocumentMetadata = "get_document_metadata";
     public const string SearchDataSources = "search_data_sources";
     public const string GenerateImage = "generate_image";
     public const string GenerateChart = "generate_chart";

src/Primitives/CrestApps.Core.AI.Documents.OpenXml/OpenXmlServiceCollectionExtensions.cs

@@ -1,6 +1,7 @@
 using CrestApps.Core.AI.Documents.Generation;
 using CrestApps.Core.AI.Documents.Models;
 using CrestApps.Core.AI.Documents.OpenXml.Services;
+using CrestApps.Core.AI.Documents.Tabular;
 using CrestApps.Core.Builders;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -23,6 +24,14 @@ public static IServiceCollection AddCoreAIOpenXmlDocumentProcessing(this IServic
             ".docx",
             new ExtractorExtension(".xlsx", embeddable: false, isTabular: true),
             ".pptx");
+        services.AddSingleton<OpenXmlTabularDocumentArtifactBuilder>();
+        services.AddSingleton<OpenXmlTabularWorkspaceImporter>();
+        services.AddKeyedSingleton<ITabularDocumentArtifactBuilder>(
+            ".xlsx",
+            (sp, _) => sp.GetRequiredService<OpenXmlTabularDocumentArtifactBuilder>());
+        services.AddKeyedSingleton<ITabularWorkspaceImporter>(
+            ".xlsx",
+            (sp, _) => sp.GetRequiredService<OpenXmlTabularWorkspaceImporter>());
 
         // Register Open XML output writers so generated files and tabular exports can target xlsx/docx.
         services.AddGeneratedFileWriter<SpreadsheetGeneratedFileWriter>(".xlsx");

src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlIngestionDocumentReader.cs

@@ -37,28 +37,44 @@ public override async Task<IngestionDocument> ReadAsync(
 
         var document = new IngestionDocument(identifier);
 
+        MemoryStream buffer = null;
+        var workingStream = source;
+
         if (source.CanSeek)
         {
             source.Position = 0;
         }
-
-        await using var buffer = new MemoryStream();
-        await source.CopyToAsync(buffer, cancellationToken);
-        buffer.Position = 0;
+        else
+        {
+            buffer = new MemoryStream();
+            await source.CopyToAsync(buffer, cancellationToken);
+            buffer.Position = 0;
+            workingStream = buffer;
+        }
 
         cancellationToken.ThrowIfCancellationRequested();
 
-        var section = mediaType switch
+        try
         {
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => ExtractWord(buffer, cancellationToken),
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => ExtractExcel(buffer, cancellationToken),
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation" => ExtractPowerPoint(buffer, cancellationToken),
-            _ => null,
-        };
+            var section = mediaType switch
+            {
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => ExtractWord(workingStream, cancellationToken),
+                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => ExtractExcel(workingStream, cancellationToken),
+                "application/vnd.openxmlformats-officedocument.presentationml.presentation" => ExtractPowerPoint(workingStream, cancellationToken),
+                _ => null,
+            };
 
-        if (section != null)
+            if (section != null)
+            {
+                document.Sections.Add(section);
+            }
+        }
+        finally
         {
-            document.Sections.Add(section);
+            if (buffer != null)
+            {
+                await buffer.DisposeAsync();
+            }
         }
 
         return document;

src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularDocumentArtifactBuilder.cs

@@ -0,0 +1,102 @@
+using System.Diagnostics;
+using CrestApps.Core.AI.Documents.Tabular;
+using DocumentFormat.OpenXml.Packaging;
+using Microsoft.Extensions.Logging;
+
+namespace CrestApps.Core.AI.Documents.OpenXml.Services;
+
+/// <summary>
+/// Builds tabular artifacts from Open XML spreadsheets using a sheet-streaming fast path that avoids
+/// materializing the generic ingestion document graph first.
+/// </summary>
+public sealed class OpenXmlTabularDocumentArtifactBuilder : ITabularDocumentArtifactBuilder
+{
+    private readonly ILogger<OpenXmlTabularDocumentArtifactBuilder> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="OpenXmlTabularDocumentArtifactBuilder"/> class.
+    /// </summary>
+    /// <param name="logger">The logger.</param>
+    public OpenXmlTabularDocumentArtifactBuilder(ILogger<OpenXmlTabularDocumentArtifactBuilder> logger)
+    {
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Creates a tabular artifact from an Open XML spreadsheet stream.
+    /// </summary>
+    /// <param name="source">The spreadsheet stream.</param>
+    /// <param name="fileName">The source file name.</param>
+    /// <param name="contentType">The source content type.</param>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns>The parsed tabular artifact.</returns>
+    public Task<TabularDocumentArtifact> CreateAsync(
+        Stream source,
+        string fileName,
+        string contentType,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(source);
+
+        if (source.CanSeek)
+        {
+            source.Position = 0;
+        }
+
+        var stopwatch = Stopwatch.StartNew();
+
+        using var document = SpreadsheetDocument.Open(source, false);
+
+        var workbookPart = document.WorkbookPart;
+
+        if (workbookPart == null)
+        {
+            return Task.FromResult(new TabularDocumentArtifact());
+        }
+
+        List<string> header = null;
+        var rows = new List<List<string>>(4096);
+        OpenXmlTabularWorksheetReader.ReadNonEmptyRows(
+            workbookPart,
+            fileName,
+            _logger,
+            (row, firstNonEmptyRowInWorksheet) =>
+            {
+                if (header == null)
+                {
+                    header = row;
+
+                    return;
+                }
+
+                if (!firstNonEmptyRowInWorksheet)
+                {
+                    rows.Add(row);
+                }
+            },
+            cancellationToken);
+
+        if (header == null)
+        {
+            return Task.FromResult(new TabularDocumentArtifact());
+        }
+
+        var artifact = new TabularDocumentArtifact
+        {
+            Header = header,
+            Rows = rows,
+        };
+
+        if (_logger.IsEnabled(LogLevel.Debug))
+        {
+            _logger.LogDebug(
+                "OpenXml tabular builder created artifact for '{FileName}' with {ColumnCount} column(s) and {RowCount} row(s) in {ElapsedMilliseconds} ms.",
+                fileName,
+                artifact.Header.Count,
+                artifact.Rows.Count,
+                stopwatch.ElapsedMilliseconds);
+        }
+
+        return Task.FromResult(artifact);
+    }
+}