diff --git a/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs b/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs index 16a7b948..fa1db190 100644 --- a/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs +++ b/src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/SystemToolNames.cs @@ -9,6 +9,7 @@ public static class SystemToolNames public const string ListDocuments = "list_documents"; public const string ReadDocument = "read_document"; public const string SearchDocuments = "search_documents"; + public const string GetDocumentMetadata = "get_document_metadata"; public const string SearchDataSources = "search_data_sources"; public const string GenerateImage = "generate_image"; public const string GenerateChart = "generate_chart"; diff --git a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/OpenXmlServiceCollectionExtensions.cs b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/OpenXmlServiceCollectionExtensions.cs index e52f915a..2fd61381 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/OpenXmlServiceCollectionExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/OpenXmlServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using CrestApps.Core.AI.Documents.Generation; using CrestApps.Core.AI.Documents.Models; using CrestApps.Core.AI.Documents.OpenXml.Services; +using CrestApps.Core.AI.Documents.Tabular; using CrestApps.Core.Builders; using Microsoft.Extensions.DependencyInjection; @@ -23,6 +24,14 @@ public static IServiceCollection AddCoreAIOpenXmlDocumentProcessing(this IServic ".docx", new ExtractorExtension(".xlsx", embeddable: false, isTabular: true), ".pptx"); + services.AddSingleton(); + services.AddSingleton(); + services.AddKeyedSingleton( + ".xlsx", + (sp, _) => sp.GetRequiredService()); + services.AddKeyedSingleton( + ".xlsx", + (sp, _) => sp.GetRequiredService()); // Register Open XML output writers so generated files and tabular exports can target xlsx/docx. services.AddGeneratedFileWriter(".xlsx"); diff --git a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlIngestionDocumentReader.cs b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlIngestionDocumentReader.cs index 29377b3b..4b0dcc0a 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlIngestionDocumentReader.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlIngestionDocumentReader.cs @@ -37,28 +37,44 @@ public override async Task ReadAsync( var document = new IngestionDocument(identifier); + MemoryStream buffer = null; + var workingStream = source; + if (source.CanSeek) { source.Position = 0; } - - await using var buffer = new MemoryStream(); - await source.CopyToAsync(buffer, cancellationToken); - buffer.Position = 0; + else + { + buffer = new MemoryStream(); + await source.CopyToAsync(buffer, cancellationToken); + buffer.Position = 0; + workingStream = buffer; + } cancellationToken.ThrowIfCancellationRequested(); - var section = mediaType switch + try { - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => ExtractWord(buffer, cancellationToken), - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => ExtractExcel(buffer, cancellationToken), - "application/vnd.openxmlformats-officedocument.presentationml.presentation" => ExtractPowerPoint(buffer, cancellationToken), - _ => null, - }; + var section = mediaType switch + { + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => ExtractWord(workingStream, cancellationToken), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => ExtractExcel(workingStream, cancellationToken), + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => ExtractPowerPoint(workingStream, cancellationToken), + _ => null, + }; - if (section != null) + if (section != null) + { + document.Sections.Add(section); + } + } + finally { - document.Sections.Add(section); + if (buffer != null) + { + await buffer.DisposeAsync(); + } } return document; diff --git a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularDocumentArtifactBuilder.cs b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularDocumentArtifactBuilder.cs new file mode 100644 index 00000000..7f9ddda4 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularDocumentArtifactBuilder.cs @@ -0,0 +1,102 @@ +using System.Diagnostics; +using CrestApps.Core.AI.Documents.Tabular; +using DocumentFormat.OpenXml.Packaging; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Documents.OpenXml.Services; + +/// +/// Builds tabular artifacts from Open XML spreadsheets using a sheet-streaming fast path that avoids +/// materializing the generic ingestion document graph first. +/// +public sealed class OpenXmlTabularDocumentArtifactBuilder : ITabularDocumentArtifactBuilder +{ + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + public OpenXmlTabularDocumentArtifactBuilder(ILogger logger) + { + _logger = logger; + } + + /// + /// Creates a tabular artifact from an Open XML spreadsheet stream. + /// + /// The spreadsheet stream. + /// The source file name. + /// The source content type. + /// The cancellation token. + /// The parsed tabular artifact. + public Task CreateAsync( + Stream source, + string fileName, + string contentType, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(source); + + if (source.CanSeek) + { + source.Position = 0; + } + + var stopwatch = Stopwatch.StartNew(); + + using var document = SpreadsheetDocument.Open(source, false); + + var workbookPart = document.WorkbookPart; + + if (workbookPart == null) + { + return Task.FromResult(new TabularDocumentArtifact()); + } + + List header = null; + var rows = new List>(4096); + OpenXmlTabularWorksheetReader.ReadNonEmptyRows( + workbookPart, + fileName, + _logger, + (row, firstNonEmptyRowInWorksheet) => + { + if (header == null) + { + header = row; + + return; + } + + if (!firstNonEmptyRowInWorksheet) + { + rows.Add(row); + } + }, + cancellationToken); + + if (header == null) + { + return Task.FromResult(new TabularDocumentArtifact()); + } + + var artifact = new TabularDocumentArtifact + { + Header = header, + Rows = rows, + }; + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "OpenXml tabular builder created artifact for '{FileName}' with {ColumnCount} column(s) and {RowCount} row(s) in {ElapsedMilliseconds} ms.", + fileName, + artifact.Header.Count, + artifact.Rows.Count, + stopwatch.ElapsedMilliseconds); + } + + return Task.FromResult(artifact); + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorksheetReader.cs b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorksheetReader.cs new file mode 100644 index 00000000..bd2ca9c7 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorksheetReader.cs @@ -0,0 +1,294 @@ +using System.Text; +using System.Xml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Documents.OpenXml.Services; + +internal static class OpenXmlTabularWorksheetReader +{ + private static readonly XmlReaderSettings _xmlReaderSettings = new() + { + IgnoreComments = true, + IgnoreWhitespace = true, + }; + + public static string[] CreateSharedStringCache(WorkbookPart workbookPart) + { + var table = workbookPart.SharedStringTablePart?.SharedStringTable; + + if (table == null) + { + return null; + } + + var cache = new string[table.ChildElements.Count]; + var index = 0; + + foreach (SharedStringItem item in table.Elements()) + { + cache[index++] = item.InnerText; + } + + return cache; + } + + public static void ReadNonEmptyRows( + WorkbookPart workbookPart, + string fileName, + ILogger logger, + Action, bool> rowHandler, + CancellationToken cancellationToken) + { + var sharedStrings = CreateSharedStringCache(workbookPart); + var expectedColumnCount = 16; + + foreach (var worksheetPart in workbookPart.WorksheetParts) + { + cancellationToken.ThrowIfCancellationRequested(); + + var firstNonEmptyRowInWorksheet = true; + var sheetRowCount = 0; + + using var stream = worksheetPart.GetStream(FileMode.Open, FileAccess.Read); + using var reader = XmlReader.Create(stream, _xmlReaderSettings); + + while (!reader.EOF) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (reader.NodeType != XmlNodeType.Element || + !string.Equals(reader.LocalName, "row", StringComparison.Ordinal)) + { + reader.Read(); + + continue; + } + + using var rowReader = reader.ReadSubtree(); + var row = ReadRow(rowReader, sharedStrings, expectedColumnCount, out var hasValue); + reader.Skip(); + + if (!hasValue) + { + continue; + } + + expectedColumnCount = Math.Max(expectedColumnCount, row.Count); + rowHandler(row, firstNonEmptyRowInWorksheet); + firstNonEmptyRowInWorksheet = false; + sheetRowCount++; + } + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug( + "OpenXml tabular builder read {RowCount} non-empty row(s) from worksheet '{WorksheetUri}' for '{FileName}'.", + sheetRowCount, + worksheetPart.Uri, + fileName); + } + } + } + + private static List ReadRow( + XmlReader reader, + string[] sharedStrings, + int expectedColumnCount, + out bool hasValue) + { + hasValue = false; + var values = new List(expectedColumnCount); + + if (!reader.Read() || reader.IsEmptyElement) + { + return values; + } + + while (reader.Read()) + { + if (reader.NodeType != XmlNodeType.Element || + !string.Equals(reader.LocalName, "c", StringComparison.Ordinal)) + { + continue; + } + + var columnIndex = GetColumnIndex( + reader.GetAttribute("r"), + values.Count); + + while (values.Count < columnIndex) + { + values.Add(string.Empty); + } + + var value = GetCellValue(reader, reader.GetAttribute("t"), sharedStrings); + + values.Add(value); + hasValue |= !string.IsNullOrEmpty(value); + } + + TrimTrailingEmptyValues(values); + + return values; + } + + private static void TrimTrailingEmptyValues(List values) + { + var last = values.Count - 1; + + while (last >= 0 && string.IsNullOrEmpty(values[last])) + { + last--; + } + + var removeCount = values.Count - last - 1; + + if (removeCount > 0) + { + values.RemoveRange( + last + 1, + removeCount); + } + } + + private static int GetColumnIndex(string cellReference, int fallbackIndex) + { + if (string.IsNullOrEmpty(cellReference)) + { + return fallbackIndex; + } + + var columnIndex = 0; + var foundColumn = false; + + foreach (var c in cellReference) + { + if (c >= 'A' && c <= 'Z') + { + columnIndex = columnIndex * 26 + c - 'A' + 1; + foundColumn = true; + } + else if (c >= 'a' && c <= 'z') + { + columnIndex = columnIndex * 26 + c - 'a' + 1; + foundColumn = true; + } + else + { + break; + } + } + + return foundColumn + ? columnIndex - 1 + : fallbackIndex; + } + + private static string GetCellValue( + XmlReader reader, + string cellType, + string[] sharedStrings) + { + if (reader.IsEmptyElement) + { + return string.Empty; + } + + var cellDepth = reader.Depth; + string value = null; + string inlineText = null; + StringBuilder inlineBuilder = null; + + while (reader.Read()) + { + if (reader.NodeType == XmlNodeType.EndElement && + reader.Depth == cellDepth && + string.Equals(reader.LocalName, "c", StringComparison.Ordinal)) + { + break; + } + + if (reader.NodeType != XmlNodeType.Element) + { + continue; + } + + if (string.Equals(reader.LocalName, "v", StringComparison.Ordinal)) + { + value = reader.ReadElementContentAsString(); + + if (reader.NodeType == XmlNodeType.EndElement && + reader.Depth == cellDepth && + string.Equals(reader.LocalName, "c", StringComparison.Ordinal)) + { + break; + } + + continue; + } + + if (!string.Equals(cellType, "inlineStr", StringComparison.Ordinal) || + !string.Equals(reader.LocalName, "t", StringComparison.Ordinal)) + { + continue; + } + + var text = reader.ReadElementContentAsString(); + + if (inlineBuilder != null) + { + inlineBuilder.Append(text); + } + else if (inlineText == null) + { + inlineText = text; + } + else + { + inlineBuilder = new StringBuilder(inlineText.Length + text.Length); + inlineBuilder.Append(inlineText); + inlineBuilder.Append(text); + } + + if (reader.NodeType == XmlNodeType.EndElement && + reader.Depth == cellDepth && + string.Equals(reader.LocalName, "c", StringComparison.Ordinal)) + { + break; + } + } + + if (inlineBuilder != null) + { + value = inlineBuilder.ToString(); + } + else if (inlineText != null) + { + value = inlineText; + } + + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + if (string.Equals(cellType, "s", StringComparison.Ordinal) && + sharedStrings != null && + int.TryParse(value, out var index) && + (uint)index < (uint)sharedStrings.Length) + { + return sharedStrings[index]; + } + + if (string.Equals(cellType, "b", StringComparison.Ordinal)) + { + return value == "1" + ? "TRUE" + : "FALSE"; + } + + return value; + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorkspaceImporter.cs b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorkspaceImporter.cs new file mode 100644 index 00000000..260831a4 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents.OpenXml/Services/OpenXmlTabularWorkspaceImporter.cs @@ -0,0 +1,184 @@ +using System.Diagnostics; +using CrestApps.Core.AI.Documents.Tabular; +using DocumentFormat.OpenXml.Packaging; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Documents.OpenXml.Services; + +/// +/// Streams Open XML spreadsheet rows directly into a SQLite tabular workspace. +/// +public sealed class OpenXmlTabularWorkspaceImporter : ITabularWorkspaceImporter +{ + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + public OpenXmlTabularWorkspaceImporter(ILogger logger) + { + _logger = logger; + } + + /// + /// Imports an Open XML spreadsheet into the supplied SQLite workspace table. + /// + /// The spreadsheet stream. + /// The source file name. + /// The source content type. + /// The SQLite workspace connection. + /// The destination table name. + /// The cancellation token. + /// The import result. + public Task ImportAsync( + Stream source, + string fileName, + string contentType, + SqliteConnection connection, + string tableName, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(connection); + ArgumentException.ThrowIfNullOrEmpty(tableName); + + if (source.CanSeek) + { + source.Position = 0; + } + + var stopwatch = Stopwatch.StartNew(); + using var document = SpreadsheetDocument.Open(source, false); + var workbookPart = document.WorkbookPart; + + if (workbookPart == null) + { + TabularWorkspaceSqliteHelpers.CreateEmptyPlaceholderTable(connection, tableName); + + return Task.FromResult(new TabularWorkspaceImportResult( + [new TabularColumnInfo("value", "TEXT")], + 0, + 0, + 1)); + } + + List header = null; + IReadOnlyList columns = null; + SqliteCommand insertCommand = null; + SqliteTransaction transaction = null; + var rowCount = 0; + var insertCommandCount = 0; + + try + { + OpenXmlTabularWorksheetReader.ReadNonEmptyRows( + workbookPart, + fileName, + _logger, + (row, firstNonEmptyRowInWorksheet) => + { + if (header == null) + { + header = row; + columns = TabularWorkspaceSqliteHelpers.BuildColumns(header); + TabularWorkspaceSqliteHelpers.CreateTable(connection, tableName, columns); + transaction = connection.BeginTransaction(); + insertCommand = CreateInsertCommand(connection, transaction, tableName, columns); + + return; + } + + if (firstNonEmptyRowInWorksheet) + { + return; + } + + BindInsertParameters(insertCommand, row, columns.Count); + insertCommand.ExecuteNonQuery(); + rowCount++; + insertCommandCount++; + }, + cancellationToken); + + if (header == null) + { + TabularWorkspaceSqliteHelpers.CreateEmptyPlaceholderTable(connection, tableName); + + return Task.FromResult(new TabularWorkspaceImportResult( + [new TabularColumnInfo("value", "TEXT")], + 0, + 0, + 1)); + } + + transaction?.Commit(); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "OpenXml workspace importer loaded '{FileName}' into table '{TableName}' with {ColumnCount} column(s) and {RowCount} row(s) in {ElapsedMilliseconds} ms.", + fileName, + tableName, + columns.Count, + rowCount, + stopwatch.ElapsedMilliseconds); + } + + return Task.FromResult(new TabularWorkspaceImportResult(columns, rowCount, insertCommandCount, 1)); + } + catch + { + transaction?.Rollback(); + + throw; + } + finally + { + insertCommand?.Dispose(); + transaction?.Dispose(); + } + } + + private static SqliteCommand CreateInsertCommand( + SqliteConnection connection, + SqliteTransaction transaction, + string tableName, + IReadOnlyList columns) + { + var command = connection.CreateCommand(); + command.Transaction = transaction; + var parameterNames = new string[columns.Count]; + + for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) + { + var parameterName = $"$p{columnIndex}"; + parameterNames[columnIndex] = parameterName; + + var parameter = command.CreateParameter(); + parameter.ParameterName = parameterName; + parameter.Value = DBNull.Value; + command.Parameters.Add(parameter); + } + + var columnList = string.Join(", ", columns.Select(column => TabularWorkspaceSqliteHelpers.QuoteIdentifier(column.Name))); + command.CommandText = $"INSERT INTO {TabularWorkspaceSqliteHelpers.QuoteIdentifier(tableName)} ({columnList}) VALUES ({string.Join(", ", parameterNames)})"; + command.Prepare(); + + return command; + } + + private static void BindInsertParameters( + SqliteCommand command, + List row, + int columnCount) + { + for (var columnIndex = 0; columnIndex < columnCount; columnIndex++) + { + command.Parameters[columnIndex].Value = columnIndex < row.Count + ? (object)(row[columnIndex] ?? string.Empty) + : DBNull.Value; + } + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs index 4c4c0479..98ea30ef 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Endpoints/AIChatDocumentEndpointBase.cs @@ -94,6 +94,24 @@ public abstract class AIChatDocumentEndpointBase try { + if (documentOptions.IsTabularFileExtension(file.FileName)) + { + if (logger.IsEnabled(LogLevel.Information)) + { + logger.LogInformation( + "Deferring tabular extraction for uploaded file '{FileName}' until the document is queried.", + file.FileName); + } + + return await ProcessTabularDocumentAsync( + file, + referenceId, + referenceType, + documentStore, + fileStore, + timeProvider); + } + if (allowVisionImages && MediaTypeHelper.IsVisionImageExtension(extension)) { return await ProcessVisionImageAsync( @@ -335,6 +353,56 @@ protected static async Task InvokeRemovedHandlersAsync(IEnumerable ProcessTabularDocumentAsync( + IFormFile file, + string referenceId, + string referenceType, + IAIDocumentStore documentStore, + IDocumentFileStore fileStore, + TimeProvider timeProvider) + { + var now = timeProvider.GetUtcNow().UtcDateTime; + var contentType = MediaTypeHelper.InferMediaType(Path.GetExtension(file.FileName), file.ContentType); + var document = new AIDocument + { + ItemId = UniqueId.GenerateId(), + ReferenceId = referenceId, + ReferenceType = referenceType, + FileName = file.FileName, + ContentType = contentType, + FileSize = file.Length, + UploadedUtc = now, + }; + + var (storedFileName, storagePath) = DocumentFileStoragePath.Create(referenceType, referenceId, file.FileName); + + using (var stream = file.OpenReadStream()) + { + await fileStore.SaveFileAsync(storagePath, stream); + } + + document.StoredFileName = storedFileName; + document.StoredFilePath = storagePath; + + var documentInfo = new ChatDocumentInfo + { + DocumentId = document.ItemId, + FileName = document.FileName, + FileSize = document.FileSize, + ContentType = document.ContentType, + }; + + await documentStore.CreateAsync(document); + + return (true, null, new AIChatUploadedDocument + { + File = file, + Document = document, + DocumentInfo = documentInfo, + Chunks = [], + }); + } + private static async Task> AnalyzeAndStoreImageChunksAsync( IFormFile file, AIDocument document, diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/TabularWorkspaceDocumentEventHandler.cs b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/TabularWorkspaceDocumentEventHandler.cs index 25a8a0b2..01c925d0 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/TabularWorkspaceDocumentEventHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/TabularWorkspaceDocumentEventHandler.cs @@ -28,12 +28,7 @@ public TabularWorkspaceDocumentEventHandler( public Task UploadedAsync(AIChatDocumentUploadContext context, CancellationToken cancellationToken = default) { - if (context?.UploadedDocuments is null) - { - return Task.CompletedTask; - } - - return UploadedCoreAsync(context, cancellationToken); + return Task.CompletedTask; } public async Task RemovedAsync(AIChatDocumentRemoveContext context, CancellationToken cancellationToken = default) @@ -45,21 +40,6 @@ public async Task RemovedAsync(AIChatDocumentRemoveContext context, Cancellation } } - private async Task UploadedCoreAsync(AIChatDocumentUploadContext context, CancellationToken cancellationToken) - { - foreach (var uploadedDocument in context.UploadedDocuments) - { - if (!IsTabular(uploadedDocument.DocumentInfo)) - { - continue; - } - - var content = string.Concat(uploadedDocument.Chunks.OrderBy(chunk => chunk.Index).Select(chunk => chunk.Content)); - var artifact = TabularDocumentArtifact.FromDelimitedContent(content, uploadedDocument.DocumentInfo.FileName); - await _artifactStore.SaveAsync(uploadedDocument.DocumentInfo.DocumentId, artifact, cancellationToken); - } - } - /// /// Opens the workspace database for the scope, drops the table associated with the removed /// document, removes the metadata row, and deletes the database file if no tables remain. diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs index 5ec041c9..d66be2a5 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Models/ChatDocumentsOptionsExtensions.cs @@ -100,7 +100,7 @@ public static bool IsTabularFileExtension(this ChatDocumentsOptions options, str return false; } - var extension = System.IO.Path.GetExtension(fileNameOrExtension); + var extension = Path.GetExtension(fileNameOrExtension); if (string.IsNullOrEmpty(extension)) { diff --git a/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs b/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs index a95b3e68..4c3c19ad 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/ServiceCollectionExtensions.cs @@ -87,6 +87,7 @@ public static IServiceCollection AddCoreAIDocumentProcessing(this IServiceCollec // Per-prompt tabular workspace options + the system tabular data agent that queries it. services.AddOptions(); services.TryAddSingleton(); + services.TryAddScoped(); services.AddTemplatesFromAssembly(typeof(ServiceCollectionExtensions).Assembly); services.TryAddEnumerable(ServiceDescriptor.Scoped()); @@ -138,6 +139,11 @@ public static IServiceCollection AddCoreAIDocumentProcessing(this IServiceCollec .WithDescription("Reads the full text content of a specific document.") .WithPurpose(AIToolPurposes.DocumentProcessing); + services.AddCoreAITool(GetDocumentMetadataTool.TheName) + .WithTitle("Get Document Metadata") + .WithDescription("Returns metadata for an attached document, including tabular headers, row counts, and normalized column names when applicable.") + .WithPurpose(AIToolPurposes.DocumentProcessing); + services.AddCoreAITool(ListTabularDataTool.TheName) .WithTitle("List Tabular Data") .WithDescription("Lists the tabular tables available in the conversation with their columns and row counts.") diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultConversationDocumentCleanupService.cs b/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultConversationDocumentCleanupService.cs index fd426cfd..164fabdb 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultConversationDocumentCleanupService.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Services/DefaultConversationDocumentCleanupService.cs @@ -1,10 +1,10 @@ +using System.Text.RegularExpressions; using CrestApps.Core.AI.Documents.Generation; using CrestApps.Core.AI.Documents.Tabular; using CrestApps.Core.AI.Models; using CrestApps.Core.Support; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using System.Text.RegularExpressions; namespace CrestApps.Core.AI.Documents.Services; diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularDocumentArtifactBuilder.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularDocumentArtifactBuilder.cs new file mode 100644 index 00000000..aa7e804a --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularDocumentArtifactBuilder.cs @@ -0,0 +1,22 @@ +namespace CrestApps.Core.AI.Documents.Tabular; + +/// +/// Builds a parsed tabular artifact directly from a stored source file for a specific extension. +/// Implementations can use a format-specific fast path to avoid generic document-ingestion overhead. +/// +public interface ITabularDocumentArtifactBuilder +{ + /// + /// Creates a parsed tabular artifact from the supplied file stream. + /// + /// The source file stream. + /// The original file name. + /// The source content type. + /// The cancellation token. + /// The parsed tabular artifact. + Task CreateAsync( + Stream source, + string fileName, + string contentType, + CancellationToken cancellationToken = default); +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularWorkspaceImporter.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularWorkspaceImporter.cs new file mode 100644 index 00000000..28585657 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/ITabularWorkspaceImporter.cs @@ -0,0 +1,29 @@ +using Microsoft.Data.Sqlite; + +namespace CrestApps.Core.AI.Documents.Tabular; + +/// +/// Streams a tabular source file directly into a SQLite workspace for a specific extension. +/// Implementations can avoid materializing a full when a +/// format-specific fast path is available. +/// +public interface ITabularWorkspaceImporter +{ + /// + /// Imports the supplied source file directly into the target SQLite table. + /// + /// The source file stream. + /// The original file name. + /// The source content type. + /// The SQLite connection that owns the workspace. + /// The destination table name. + /// The cancellation token. + /// The import result, or when the importer cannot handle the source. + Task ImportAsync( + Stream source, + string fileName, + string contentType, + SqliteConnection connection, + string tableName, + CancellationToken cancellationToken = default); +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularDocumentArtifactFactory.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularDocumentArtifactFactory.cs new file mode 100644 index 00000000..ddebe7e4 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularDocumentArtifactFactory.cs @@ -0,0 +1,186 @@ +using System.Diagnostics; +using CrestApps.Core.AI.Models; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DataIngestion; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Documents.Tabular; + +internal sealed class TabularDocumentArtifactFactory +{ + private readonly IServiceProvider _serviceProvider; + private readonly IDocumentFileStore _fileStore; + private readonly ILogger _logger; + + public TabularDocumentArtifactFactory( + IServiceProvider serviceProvider, + IDocumentFileStore fileStore, + ILogger logger) + { + _serviceProvider = serviceProvider; + _fileStore = fileStore; + _logger = logger; + } + + public async Task CreateAsync(AIDocument document, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + + if (string.IsNullOrWhiteSpace(document.StoredFilePath)) + { + return null; + } + + var extension = Path.GetExtension(document.FileName); + + await using var stream = await _fileStore.GetFileAsync(document.StoredFilePath); + + if (stream is null) + { + return null; + } + + var stopwatch = Stopwatch.StartNew(); + var builder = _serviceProvider.GetKeyedService(extension); + TabularDocumentArtifact artifact = null; + + if (builder != null) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Using keyed tabular artifact builder '{BuilderType}' for '{FileName}'.", + builder.GetType().FullName, + document.FileName); + } + + artifact = await builder.CreateAsync(stream, document.FileName, document.ContentType, cancellationToken); + } + else + { + var reader = _serviceProvider.GetKeyedService(extension); + + if (reader == null) + { + return null; + } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Falling back to ingestion reader '{ReaderType}' for tabular artifact '{FileName}'.", + reader.GetType().FullName, + document.FileName); + } + + var mediaType = MediaTypeHelper.InferMediaType(extension, document.ContentType); + var ingestionDoc = await reader.ReadAsync(stream, document.FileName, mediaType, cancellationToken); + artifact = string.Equals(extension, ".xlsx", StringComparison.OrdinalIgnoreCase) + ? CreateSpreadsheetArtifact(ingestionDoc) + : CreateDelimitedArtifact(ingestionDoc, document.FileName); + } + + if (_logger.IsEnabled(LogLevel.Debug) && artifact != null) + { + _logger.LogDebug( + "Built tabular artifact for '{FileName}' with {ColumnCount} column(s) and {RowCount} row(s) in {ElapsedMilliseconds} ms.", + document.FileName, + artifact.Header.Count, + artifact.Rows.Count, + stopwatch.ElapsedMilliseconds); + } + + return artifact; + } + + public async Task ImportToWorkspaceAsync( + AIDocument document, + SqliteConnection connection, + string tableName, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(connection); + ArgumentException.ThrowIfNullOrEmpty(tableName); + + if (string.IsNullOrWhiteSpace(document.StoredFilePath)) + { + return null; + } + + var extension = Path.GetExtension(document.FileName); + var importer = _serviceProvider.GetKeyedService(extension); + + if (importer == null) + { + return null; + } + + await using var stream = await _fileStore.GetFileAsync(document.StoredFilePath); + + if (stream is null) + { + return null; + } + + var stopwatch = Stopwatch.StartNew(); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Using keyed tabular workspace importer '{ImporterType}' for '{FileName}'.", + importer.GetType().FullName, + document.FileName); + } + + var result = await importer.ImportAsync( + stream, + document.FileName, + document.ContentType, + connection, + tableName, + cancellationToken); + + if (_logger.IsEnabled(LogLevel.Debug) && result != null) + { + _logger.LogDebug( + "Imported tabular workspace data for '{FileName}' with {ColumnCount} column(s) and {RowCount} row(s) in {ElapsedMilliseconds} ms.", + document.FileName, + result.Columns.Count, + result.RowCount, + stopwatch.ElapsedMilliseconds); + } + + return result; + } + + private static TabularDocumentArtifact CreateDelimitedArtifact(IngestionDocument ingestionDoc, string fileName) + { + var content = string.Join('\n', ingestionDoc.EnumerateContent() + .Select(element => element.Text) + .Where(text => !string.IsNullOrWhiteSpace(text))); + + return TabularDocumentArtifact.FromDelimitedContent(content, fileName); + } + + private static TabularDocumentArtifact CreateSpreadsheetArtifact(IngestionDocument ingestionDoc) + { + var rows = ingestionDoc.EnumerateContent() + .Select(element => element.Text) + .Where(text => !string.IsNullOrWhiteSpace(text)) + .Select(text => text.Split('\t').ToList()) + .ToList(); + + if (rows.Count == 0) + { + return new TabularDocumentArtifact(); + } + + return new TabularDocumentArtifact + { + Header = rows[0], + Rows = rows.Count > 1 ? rows.Skip(1).ToList() : [], + }; + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolContext.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolContext.cs index fc671ff8..2ebb629c 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolContext.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolContext.cs @@ -3,6 +3,7 @@ using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Orchestration; using Cysharp.Text; +using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -15,20 +16,26 @@ namespace CrestApps.Core.AI.Documents.Tabular; /// internal sealed class TabularToolContext { + private readonly IAIDocumentStore _documentStore; private readonly IAIDocumentChunkStore _chunkStore; private readonly ITabularDocumentArtifactStore _artifactStore; + private readonly TabularDocumentArtifactFactory _artifactFactory; private TabularToolContext( IReadOnlyList documents, + IAIDocumentStore documentStore, IAIDocumentChunkStore chunkStore, ITabularDocumentArtifactStore artifactStore, + TabularDocumentArtifactFactory artifactFactory, string databasePath, string exportReferenceId, string exportReferenceType) { Documents = documents; + _documentStore = documentStore; _chunkStore = chunkStore; _artifactStore = artifactStore; + _artifactFactory = artifactFactory; DatabasePath = databasePath; ExportReferenceId = exportReferenceId; ExportReferenceType = exportReferenceType; @@ -96,13 +103,48 @@ public async Task LoadArtifactAsync(TabularDocumentRef return artifact; } + var storedDocument = await _documentStore.FindByIdAsync(document.DocumentId, cancellationToken); + artifact = storedDocument == null ? null : await _artifactFactory.CreateAsync(storedDocument, cancellationToken); + + if (artifact != null) + { + await _artifactStore.SaveAsync(document.DocumentId, artifact, cancellationToken); + + return artifact; + } + var content = await LoadContentAsync(document.DocumentId, cancellationToken); artifact = TabularDocumentArtifact.FromDelimitedContent(content, document.FileName); + await _artifactStore.SaveAsync(document.DocumentId, artifact, cancellationToken); return artifact; } + public async Task ImportToWorkspaceAsync( + TabularDocumentRef document, + SqliteConnection connection, + string tableName, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(connection); + ArgumentException.ThrowIfNullOrEmpty(tableName); + + var storedDocument = await _documentStore.FindByIdAsync(document.DocumentId, cancellationToken); + + if (storedDocument == null) + { + return null; + } + + return await _artifactFactory.ImportToWorkspaceAsync( + storedDocument, + connection, + tableName, + cancellationToken); + } + /// /// Resolves the tabular tool context from the current . /// @@ -114,16 +156,12 @@ public static async Task ResolveAsync(IServiceProvider servi var invocationContext = AIInvocationScope.Current; var executionContext = invocationContext?.ToolExecutionContext; - if (executionContext is null) - { - return null; - } - var documentStore = services.GetService(); var chunkStore = services.GetService(); var artifactStore = services.GetService(); + var artifactFactory = services.GetService(); - if (documentStore is null || chunkStore is null || artifactStore is null) + if (documentStore is null || chunkStore is null || artifactStore is null || artifactFactory is null) { return null; } @@ -134,6 +172,11 @@ public static async Task ResolveAsync(IServiceProvider servi string exportReferenceId = null; string exportReferenceType = null; + if (executionContext is null) + { + return null; + } + switch (executionContext.Resource) { case ChatInteraction interaction: @@ -190,8 +233,10 @@ public static async Task ResolveAsync(IServiceProvider servi return new TabularToolContext( documents, + documentStore, chunkStore, artifactStore, + artifactFactory, databasePath, exportReferenceId, exportReferenceType); diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolRunner.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolRunner.cs index 2d5a4e81..7d559e55 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolRunner.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularToolRunner.cs @@ -1,4 +1,6 @@ +using System.Diagnostics; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace CrestApps.Core.AI.Documents.Tabular; @@ -32,6 +34,7 @@ public readonly record struct PreparationResult( /// The preparation result. public static async Task PrepareAsync(IServiceProvider services, CancellationToken cancellationToken = default) { + var logger = services.GetService()?.CreateLogger(typeof(TabularToolRunner).FullName); var context = await TabularToolContext.ResolveAsync(services, cancellationToken); if (context is null) @@ -45,8 +48,23 @@ public static async Task PrepareAsync(IServiceProvider servic } var options = services.GetRequiredService>().Value; - var workspace = new TabularWorkspace(options, context.DatabasePath); - var tables = await workspace.EnsureReadyAsync(context.Documents, context.LoadArtifactAsync, cancellationToken); + var workspaceLogger = services.GetRequiredService>(); + var workspace = new TabularWorkspace(options, context.DatabasePath, workspaceLogger); + var stopwatch = Stopwatch.StartNew(); + var tables = await workspace.EnsureReadyAsync( + context.Documents, + context.LoadArtifactAsync, + context.ImportToWorkspaceAsync, + cancellationToken); + + if (logger?.IsEnabled(LogLevel.Debug) == true) + { + logger.LogDebug( + "Prepared tabular workspace for {DocumentCount} document(s) with {TableCount} table(s) in {ElapsedMilliseconds} ms.", + context.Documents.Count, + tables.Count, + stopwatch.ElapsedMilliseconds); + } return new PreparationResult(workspace, tables, context, null); } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspace.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspace.cs index 071b02a6..582ad6f7 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspace.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspace.cs @@ -1,7 +1,10 @@ +using System.Diagnostics; using System.Globalization; using System.Text; using System.Text.Json; using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace CrestApps.Core.AI.Documents.Tabular; @@ -16,11 +19,13 @@ namespace CrestApps.Core.AI.Documents.Tabular; internal sealed class TabularWorkspace : IDisposable { private const string MetadataTableName = "_workspace_meta"; + private const int ImportProgressIntervalRows = 250; private static readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.Web); private readonly TabularWorkspaceOptions _options; private readonly string _databasePath; + private readonly ILogger _logger; private readonly SemaphoreSlim _gate = new(1, 1); private readonly Dictionary _tables = new(StringComparer.Ordinal); private int _mutationVersion; @@ -37,10 +42,12 @@ internal sealed class TabularWorkspace : IDisposable /// public TabularWorkspace( TabularWorkspaceOptions options, - string databasePath = null) + string databasePath = null, + ILogger logger = null) { _options = options; _databasePath = databasePath; + _logger = logger ?? NullLogger.Instance; } /// @@ -65,6 +72,7 @@ public async Task> EnsureReadyAsync( async (document, token) => TabularDocumentArtifact.FromDelimitedContent( await contentLoader(document.DocumentId, token), document.FileName), + workspaceImporter: null, cancellationToken); } @@ -81,6 +89,7 @@ await contentLoader(document.DocumentId, token), public async Task> EnsureReadyAsync( IReadOnlyList documents, Func> artifactLoader, + Func> workspaceImporter, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(documents); @@ -90,6 +99,7 @@ public async Task> EnsureReadyAsync( try { + var stopwatch = Stopwatch.StartNew(); _connection ??= OpenConnection(); if (_tables.Count == 0) @@ -99,7 +109,16 @@ public async Task> EnsureReadyAsync( var desiredTableNames = ComputeTableNames(documents); - await SynchronizeTablesAsync(documents, desiredTableNames, artifactLoader, cancellationToken); + await SynchronizeTablesAsync(documents, desiredTableNames, artifactLoader, workspaceImporter, cancellationToken); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Tabular workspace ready with {LoadedTableCount} loaded table(s) for {RequestedDocumentCount} requested document(s) in {ElapsedMilliseconds} ms.", + _tables.Count, + documents.Count, + stopwatch.ElapsedMilliseconds); + } return BuildTableInfos(); } @@ -506,23 +525,72 @@ private async Task SynchronizeTablesAsync( IReadOnlyList documents, Dictionary desiredTableNames, Func> artifactLoader, + Func> workspaceImporter, CancellationToken cancellationToken) { foreach (var document in documents) { if (_tables.ContainsKey(document.DocumentId)) { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Skipping tabular document '{DocumentId}' because table '{TableName}' is already loaded.", + document.DocumentId, + _tables[document.DocumentId].TableName); + } + continue; } var tableName = desiredTableNames[document.DocumentId]; - var artifact = await artifactLoader(document, cancellationToken); + var importStopwatch = Stopwatch.StartNew(); + var importResult = workspaceImporter == null + ? null + : await workspaceImporter(document, _connection, tableName, cancellationToken); + TabularDocumentArtifact artifact = null; + IReadOnlyList columns; + int insertCommandCount; + int rowsPerBatch; + long loadedRowCount; + var loadStopwatch = Stopwatch.StartNew(); + + if (importResult != null) + { + loadStopwatch.Stop(); + columns = importResult.Columns; + insertCommandCount = importResult.InsertCommandCount; + rowsPerBatch = importResult.RowsPerBatch; + loadedRowCount = importResult.RowCount; + } + else + { + artifact = await artifactLoader(document, cancellationToken); + loadStopwatch.Stop(); + columns = CreateTable(_connection, tableName, artifact, out insertCommandCount, out rowsPerBatch, cancellationToken); + loadedRowCount = artifact?.Rows?.Count ?? 0; + } + + importStopwatch.Stop(); - var columns = CreateTable(_connection, tableName, artifact); var sourceNames = columns.ToDictionary(c => c.Name, c => c.SourceName, StringComparer.OrdinalIgnoreCase); _tables[document.DocumentId] = new LoadedTable(tableName, document.FileName, sourceNames); SaveMetadataEntry(document.DocumentId, tableName, document.FileName, sourceNames); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Loaded tabular document '{FileName}' into table '{TableName}' with {ColumnCount} column(s), {RowCount} row(s), batch size {RowsPerBatch}, and {InsertCommandCount} insert command execution(s) in {ArtifactLoadMilliseconds} ms load + {ImportMilliseconds} ms import.", + document.FileName, + tableName, + columns.Count, + loadedRowCount, + rowsPerBatch, + insertCommandCount, + loadStopwatch.ElapsedMilliseconds, + importStopwatch.ElapsedMilliseconds); + } } } @@ -549,6 +617,13 @@ private SqliteConnection OpenConnection() var connection = new SqliteConnection(connectionString); connection.Open(); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Opened tabular workspace database at '{DatabasePath}'.", + string.IsNullOrEmpty(_databasePath) ? ":memory:" : _databasePath); + } + using var walCommand = connection.CreateCommand(); walCommand.CommandText = "PRAGMA journal_mode=WAL"; walCommand.ExecuteNonQuery(); @@ -591,6 +666,11 @@ private void LoadMetadataFromDatabase() _tables[documentId] = new LoadedTable(tableName, fileName, sourceNames); } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("Loaded {TableCount} tabular workspace metadata entr{Suffix} from the database.", _tables.Count, _tables.Count == 1 ? "y" : "ies"); + } } private void SaveMetadataEntry(string documentId, string tableName, string fileName, IReadOnlyDictionary sourceNames) @@ -607,11 +687,19 @@ private void SaveMetadataEntry(string documentId, string tableName, string fileN command.ExecuteNonQuery(); } - private static List CreateTable(SqliteConnection connection, string tableName, TabularDocumentArtifact artifact) + private List CreateTable( + SqliteConnection connection, + string tableName, + TabularDocumentArtifact artifact, + out int insertCommandCount, + out int rowsPerBatch, + CancellationToken cancellationToken) { artifact ??= new TabularDocumentArtifact(); var header = artifact.Header ?? []; var rows = artifact.Rows ?? []; + insertCommandCount = 0; + rowsPerBatch = 0; if (header.Count == 0) { @@ -638,41 +726,105 @@ private static List CreateTable(SqliteConnection connection, return columns; } - InsertRows(connection, tableName, columnNames, rows); + insertCommandCount = InsertRows(connection, tableName, columnNames, rows, out rowsPerBatch, cancellationToken); return columns; } - private static void InsertRows(SqliteConnection connection, string tableName, List columns, IReadOnlyList> rows) + private int InsertRows( + SqliteConnection connection, + string tableName, + List columns, + List> rows, + out int rowsPerBatch, + CancellationToken cancellationToken) { + rowsPerBatch = 1; + var stopwatch = Stopwatch.StartNew(); + var columnList = string.Join(", ", columns.Select(QuoteIdentifier)); using var transaction = connection.BeginTransaction(); using var command = connection.CreateCommand(); command.Transaction = transaction; - var columnList = string.Join(", ", columns.Select(QuoteIdentifier)); - var parameterList = string.Join(", ", columns.Select((_, i) => $"$p{i}")); - command.CommandText = $"INSERT INTO {QuoteIdentifier(tableName)} ({columnList}) VALUES ({parameterList})"; + var parameterNames = new string[columns.Count]; - var parameters = new SqliteParameter[columns.Count]; + for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) + { + var parameterName = $"$p{columnIndex}"; + parameterNames[columnIndex] = parameterName; + + var parameter = command.CreateParameter(); + parameter.ParameterName = parameterName; + parameter.Value = DBNull.Value; + command.Parameters.Add(parameter); + } - for (var i = 0; i < columns.Count; i++) + command.CommandText = $"INSERT INTO {QuoteIdentifier(tableName)} ({columnList}) VALUES ({string.Join(", ", parameterNames)})"; + command.Prepare(); + + if (_logger.IsEnabled(LogLevel.Debug)) { - parameters[i] = command.CreateParameter(); - parameters[i].ParameterName = $"$p{i}"; - command.Parameters.Add(parameters[i]); + _logger.LogDebug( + "Starting SQLite import for table '{TableName}' with {ColumnCount} column(s), {RowCount} row(s), and prepared batch size {RowsPerBatch}.", + tableName, + columns.Count, + rows.Count, + rowsPerBatch); } - foreach (var row in rows) + var insertCommandCount = 0; + + try { - for (var i = 0; i < columns.Count; i++) + for (var rowIndex = 0; rowIndex < rows.Count; rowIndex++) { - parameters[i].Value = i < row.Count ? (object)(row[i] ?? string.Empty) : DBNull.Value; + cancellationToken.ThrowIfCancellationRequested(); + + var row = rows[rowIndex]; + + for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) + { + command.Parameters[columnIndex].Value = columnIndex < row.Count + ? (object)(row[columnIndex] ?? string.Empty) + : DBNull.Value; + } + + command.ExecuteNonQuery(); + insertCommandCount++; + + if (_logger.IsEnabled(LogLevel.Debug) && + ((rowIndex + 1) % ImportProgressIntervalRows == 0 || rowIndex == rows.Count - 1)) + { + _logger.LogDebug( + "SQLite import for table '{TableName}' processed {ProcessedRowCount}/{TotalRowCount} row(s) in {ElapsedMilliseconds} ms.", + tableName, + rowIndex + 1, + rows.Count, + stopwatch.ElapsedMilliseconds); + } } - command.ExecuteNonQuery(); + transaction.Commit(); + } + catch + { + transaction.Rollback(); + + throw; } - transaction.Commit(); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Completed SQLite import for table '{TableName}' with {RowCount} row(s), batch size {RowsPerBatch}, and {InsertCommandCount} insert command execution(s) in {ElapsedMilliseconds} ms.", + tableName, + rows.Count, + rowsPerBatch, + insertCommandCount, + stopwatch.ElapsedMilliseconds); + } + + return insertCommandCount; } private List BuildTableInfos() @@ -914,4 +1066,5 @@ public LoadedTable( public IReadOnlyDictionary SourceNames { get; } } + } diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceImportResult.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceImportResult.cs new file mode 100644 index 00000000..7836611b --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceImportResult.cs @@ -0,0 +1,46 @@ +namespace CrestApps.Core.AI.Documents.Tabular; + +/// +/// Describes the outcome of importing a tabular source directly into a SQLite workspace. +/// +public sealed class TabularWorkspaceImportResult +{ + /// + /// Initializes a new instance of the class. + /// + /// The imported table columns. + /// The imported row count. + /// The number of executed insert commands. + /// The effective rows-per-batch value used during import. + public TabularWorkspaceImportResult( + IReadOnlyList columns, + int rowCount, + int insertCommandCount, + int rowsPerBatch) + { + Columns = columns; + RowCount = rowCount; + InsertCommandCount = insertCommandCount; + RowsPerBatch = rowsPerBatch; + } + + /// + /// Gets the imported table columns. + /// + public IReadOnlyList Columns { get; } + + /// + /// Gets the imported row count. + /// + public int RowCount { get; } + + /// + /// Gets the number of executed insert commands. + /// + public int InsertCommandCount { get; } + + /// + /// Gets the effective rows-per-batch value used during import. + /// + public int RowsPerBatch { get; } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceSqliteHelpers.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceSqliteHelpers.cs new file mode 100644 index 00000000..1e8ed31f --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tabular/TabularWorkspaceSqliteHelpers.cs @@ -0,0 +1,151 @@ +using System.Text; +using Microsoft.Data.Sqlite; + +namespace CrestApps.Core.AI.Documents.Tabular; + +/// +/// Shared SQLite table-shaping helpers used by tabular workspace importers. +/// +public static class TabularWorkspaceSqliteHelpers +{ + /// + /// Builds the SQLite column definitions for a tabular header row. + /// + /// The source header row. + /// The normalized workspace columns. + public static IReadOnlyList BuildColumns(IReadOnlyList header) + { + ArgumentNullException.ThrowIfNull(header); + + var columns = new List(header.Count); + var used = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var i = 0; i < header.Count; i++) + { + var sourceName = header[i]; + var name = SanitizeIdentifier(GetPreferredHeaderName(sourceName), $"column_{i + 1}"); + var candidate = name; + var suffix = 2; + + while (!used.Add(candidate)) + { + candidate = $"{name}_{suffix}"; + suffix++; + } + + columns.Add(new TabularColumnInfo(candidate, "TEXT", sourceName)); + } + + return columns; + } + + /// + /// Creates a SQLite table using the supplied normalized columns. + /// + /// The SQLite connection. + /// The destination table name. + /// The normalized columns. + public static void CreateTable( + SqliteConnection connection, + string tableName, + IReadOnlyList columns) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentException.ThrowIfNullOrEmpty(tableName); + ArgumentNullException.ThrowIfNull(columns); + + using var createCommand = connection.CreateCommand(); + var columnDefinitions = string.Join(", ", columns.Select(c => $"{QuoteIdentifier(c.Name)} TEXT")); + createCommand.CommandText = $"CREATE TABLE {QuoteIdentifier(tableName)} ({columnDefinitions})"; + createCommand.ExecuteNonQuery(); + } + + /// + /// Creates an empty placeholder table for a document with no header row. + /// + /// The SQLite connection. + /// The destination table name. + public static void CreateEmptyPlaceholderTable(SqliteConnection connection, string tableName) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentException.ThrowIfNullOrEmpty(tableName); + + using var command = connection.CreateCommand(); + command.CommandText = $"CREATE TABLE {QuoteIdentifier(tableName)} (\"value\" TEXT)"; + command.ExecuteNonQuery(); + } + + /// + /// Quotes a SQLite identifier safely. + /// + /// The unquoted identifier. + /// The quoted identifier. + public static string QuoteIdentifier(string identifier) + { + ArgumentException.ThrowIfNullOrEmpty(identifier); + + return "\"" + identifier.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""; + } + + private static string GetPreferredHeaderName(string header) + { + if (string.IsNullOrWhiteSpace(header)) + { + return header; + } + + var trimmed = header.Trim(); + var slashIndex = trimmed.IndexOf('/'); + + if (slashIndex > 0) + { + var prefix = trimmed[..slashIndex].Trim(); + + if (IsCompactHeaderCode(prefix)) + { + return prefix; + } + } + + return trimmed; + } + + private static bool IsCompactHeaderCode(string value) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 64) + { + return false; + } + + return value.All(c => char.IsLetterOrDigit(c) || c == '_'); + } + + private static string SanitizeIdentifier(string value, string fallback) + { + if (string.IsNullOrWhiteSpace(value)) + { + return fallback; + } + + var builder = new StringBuilder(value.Length); + + foreach (var c in value.Trim()) + { + builder.Append(char.IsLetterOrDigit(c) || c == '_' ? c : '_'); + } + + var sanitized = builder.ToString().Trim('_'); + + if (string.IsNullOrEmpty(sanitized)) + { + return fallback; + } + + if (char.IsDigit(sanitized[0])) + { + sanitized = "_" + sanitized; + } + + return sanitized; + } +} diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Templates/Prompts/tabular-data-agent.md b/src/Primitives/CrestApps.Core.AI.Documents/Templates/Prompts/tabular-data-agent.md index a08ea6e1..155994a6 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Templates/Prompts/tabular-data-agent.md +++ b/src/Primitives/CrestApps.Core.AI.Documents/Templates/Prompts/tabular-data-agent.md @@ -10,12 +10,17 @@ You are the Tabular Data Agent. You answer questions and perform tasks over tabu in-memory SQLite database so you can work with very large files efficiently. How to work: -1. Call list_tabular_data first to discover the available tables, source files, row counts, SQL - column names, and original source headers. -2. Use query_tabular_data to run read-only SQL (SQLite dialect) that directly answers the request. +1. For file structure, row counts, original headers, normalized column names, or inferred column + data types, call + get_document_metadata first. Use `scope: "tabular_summary"` for row/column counts, + `scope: "headers"` for original uploaded headers with inferred data types, and + `scope: "columns"` for normalized SQL column names with inferred data types. +2. Call list_tabular_data when you need the current in-memory table names or a full table listing + before composing SQL. +3. Use query_tabular_data to run read-only SQL (SQLite dialect) that directly answers the request. Prefer aggregation, filtering, GROUP BY, and small LIMITs. Never try to read every row into your answer — push the computation into SQL and return only the result the user needs. -3. Use execute_tabular_command only when the user asks to modify the data (for example adding or +4. Use execute_tabular_command only when the user asks to modify the data (for example adding or removing a column, updating values, or inserting rows). These changes apply to the in-memory copy and persist for the rest of the conversation so they can be exported later; the originally uploaded file itself is never modified. Always apply every requested change with execute_tabular_command @@ -26,7 +31,7 @@ How to work: large files. When a request needs several different changes, put all of them in ONE execute_tabular_command call by separating the statements with semicolons (they run together in a single transaction). Do not make many separate execute_tabular_command calls. -4. Use export_tabular_data when the user asks for a downloadable/new version of a tabular file (for +5. Use export_tabular_data when the user asks for a downloadable/new version of a tabular file (for example a sorted file, filtered file, or file with generated columns). To give the user the file with their updated data, call export_tabular_data WITHOUT a sql argument: this exports the entire current in-memory table (all rows and all columns, including every change you applied). The export @@ -49,8 +54,9 @@ How to work: Guidelines: - All columns are stored as TEXT. CAST values when you need numeric or date comparisons or math. - Quote identifiers with double quotes when they contain spaces or special characters. -- If the user asks for a general summary, record count, file structure, data type, or column list, - answer from list_tabular_data and run aggregate queries when counts or examples are needed. +- If the user asks for a general summary, record count, file structure, original headers, column + list, or schema/data types, prefer get_document_metadata first and run aggregate queries only when + counts or examples are needed beyond the returned metadata. - If a source header includes a survey/question code such as `Q3_C28/...`, use the SQL column name reported by list_tabular_data (for example `Q3_C28`) and mention the original source header when helpful. - If a query fails, read the error, correct the SQL, and try again. diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tools/GetDocumentMetadataTool.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tools/GetDocumentMetadataTool.cs new file mode 100644 index 00000000..080f3112 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tools/GetDocumentMetadataTool.cs @@ -0,0 +1,538 @@ +using System.Globalization; +using System.Text.Json; +using CrestApps.Core.AI.Documents.Models; +using CrestApps.Core.AI.Documents.Tabular; +using CrestApps.Core.AI.Extensions; +using CrestApps.Core.AI.Models; +using CrestApps.Core.AI.Orchestration; +using CrestApps.Core.AI.Tooling; +using Cysharp.Text; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace CrestApps.Core.AI.Documents.Tools; + +/// +/// System tool that returns metadata for an attached document. For tabular files, it can surface +/// row counts, original headers, inferred column data types, and normalized SQL column names +/// without requiring the model to request the full workspace/table listing first. +/// +public sealed class GetDocumentMetadataTool : AIFunction +{ + public const string TheName = SystemToolNames.GetDocumentMetadata; + + private static readonly JsonElement _jsonSchema = JsonSerializer.Deserialize( + """ + { + "type": "object", + "properties": { + "document_id": { + "type": "string", + "description": "Optional unique identifier of the document to inspect. Omit this when only one attached document is relevant." + }, + "scope": { + "type": "string", + "description": "The metadata scope to return: 'basic' for general file metadata, 'tabular_summary' for row/column counts on tabular files, 'headers' for original tabular headers with inferred data types, or 'columns' for normalized SQL column names with inferred data types.", + "enum": ["basic", "tabular_summary", "headers", "columns"] + } + }, + "additionalProperties": false + } + """); + + /// + /// Gets the name. + /// + public override string Name => TheName; + + /// + /// Gets the description. + /// + public override string Description => "Returns metadata for an attached document. For tabular files it can provide row counts, original headers, normalized SQL column names, and inferred column data types."; + + /// + /// Gets the JSON schema. + /// + public override JsonElement JsonSchema => _jsonSchema; + + /// + /// Gets the additional properties. + /// + public override IReadOnlyDictionary AdditionalProperties { get; } = + new Dictionary() + { + ["Strict"] = false, + }; + + /// + /// Invokes the tool. + /// + /// The arguments. + /// The cancellation token. + protected override async ValueTask InvokeCoreAsync( + AIFunctionArguments arguments, + CancellationToken cancellationToken) + { + var logger = arguments.Services.GetRequiredService>(); + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("AI tool '{ToolName}' invoked.", Name); + } + + arguments.TryGetFirstString("document_id", out var documentId); + arguments.TryGetFirstString("scope", out var scopeValue); + + if (!TryParseScope(scopeValue, out var scope)) + { + return $"Unknown metadata scope '{scopeValue}'. Use one of: basic, tabular_summary, headers, columns."; + } + + var documents = await ResolveAccessibleDocumentsAsync(arguments.Services); + + if (documents.Count == 0) + { + return "No documents are attached to the current conversation or profile."; + } + + var document = ResolveTargetDocument(documents, documentId); + + if (document is null) + { + return string.IsNullOrWhiteSpace(documentId) + ? "Multiple documents are attached. Provide 'document_id' to specify which document metadata to inspect." + : $"Document with ID '{documentId}' was not found in the current conversation or profile."; + } + + var documentOptions = arguments.Services.GetRequiredService>().Value; + var isTabular = documentOptions.IsTabularFileExtension(document.FileName); + + if (!isTabular || scope == MetadataScope.Basic) + { + var basicMetadata = isTabular + ? await FormatTabularBasicMetadataAsync(arguments.Services, document, cancellationToken) + : FormatBasicMetadata(document); + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("AI tool '{ToolName}' completed.", Name); + } + + return basicMetadata; + } + + var tabularContext = await TabularToolContext.ResolveAsync(arguments.Services, cancellationToken); + + if (tabularContext is null) + { + return "Tabular metadata requires an active chat interaction or AI profile context."; + } + + var tabularDocument = tabularContext.Documents.FirstOrDefault(entry => string.Equals(entry.DocumentId, document.ItemId, StringComparison.OrdinalIgnoreCase)); + + if (tabularDocument is null) + { + return $"Document '{document.FileName}' is not available in the active tabular scope."; + } + + var artifact = await tabularContext.LoadArtifactAsync(tabularDocument, cancellationToken); + var result = scope switch + { + MetadataScope.TabularSummary => FormatTabularSummary(document, artifact), + MetadataScope.Headers => FormatTabularHeaders(document, artifact), + MetadataScope.Columns => FormatTabularColumns(document, artifact), + _ => FormatBasicMetadata(document), + }; + + if (logger.IsEnabled(LogLevel.Debug)) + { + logger.LogDebug("AI tool '{ToolName}' completed.", Name); + } + + return result; + } + + private static async Task> ResolveAccessibleDocumentsAsync(IServiceProvider services) + { + var executionContext = AIInvocationScope.Current?.ToolExecutionContext; + var documentStore = services.GetService(); + + if (executionContext is null || documentStore is null) + { + return []; + } + + var documents = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + switch (executionContext.Resource) + { + case ChatInteraction interaction: + await AddDocumentsAsync(documentStore, interaction.ItemId, AIReferenceTypes.Document.ChatInteraction, documents, seen); + break; + + case AIProfile profile: + await AddDocumentsAsync(documentStore, profile.ItemId, AIReferenceTypes.Document.Profile, documents, seen); + + if (AIInvocationScope.Current?.Items.TryGetValue(nameof(AIChatSession), out var sessionObj) == true && + sessionObj is AIChatSession session && + !string.IsNullOrWhiteSpace(session.SessionId)) + { + await AddDocumentsAsync(documentStore, session.SessionId, AIReferenceTypes.Document.ChatSession, documents, seen); + } + + break; + } + + return documents; + } + + private static async Task AddDocumentsAsync( + IAIDocumentStore documentStore, + string referenceId, + string referenceType, + List destination, + HashSet seen) + { + if (string.IsNullOrWhiteSpace(referenceId)) + { + return; + } + + var found = await documentStore.GetDocumentsAsync(referenceId, referenceType); + + foreach (var document in found) + { + if (seen.Add(document.ItemId)) + { + destination.Add(document); + } + } + } + + private static AIDocument ResolveTargetDocument(IReadOnlyList documents, string documentId) + { + if (!string.IsNullOrWhiteSpace(documentId)) + { + return documents.FirstOrDefault(document => string.Equals(document.ItemId, documentId.Trim(), StringComparison.OrdinalIgnoreCase)); + } + + return documents.Count == 1 ? documents[0] : null; + } + + private static string FormatBasicMetadata(AIDocument document) + { + using var builder = ZString.CreateStringBuilder(); + builder.Append('"'); + builder.Append(document.FileName); + builder.AppendLine("\" metadata:"); + builder.Append("- document_id: "); + builder.AppendLine(document.ItemId); + builder.Append("- content_type: "); + builder.AppendLine(string.IsNullOrWhiteSpace(document.ContentType) ? "(unknown)" : document.ContentType); + builder.Append("- file_size_bytes: "); + builder.AppendLine(document.FileSize.ToString()); + + return builder.ToString(); + } + + private static async Task FormatTabularBasicMetadataAsync( + IServiceProvider services, + AIDocument document, + CancellationToken cancellationToken) + { + var tabularContext = await TabularToolContext.ResolveAsync(services, cancellationToken); + + if (tabularContext is null) + { + return FormatBasicMetadata(document); + } + + var tabularDocument = tabularContext.Documents.FirstOrDefault(entry => string.Equals(entry.DocumentId, document.ItemId, StringComparison.OrdinalIgnoreCase)); + + if (tabularDocument is null) + { + return FormatBasicMetadata(document); + } + + var artifact = await tabularContext.LoadArtifactAsync(tabularDocument, cancellationToken); + + var inferredTypes = InferColumnTypes(artifact); + + using var builder = ZString.CreateStringBuilder(); + builder.Append(FormatBasicMetadata(document)); + builder.Append("- tabular_rows: "); + builder.AppendLine((artifact?.Rows?.Count ?? 0).ToString()); + builder.Append("- tabular_columns: "); + builder.AppendLine((artifact?.Header?.Count ?? 0).ToString()); + builder.Append("- inferred_column_types: "); + builder.AppendLine(string.Join(", ", inferredTypes.Distinct(StringComparer.Ordinal))); + builder.AppendLine("- available_scopes: tabular_summary, headers, columns"); + + return builder.ToString(); + } + + private static string FormatTabularSummary(AIDocument document, TabularDocumentArtifact artifact) + { + var inferredTypes = InferColumnTypes(artifact); + + using var builder = ZString.CreateStringBuilder(); + builder.Append('"'); + builder.Append(document.FileName); + builder.Append("\" is a tabular document with "); + builder.Append(artifact?.Rows?.Count ?? 0); + builder.Append(" data row(s) and "); + builder.Append(artifact?.Header?.Count ?? 0); + builder.AppendLine(" column(s)."); + builder.Append("- inferred_column_types: "); + builder.AppendLine(string.Join(", ", inferredTypes.Distinct(StringComparer.Ordinal))); + + return builder.ToString(); + } + + private static string FormatTabularHeaders(AIDocument document, TabularDocumentArtifact artifact) + { + var headers = artifact?.Header ?? []; + var inferredTypes = InferColumnTypes(artifact); + using var builder = ZString.CreateStringBuilder(); + builder.Append('"'); + builder.Append(document.FileName); + builder.Append("\" has "); + builder.Append(headers.Count); + builder.AppendLine(headers.Count == 1 ? " header." : " headers."); + builder.AppendLine(); + + for (var i = 0; i < headers.Count; i++) + { + var header = headers[i]; + builder.Append("- "); + builder.Append(string.IsNullOrWhiteSpace(header) ? "(blank header)" : header); + builder.Append(" (inferred type: "); + builder.Append(inferredTypes[i]); + builder.AppendLine(")"); + } + + return builder.ToString(); + } + + private static string FormatTabularColumns(AIDocument document, TabularDocumentArtifact artifact) + { + var columns = TabularWorkspaceSqliteHelpers.BuildColumns(artifact?.Header ?? []); + var inferredTypes = InferColumnTypes(artifact); + using var builder = ZString.CreateStringBuilder(); + builder.Append('"'); + builder.Append(document.FileName); + builder.Append("\" exposes "); + builder.Append(columns.Count); + builder.AppendLine(columns.Count == 1 ? " SQL column." : " SQL columns."); + builder.AppendLine(); + + for (var i = 0; i < columns.Count; i++) + { + var column = columns[i]; + builder.Append("- "); + builder.Append(column.Name); + + if (!string.IsNullOrWhiteSpace(column.SourceName) && + !string.Equals(column.Name, column.SourceName, StringComparison.OrdinalIgnoreCase)) + { + builder.Append(" (source header: "); + builder.Append(column.SourceName); + builder.Append(')'); + } + + builder.Append(" — inferred type: "); + builder.Append(inferredTypes[i]); + builder.AppendLine(); + } + + return builder.ToString(); + } + + private static string[] InferColumnTypes(TabularDocumentArtifact artifact) + { + var headerCount = artifact?.Header?.Count ?? 0; + + if (headerCount == 0) + { + return []; + } + + var states = new InferredColumnType[headerCount]; + var sampleCounts = new int[headerCount]; + var rows = artifact?.Rows ?? []; + const int targetSamplesPerColumn = 32; + + for (var rowIndex = 0; rowIndex < rows.Count; rowIndex++) + { + var row = rows[rowIndex]; + var hasPendingColumns = false; + + for (var columnIndex = 0; columnIndex < headerCount; columnIndex++) + { + if (sampleCounts[columnIndex] >= targetSamplesPerColumn) + { + continue; + } + + hasPendingColumns = true; + var value = columnIndex < row.Count ? row[columnIndex] : null; + + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + states[columnIndex] = CombineTypes(states[columnIndex], ClassifyValue(value)); + sampleCounts[columnIndex]++; + } + + if (!hasPendingColumns) + { + break; + } + } + + var inferredTypes = new string[headerCount]; + + for (var i = 0; i < headerCount; i++) + { + inferredTypes[i] = FormatType(states[i]); + } + + return inferredTypes; + } + + private static InferredColumnType ClassifyValue(string value) + { + if (bool.TryParse(value, out _)) + { + return InferredColumnType.Boolean; + } + + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) + { + return InferredColumnType.Integer; + } + + if (decimal.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out _)) + { + return InferredColumnType.Decimal; + } + + if (LooksLikeDate(value) && + DateOnly.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AllowWhiteSpaces, out _)) + { + return InferredColumnType.Date; + } + + if (LooksLikeDateTime(value) && + DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AllowWhiteSpaces, out _)) + { + return InferredColumnType.DateTime; + } + + return InferredColumnType.Text; + } + + private static InferredColumnType CombineTypes(InferredColumnType current, InferredColumnType next) + { + if (current == InferredColumnType.Unknown) + { + return next; + } + + if (current == next) + { + return current; + } + + if ((current == InferredColumnType.Integer && next == InferredColumnType.Decimal) || + (current == InferredColumnType.Decimal && next == InferredColumnType.Integer)) + { + return InferredColumnType.Decimal; + } + + if ((current == InferredColumnType.Date && next == InferredColumnType.DateTime) || + (current == InferredColumnType.DateTime && next == InferredColumnType.Date)) + { + return InferredColumnType.DateTime; + } + + return InferredColumnType.Text; + } + + private static string FormatType(InferredColumnType value) + { + return value switch + { + InferredColumnType.Boolean => "boolean", + InferredColumnType.Integer => "integer", + InferredColumnType.Decimal => "decimal", + InferredColumnType.Date => "date", + InferredColumnType.DateTime => "datetime", + InferredColumnType.Text => "text", + _ => "empty", + }; + } + + private static bool LooksLikeDate(string value) + { + return value.Contains('-', StringComparison.Ordinal) || + value.Contains('/', StringComparison.Ordinal); + } + + private static bool LooksLikeDateTime(string value) + { + return LooksLikeDate(value) || + value.Contains(':', StringComparison.Ordinal) || + value.Contains('T', StringComparison.Ordinal); + } + + private static bool TryParseScope(string value, out MetadataScope scope) + { + scope = MetadataScope.Basic; + + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + return value.Trim().ToLowerInvariant() switch + { + "basic" => true, + "tabular_summary" => SetScope(MetadataScope.TabularSummary, out scope), + "headers" => SetScope(MetadataScope.Headers, out scope), + "columns" => SetScope(MetadataScope.Columns, out scope), + _ => false, + }; + } + + private static bool SetScope(MetadataScope value, out MetadataScope scope) + { + scope = value; + + return true; + } + + private enum MetadataScope + { + Basic, + TabularSummary, + Headers, + Columns, + } + + private enum InferredColumnType + { + Unknown, + Boolean, + Integer, + Decimal, + Date, + DateTime, + Text, + } +} diff --git a/src/Primitives/CrestApps.Core/Services/MultiSourceNamedCatalog.cs b/src/Primitives/CrestApps.Core/Services/MultiSourceNamedCatalog.cs index 3bb197e6..688ac6c3 100644 --- a/src/Primitives/CrestApps.Core/Services/MultiSourceNamedCatalog.cs +++ b/src/Primitives/CrestApps.Core/Services/MultiSourceNamedCatalog.cs @@ -13,6 +13,7 @@ public abstract class MultiSourceNamedCatalog : INamedCatalog { private readonly IEnumerable> _sources; private readonly IWritableNamedCatalogSource? _writableSource; + private IReadOnlyCollection? _cachedEntries; /// /// Initializes a new instance of the class. @@ -95,11 +96,18 @@ public async ValueTask> PageAsync(int page, int pageSize, /// /// The entry. /// The cancellation token. - public ValueTask DeleteAsync(T entry, CancellationToken cancellationToken = default) + public async ValueTask DeleteAsync(T entry, CancellationToken cancellationToken = default) { EnsureWritableSource(); - return _writableSource!.DeleteAsync(entry, cancellationToken); + var deleted = await _writableSource!.DeleteAsync(entry, cancellationToken); + + if (deleted) + { + InvalidateCache(); + } + + return deleted; } /// @@ -107,11 +115,13 @@ public ValueTask DeleteAsync(T entry, CancellationToken cancellationToken /// /// The entry. /// The cancellation token. - public ValueTask CreateAsync(T entry, CancellationToken cancellationToken = default) + public async ValueTask CreateAsync(T entry, CancellationToken cancellationToken = default) { EnsureWritableSource(); - return _writableSource!.CreateAsync(entry, cancellationToken); + await _writableSource!.CreateAsync(entry, cancellationToken); + + InvalidateCache(); } /// @@ -119,11 +129,13 @@ public ValueTask CreateAsync(T entry, CancellationToken cancellationToken = defa /// /// The entry. /// The cancellation token. - public ValueTask UpdateAsync(T entry, CancellationToken cancellationToken = default) + public async ValueTask UpdateAsync(T entry, CancellationToken cancellationToken = default) { EnsureWritableSource(); - return _writableSource!.UpdateAsync(entry, cancellationToken); + await _writableSource!.UpdateAsync(entry, cancellationToken); + + InvalidateCache(); } /// @@ -172,6 +184,11 @@ protected virtual IEnumerable ApplyFilters(QueryContext? context, IEnumerable /// The cancellation token. protected async ValueTask> GetMergedEntriesAsync(CancellationToken cancellationToken = default) { + if (_cachedEntries is not null) + { + return _cachedEntries; + } + var seenNames = new HashSet(StringComparer.OrdinalIgnoreCase); var merged = new List(); @@ -190,7 +207,17 @@ protected async ValueTask> GetMergedEntriesAsync(Cancella } } - return merged; + _cachedEntries = merged.ToArray(); + + return _cachedEntries; + } + + /// + /// Invalidates the scoped merged-entry cache so subsequent reads observe writes. + /// + protected virtual void InvalidateCache() + { + _cachedEntries = null; } /// diff --git a/src/Resources/CrestApps.AI.Resources/Assets/css/chat-widget.css b/src/Resources/CrestApps.AI.Resources/Assets/css/chat-widget.css index 1417a212..a905463f 100644 --- a/src/Resources/CrestApps.AI.Resources/Assets/css/chat-widget.css +++ b/src/Resources/CrestApps.AI.Resources/Assets/css/chat-widget.css @@ -155,7 +155,6 @@ } .ai-chat-widget-messages .ai-chat-message-heading { - font-size: 0.75rem; margin-bottom: 0.35rem; } diff --git a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css index 8e4be1a1..4b0c98b2 100644 --- a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css +++ b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css @@ -155,7 +155,6 @@ } .ai-chat-widget-messages .ai-chat-message-heading { - font-size: 0.75rem; margin-bottom: 0.35rem; } diff --git a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css.map b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css.map index fa8d7a18..a26fce99 100644 --- a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css.map +++ b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.css.map @@ -1 +1 @@ -{"version":3,"sources":["chat-widget.css"],"names":[],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA","file":"chat-widget.css","sourcesContent":["/* CrestApps AI Chat Widget */\n:root {\n --ai-widget-primary: var(--bs-secondary, #6c757d);\n --ai-widget-primary-rgb: var(--bs-secondary-rgb, 108, 117, 125);\n --ai-widget-width: 380px;\n --ai-widget-height: 520px;\n}\n\n.ai-chat-widget-toggle {\n position: fixed;\n bottom: 1.25rem;\n right: 1.25rem;\n z-index: 10000;\n width: 52px;\n height: 52px;\n border-radius: 50%;\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n border: none;\n cursor: pointer;\n display: flex;\n align-items: center;\n justify-content: center;\n font-size: 1.5rem;\n box-shadow: 0 4px 12px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.2);\n transition: transform 0.2s, box-shadow 0.2s;\n}\n\n.ai-chat-widget-toggle:hover {\n transform: scale(1.1);\n box-shadow: 0 6px 16px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.3);\n}\n\n.ai-chat-widget-container {\n position: fixed;\n bottom: 5rem;\n right: 1.25rem;\n z-index: 10001;\n width: var(--ai-widget-width);\n height: var(--ai-widget-height);\n background: var(--bs-body-bg, #fff);\n color: var(--bs-body-color, #212529);\n border: 1px solid var(--bs-border-color, #dee2e6);\n border-radius: 12px;\n box-shadow: 0 8px 32px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.18);\n display: none;\n flex-direction: column;\n overflow: hidden;\n}\n\n.ai-chat-widget-container.ai-chat-widget-resizable {\n resize: both;\n min-width: 320px;\n min-height: 420px;\n max-width: min(90vw, 960px);\n max-height: min(90vh, 960px);\n}\n\n.ai-chat-widget-container.open {\n display: flex;\n}\n\n.ai-chat-widget-header {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 0.75rem 1rem;\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n}\n\n.ai-chat-widget-header.ai-chat-widget-drag-handle {\n cursor: move;\n user-select: none;\n touch-action: none;\n}\n\n.ai-chat-widget-header-actions {\n display: flex;\n align-items: center;\n gap: 0.15rem;\n}\n\n.ai-chat-widget-toggle.ai-chat-widget-draggable {\n touch-action: none;\n}\n\n.ai-chat-widget-container.ai-chat-widget-dragging,\n.ai-chat-widget-toggle.ai-chat-widget-dragging {\n transition: none;\n}\n\n.ai-chat-widget-toggle.ai-chat-widget-dragging:hover {\n transform: none;\n}\n\n.ai-chat-widget-header .title {\n font-weight: 600;\n font-size: 0.95rem;\n}\n\n.ai-chat-widget-header button {\n background: none;\n border: none;\n color: var(--bs-white, #fff);\n cursor: pointer;\n font-size: 1rem;\n padding: 0 0.25rem;\n opacity: 0.8;\n}\n\n.ai-chat-widget-header button:hover {\n opacity: 1;\n}\n\n.ai-chat-widget-profile-select {\n padding: 0.5rem 0.75rem;\n border-bottom: 1px solid var(--bs-border-color, #dee2e6);\n background: var(--bs-tertiary-bg, #f8f9fa);\n}\n\n.ai-chat-widget-profile-select select {\n width: 100%;\n padding: 0.35rem 0.5rem;\n color: var(--bs-body-color, #212529);\n background-color: var(--bs-body-bg, #fff);\n border: 1px solid var(--bs-border-color, #ced4da);\n border-radius: 6px;\n font-size: 0.85rem;\n}\n\n.ai-chat-widget-messages {\n flex: 1;\n overflow-y: auto;\n padding: 0.75rem;\n font-size: 0.9rem;\n}\n\n/* ── Widget message items (ai-chat.js Vue template) ── */\n.ai-chat-widget-messages .ai-chat-messages {\n padding: 0.5rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item {\n position: relative;\n padding: 0.5rem 0;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item + .ai-chat-message-item::before {\n content: '';\n display: block;\n width: 100%;\n margin-bottom: 0.5rem;\n border-top: 1px solid rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.08);\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading {\n font-size: 0.75rem;\n margin-bottom: 0.35rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading .ai-chat-msg-role {\n margin-bottom: 0;\n text-transform: uppercase;\n letter-spacing: 0.03em;\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading-assistant .ai-chat-msg-role {\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-role-label {\n display: inline-block;\n text-transform: inherit;\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role {\n font-weight: 600;\n margin-bottom: 2px;\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role-user {\n color: var(--bs-primary, #0d6efd);\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role-assistant {\n color: var(--ai-widget-primary, #6f42c1);\n}\n\n.ai-chat-widget-messages .ai-bot-icon {\n color: currentColor;\n display: inline-block;\n animation: none;\n}\n\n@keyframes ai-widget-streaming-effect {\n 0% { opacity: 1; transform: rotate(0deg); }\n 10% { opacity: 0.3; transform: rotate(0deg); }\n 20% { opacity: 1; transform: rotate(0deg); }\n 30% { opacity: 0.3; transform: rotate(0deg); }\n 40% { opacity: 1; transform: rotate(0deg); }\n 50% { opacity: 0.3; transform: rotate(0deg); }\n 60% { opacity: 1; transform: rotate(0deg); }\n 80% { opacity: 1; transform: rotate(360deg); }\n 100% { opacity: 1; transform: rotate(360deg); }\n}\n\n.ai-chat-widget-messages .ai-streaming-icon {\n color: currentColor;\n display: inline-block;\n animation: ai-widget-streaming-effect 4s ease-in-out infinite;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body {\n position: relative;\n padding: 0;\n overflow-wrap: break-word;\n word-break: break-word;\n min-width: 0;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body p:last-child {\n margin-bottom: 0;\n}\n\n/* ── Message action buttons (copy, thumbs up/down) ── */\n.ai-chat-widget-messages .message-buttons-container {\n position: absolute;\n right: 0.1rem;\n bottom: 0;\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n width: auto;\n max-width: max-content;\n opacity: 0;\n pointer-events: none;\n transition: opacity 0.15s ease-in-out;\n background: rgba(var(--bs-body-bg-rgb, 255, 255, 255), 0.96);\n padding: 0.2rem 0.35rem;\n border-radius: 999px;\n box-shadow: 0 1px 3px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.08);\n margin-bottom: 1rem;\n transform: translateY(50%);\n z-index: 1;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item:hover .message-buttons-container,\n.ai-chat-widget-messages .ai-chat-message-item:focus-within .message-buttons-container {\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-messages .message-buttons-container:has(.tts-playing) {\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox {\n display: inline-flex;\n align-items: center;\n justify-content: center;\n flex: 0 0 auto;\n width: 1.75rem;\n height: 1.75rem;\n margin: 0 !important;\n padding: 0 !important;\n border-radius: 50%;\n color: var(--bs-secondary-color, #6c757d) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container .ai-chat-message-assistant-feedback {\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox:hover,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox:focus {\n color: var(--bs-body-color, #212529) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success:hover,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success:focus {\n color: var(--bs-success, #198754) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success svg {\n color: inherit !important;\n fill: currentColor;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.tts-playing {\n color: var(--ai-widget-primary, #6f42c1) !important;\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-history-panel {\n position: absolute;\n top: 3.5rem;\n left: 0;\n right: 0;\n bottom: 0;\n background: var(--bs-body-bg, #fff);\n padding: 1rem;\n z-index: 10;\n display: none;\n overflow-y: auto;\n flex-direction: column;\n}\n\n.ai-chat-widget-history-panel.show {\n display: flex;\n}\n\n.ai-chat-widget-history-panel .history-header {\n display: flex;\n justify-content: space-between;\n align-items: center;\n margin-bottom: 0.75rem;\n flex-shrink: 0;\n}\n\n.ai-chat-widget-history-panel .history-header h5 {\n margin: 0;\n font-size: 1rem;\n}\n\n.ai-chat-widget-history-list {\n list-style: none;\n padding: 0;\n margin: 0;\n overflow-y: auto;\n flex: 1 1 auto;\n}\n\n.ai-chat-widget-history-list li {\n margin-bottom: 0;\n}\n\n.ai-chat-widget-history-list .chat-session-history-item {\n display: block;\n padding: 0.5rem 0.75rem;\n text-decoration: none;\n color: var(--bs-body-color, #212529);\n font-size: 0.85rem;\n white-space: nowrap;\n overflow: hidden;\n text-overflow: ellipsis;\n transition: background 0.15s;\n cursor: pointer;\n border-bottom: 1px solid var(--bs-border-color, #dee2e6);\n}\n\n.ai-chat-widget-history-list li:last-child .chat-session-history-item {\n border-bottom: none;\n}\n\n.ai-chat-widget-history-list .chat-session-history-item:hover {\n background: var(--bs-tertiary-bg, #f8f9fa);\n}\n\n/* ── Notifications ── */\n.ai-chat-widget-messages .ai-chat-notification {\n margin: 0.75rem 0.5rem 0;\n padding: 0.625rem 0.75rem;\n border: 1px solid rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.12);\n border-radius: 0.5rem;\n background: rgba(var(--bs-tertiary-bg-rgb, 248, 249, 250), 0.98);\n}\n\n.ai-chat-widget-messages .ai-chat-notification-content {\n display: flex;\n align-items: flex-start;\n gap: 0.5rem;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-icon {\n margin-top: 0.1rem;\n color: var(--bs-secondary-color, #6c757d);\n}\n\n.ai-chat-widget-messages .ai-chat-notification-text {\n flex: 1 1 auto;\n min-width: 0;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-dismiss {\n margin-left: auto !important;\n color: var(--bs-secondary-color, #6c757d) !important;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-actions {\n display: flex;\n gap: 0.5rem;\n margin-top: 0.5rem;\n}\n\n/* ── Chart containers inside widget ── */\n.ai-chat-widget-messages .chart-container {\n width: 100%;\n max-width: 100% !important;\n min-height: 280px;\n}\n\n/* ── Code blocks & inline code ── */\n.ai-chat-widget-messages .ai-chat-message-body pre {\n background: var(--bs-dark-bg-subtle, #2d2d2d);\n color: var(--bs-emphasis-color, #f8f8f2);\n padding: 0.5rem;\n border-radius: 4px;\n overflow-x: auto;\n font-size: 0.8rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body code {\n font-size: 0.85em;\n}\n\n.ai-chat-widget-input {\n border-top: 1px solid var(--bs-border-color, #dee2e6);\n padding: 0.5rem 0.75rem;\n display: flex;\n flex-wrap: wrap;\n gap: 0.5rem;\n align-items: flex-end;\n}\n\n.ai-chat-widget-input .ai-chat-doc-bar {\n flex: 0 0 100%;\n margin: -0.5rem -0.75rem 0.5rem;\n padding: 0.5rem 0.75rem;\n max-height: 4.5rem;\n overflow-y: auto;\n}\n\n.ai-chat-widget-input textarea {\n flex: 1;\n min-width: 0;\n color: var(--bs-body-color, #212529);\n background-color: var(--bs-body-bg, #fff);\n border: 1px solid var(--bs-border-color, #ced4da);\n border-radius: 8px;\n padding: 0.4rem 0.6rem;\n font-size: 0.9rem;\n resize: none;\n max-height: 80px;\n min-height: 36px;\n line-height: 1.4;\n outline: none;\n}\n\n.ai-chat-widget-input textarea:focus {\n border-color: var(--ai-widget-primary);\n box-shadow: 0 0 0 2px rgba(var(--ai-widget-primary-rgb, 108, 117, 125), 0.15);\n}\n\n.ai-chat-widget-input button {\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n border: none;\n border-radius: 8px;\n padding: 0.4rem 0.75rem;\n cursor: pointer;\n font-size: 0.9rem;\n white-space: nowrap;\n}\n\n.ai-chat-widget-input .btn {\n background: var(--bs-body-bg, #fff);\n color: var(--bs-body-color, #212529);\n border: 1px solid var(--bs-border-color, #ced4da);\n}\n\n.ai-chat-widget-input .btn.btn-link {\n background: transparent;\n color: var(--bs-secondary-color, #6c757d);\n border: none;\n padding: 0.4rem;\n}\n\n.ai-chat-widget-input .btn.btn-outline-dark {\n background: var(--bs-dark, #212529);\n color: var(--bs-white, #fff);\n border-color: var(--bs-dark, #212529);\n}\n\n.ai-chat-widget-input .btn.btn-outline-secondary {\n color: var(--bs-secondary-color, #6c757d);\n}\n\n.ai-chat-widget-input button:disabled {\n opacity: 0.5;\n cursor: not-allowed;\n}\n\n.ai-chat-widget-status {\n padding: 0.25rem 0.75rem;\n font-size: 0.8rem;\n color: var(--bs-secondary-color, #6c757d);\n display: none;\n}\n\n.ai-chat-widget-status.visible {\n display: block;\n}\n\n.ai-chat-widget-welcome {\n text-align: center;\n color: var(--bs-secondary-color, #6c757d);\n padding: 2rem 1rem;\n font-size: 0.9rem;\n}\n\n@media (max-width: 576px) {\n .ai-chat-widget-container {\n width: calc(100vw - 1rem);\n height: calc(100vh - 6rem);\n right: 0.5rem;\n bottom: 4.5rem;\n }\n}\n"]} +{"version":3,"sources":["chat-widget.css"],"names":[],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA","file":"chat-widget.css","sourcesContent":["/* CrestApps AI Chat Widget */\n:root {\n --ai-widget-primary: var(--bs-secondary, #6c757d);\n --ai-widget-primary-rgb: var(--bs-secondary-rgb, 108, 117, 125);\n --ai-widget-width: 380px;\n --ai-widget-height: 520px;\n}\n\n.ai-chat-widget-toggle {\n position: fixed;\n bottom: 1.25rem;\n right: 1.25rem;\n z-index: 10000;\n width: 52px;\n height: 52px;\n border-radius: 50%;\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n border: none;\n cursor: pointer;\n display: flex;\n align-items: center;\n justify-content: center;\n font-size: 1.5rem;\n box-shadow: 0 4px 12px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.2);\n transition: transform 0.2s, box-shadow 0.2s;\n}\n\n.ai-chat-widget-toggle:hover {\n transform: scale(1.1);\n box-shadow: 0 6px 16px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.3);\n}\n\n.ai-chat-widget-container {\n position: fixed;\n bottom: 5rem;\n right: 1.25rem;\n z-index: 10001;\n width: var(--ai-widget-width);\n height: var(--ai-widget-height);\n background: var(--bs-body-bg, #fff);\n color: var(--bs-body-color, #212529);\n border: 1px solid var(--bs-border-color, #dee2e6);\n border-radius: 12px;\n box-shadow: 0 8px 32px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.18);\n display: none;\n flex-direction: column;\n overflow: hidden;\n}\n\n.ai-chat-widget-container.ai-chat-widget-resizable {\n resize: both;\n min-width: 320px;\n min-height: 420px;\n max-width: min(90vw, 960px);\n max-height: min(90vh, 960px);\n}\n\n.ai-chat-widget-container.open {\n display: flex;\n}\n\n.ai-chat-widget-header {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 0.75rem 1rem;\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n}\n\n.ai-chat-widget-header.ai-chat-widget-drag-handle {\n cursor: move;\n user-select: none;\n touch-action: none;\n}\n\n.ai-chat-widget-header-actions {\n display: flex;\n align-items: center;\n gap: 0.15rem;\n}\n\n.ai-chat-widget-toggle.ai-chat-widget-draggable {\n touch-action: none;\n}\n\n.ai-chat-widget-container.ai-chat-widget-dragging,\n.ai-chat-widget-toggle.ai-chat-widget-dragging {\n transition: none;\n}\n\n.ai-chat-widget-toggle.ai-chat-widget-dragging:hover {\n transform: none;\n}\n\n.ai-chat-widget-header .title {\n font-weight: 600;\n font-size: 0.95rem;\n}\n\n.ai-chat-widget-header button {\n background: none;\n border: none;\n color: var(--bs-white, #fff);\n cursor: pointer;\n font-size: 1rem;\n padding: 0 0.25rem;\n opacity: 0.8;\n}\n\n.ai-chat-widget-header button:hover {\n opacity: 1;\n}\n\n.ai-chat-widget-profile-select {\n padding: 0.5rem 0.75rem;\n border-bottom: 1px solid var(--bs-border-color, #dee2e6);\n background: var(--bs-tertiary-bg, #f8f9fa);\n}\n\n.ai-chat-widget-profile-select select {\n width: 100%;\n padding: 0.35rem 0.5rem;\n color: var(--bs-body-color, #212529);\n background-color: var(--bs-body-bg, #fff);\n border: 1px solid var(--bs-border-color, #ced4da);\n border-radius: 6px;\n font-size: 0.85rem;\n}\n\n.ai-chat-widget-messages {\n flex: 1;\n overflow-y: auto;\n padding: 0.75rem;\n font-size: 0.9rem;\n}\n\n/* ── Widget message items (ai-chat.js Vue template) ── */\n.ai-chat-widget-messages .ai-chat-messages {\n padding: 0.5rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item {\n position: relative;\n padding: 0.5rem 0;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item + .ai-chat-message-item::before {\n content: '';\n display: block;\n width: 100%;\n margin-bottom: 0.5rem;\n border-top: 1px solid rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.08);\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading {\n margin-bottom: 0.35rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading .ai-chat-msg-role {\n margin-bottom: 0;\n text-transform: uppercase;\n letter-spacing: 0.03em;\n}\n\n.ai-chat-widget-messages .ai-chat-message-heading-assistant .ai-chat-msg-role {\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-role-label {\n display: inline-block;\n text-transform: inherit;\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role {\n font-weight: 600;\n margin-bottom: 2px;\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role-user {\n color: var(--bs-primary, #0d6efd);\n}\n\n.ai-chat-widget-messages .ai-chat-msg-role-assistant {\n color: var(--ai-widget-primary, #6f42c1);\n}\n\n.ai-chat-widget-messages .ai-bot-icon {\n color: currentColor;\n display: inline-block;\n animation: none;\n}\n\n@keyframes ai-widget-streaming-effect {\n 0% { opacity: 1; transform: rotate(0deg); }\n 10% { opacity: 0.3; transform: rotate(0deg); }\n 20% { opacity: 1; transform: rotate(0deg); }\n 30% { opacity: 0.3; transform: rotate(0deg); }\n 40% { opacity: 1; transform: rotate(0deg); }\n 50% { opacity: 0.3; transform: rotate(0deg); }\n 60% { opacity: 1; transform: rotate(0deg); }\n 80% { opacity: 1; transform: rotate(360deg); }\n 100% { opacity: 1; transform: rotate(360deg); }\n}\n\n.ai-chat-widget-messages .ai-streaming-icon {\n color: currentColor;\n display: inline-block;\n animation: ai-widget-streaming-effect 4s ease-in-out infinite;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body {\n position: relative;\n padding: 0;\n overflow-wrap: break-word;\n word-break: break-word;\n min-width: 0;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body p:last-child {\n margin-bottom: 0;\n}\n\n/* ── Message action buttons (copy, thumbs up/down) ── */\n.ai-chat-widget-messages .message-buttons-container {\n position: absolute;\n right: 0.1rem;\n bottom: 0;\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n width: auto;\n max-width: max-content;\n opacity: 0;\n pointer-events: none;\n transition: opacity 0.15s ease-in-out;\n background: rgba(var(--bs-body-bg-rgb, 255, 255, 255), 0.96);\n padding: 0.2rem 0.35rem;\n border-radius: 999px;\n box-shadow: 0 1px 3px rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.08);\n margin-bottom: 1rem;\n transform: translateY(50%);\n z-index: 1;\n}\n\n.ai-chat-widget-messages .ai-chat-message-item:hover .message-buttons-container,\n.ai-chat-widget-messages .ai-chat-message-item:focus-within .message-buttons-container {\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-messages .message-buttons-container:has(.tts-playing) {\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox {\n display: inline-flex;\n align-items: center;\n justify-content: center;\n flex: 0 0 auto;\n width: 1.75rem;\n height: 1.75rem;\n margin: 0 !important;\n padding: 0 !important;\n border-radius: 50%;\n color: var(--bs-secondary-color, #6c757d) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container .ai-chat-message-assistant-feedback {\n display: inline-flex;\n align-items: center;\n gap: 0.35rem;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox:hover,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox:focus {\n color: var(--bs-body-color, #212529) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success:hover,\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success:focus {\n color: var(--bs-success, #198754) !important;\n text-decoration: none;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.text-success svg {\n color: inherit !important;\n fill: currentColor;\n}\n\n.ai-chat-widget-messages .message-buttons-container > .button-message-toolbox.tts-playing {\n color: var(--ai-widget-primary, #6f42c1) !important;\n opacity: 1;\n pointer-events: auto;\n}\n\n.ai-chat-widget-history-panel {\n position: absolute;\n top: 3.5rem;\n left: 0;\n right: 0;\n bottom: 0;\n background: var(--bs-body-bg, #fff);\n padding: 1rem;\n z-index: 10;\n display: none;\n overflow-y: auto;\n flex-direction: column;\n}\n\n.ai-chat-widget-history-panel.show {\n display: flex;\n}\n\n.ai-chat-widget-history-panel .history-header {\n display: flex;\n justify-content: space-between;\n align-items: center;\n margin-bottom: 0.75rem;\n flex-shrink: 0;\n}\n\n.ai-chat-widget-history-panel .history-header h5 {\n margin: 0;\n font-size: 1rem;\n}\n\n.ai-chat-widget-history-list {\n list-style: none;\n padding: 0;\n margin: 0;\n overflow-y: auto;\n flex: 1 1 auto;\n}\n\n.ai-chat-widget-history-list li {\n margin-bottom: 0;\n}\n\n.ai-chat-widget-history-list .chat-session-history-item {\n display: block;\n padding: 0.5rem 0.75rem;\n text-decoration: none;\n color: var(--bs-body-color, #212529);\n font-size: 0.85rem;\n white-space: nowrap;\n overflow: hidden;\n text-overflow: ellipsis;\n transition: background 0.15s;\n cursor: pointer;\n border-bottom: 1px solid var(--bs-border-color, #dee2e6);\n}\n\n.ai-chat-widget-history-list li:last-child .chat-session-history-item {\n border-bottom: none;\n}\n\n.ai-chat-widget-history-list .chat-session-history-item:hover {\n background: var(--bs-tertiary-bg, #f8f9fa);\n}\n\n/* ── Notifications ── */\n.ai-chat-widget-messages .ai-chat-notification {\n margin: 0.75rem 0.5rem 0;\n padding: 0.625rem 0.75rem;\n border: 1px solid rgba(var(--bs-body-color-rgb, 33, 37, 41), 0.12);\n border-radius: 0.5rem;\n background: rgba(var(--bs-tertiary-bg-rgb, 248, 249, 250), 0.98);\n}\n\n.ai-chat-widget-messages .ai-chat-notification-content {\n display: flex;\n align-items: flex-start;\n gap: 0.5rem;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-icon {\n margin-top: 0.1rem;\n color: var(--bs-secondary-color, #6c757d);\n}\n\n.ai-chat-widget-messages .ai-chat-notification-text {\n flex: 1 1 auto;\n min-width: 0;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-dismiss {\n margin-left: auto !important;\n color: var(--bs-secondary-color, #6c757d) !important;\n}\n\n.ai-chat-widget-messages .ai-chat-notification-actions {\n display: flex;\n gap: 0.5rem;\n margin-top: 0.5rem;\n}\n\n/* ── Chart containers inside widget ── */\n.ai-chat-widget-messages .chart-container {\n width: 100%;\n max-width: 100% !important;\n min-height: 280px;\n}\n\n/* ── Code blocks & inline code ── */\n.ai-chat-widget-messages .ai-chat-message-body pre {\n background: var(--bs-dark-bg-subtle, #2d2d2d);\n color: var(--bs-emphasis-color, #f8f8f2);\n padding: 0.5rem;\n border-radius: 4px;\n overflow-x: auto;\n font-size: 0.8rem;\n}\n\n.ai-chat-widget-messages .ai-chat-message-body code {\n font-size: 0.85em;\n}\n\n.ai-chat-widget-input {\n border-top: 1px solid var(--bs-border-color, #dee2e6);\n padding: 0.5rem 0.75rem;\n display: flex;\n flex-wrap: wrap;\n gap: 0.5rem;\n align-items: flex-end;\n}\n\n.ai-chat-widget-input .ai-chat-doc-bar {\n flex: 0 0 100%;\n margin: -0.5rem -0.75rem 0.5rem;\n padding: 0.5rem 0.75rem;\n max-height: 4.5rem;\n overflow-y: auto;\n}\n\n.ai-chat-widget-input textarea {\n flex: 1;\n min-width: 0;\n color: var(--bs-body-color, #212529);\n background-color: var(--bs-body-bg, #fff);\n border: 1px solid var(--bs-border-color, #ced4da);\n border-radius: 8px;\n padding: 0.4rem 0.6rem;\n font-size: 0.9rem;\n resize: none;\n max-height: 80px;\n min-height: 36px;\n line-height: 1.4;\n outline: none;\n}\n\n.ai-chat-widget-input textarea:focus {\n border-color: var(--ai-widget-primary);\n box-shadow: 0 0 0 2px rgba(var(--ai-widget-primary-rgb, 108, 117, 125), 0.15);\n}\n\n.ai-chat-widget-input button {\n background: var(--ai-widget-primary);\n color: var(--bs-white, #fff);\n border: none;\n border-radius: 8px;\n padding: 0.4rem 0.75rem;\n cursor: pointer;\n font-size: 0.9rem;\n white-space: nowrap;\n}\n\n.ai-chat-widget-input .btn {\n background: var(--bs-body-bg, #fff);\n color: var(--bs-body-color, #212529);\n border: 1px solid var(--bs-border-color, #ced4da);\n}\n\n.ai-chat-widget-input .btn.btn-link {\n background: transparent;\n color: var(--bs-secondary-color, #6c757d);\n border: none;\n padding: 0.4rem;\n}\n\n.ai-chat-widget-input .btn.btn-outline-dark {\n background: var(--bs-dark, #212529);\n color: var(--bs-white, #fff);\n border-color: var(--bs-dark, #212529);\n}\n\n.ai-chat-widget-input .btn.btn-outline-secondary {\n color: var(--bs-secondary-color, #6c757d);\n}\n\n.ai-chat-widget-input button:disabled {\n opacity: 0.5;\n cursor: not-allowed;\n}\n\n.ai-chat-widget-status {\n padding: 0.25rem 0.75rem;\n font-size: 0.8rem;\n color: var(--bs-secondary-color, #6c757d);\n display: none;\n}\n\n.ai-chat-widget-status.visible {\n display: block;\n}\n\n.ai-chat-widget-welcome {\n text-align: center;\n color: var(--bs-secondary-color, #6c757d);\n padding: 2rem 1rem;\n font-size: 0.9rem;\n}\n\n@media (max-width: 576px) {\n .ai-chat-widget-container {\n width: calc(100vw - 1rem);\n height: calc(100vh - 6rem);\n right: 0.5rem;\n bottom: 4.5rem;\n }\n}\n"]} diff --git a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.min.css b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.min.css index 7481ea5f..792db4e9 100644 --- a/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.min.css +++ b/src/Resources/CrestApps.AI.Resources/wwwroot/styles/chat-widget.min.css @@ -1 +1 @@ -:root{--ai-widget-primary:var(--bs-secondary, #6c757d);--ai-widget-primary-rgb:var(--bs-secondary-rgb, 108, 117, 125);--ai-widget-width:380px;--ai-widget-height:520px}.ai-chat-widget-toggle{position:fixed;bottom:1.25rem;right:1.25rem;z-index:10000;width:52px;height:52px;border-radius:50%;background:var(--ai-widget-primary);color:var(--bs-white,#fff);border:none;cursor:pointer;display:flex;align-items:center;justify-content:center;font-size:1.5rem;box-shadow:0 4px 12px rgba(var(--bs-body-color-rgb,33,37,41),.2);transition:transform .2s,box-shadow .2s}.ai-chat-widget-toggle:hover{transform:scale(1.1);box-shadow:0 6px 16px rgba(var(--bs-body-color-rgb,33,37,41),.3)}.ai-chat-widget-container{position:fixed;bottom:5rem;right:1.25rem;z-index:10001;width:var(--ai-widget-width);height:var(--ai-widget-height);background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);border:1px solid var(--bs-border-color,#dee2e6);border-radius:12px;box-shadow:0 8px 32px rgba(var(--bs-body-color-rgb,33,37,41),.18);display:none;flex-direction:column;overflow:hidden}.ai-chat-widget-container.ai-chat-widget-resizable{resize:both;min-width:320px;min-height:420px;max-width:min(90vw,960px);max-height:min(90vh,960px)}.ai-chat-widget-container.open{display:flex}.ai-chat-widget-header{display:flex;align-items:center;justify-content:space-between;padding:.75rem 1rem;background:var(--ai-widget-primary);color:var(--bs-white,#fff)}.ai-chat-widget-header.ai-chat-widget-drag-handle{cursor:move;user-select:none;touch-action:none}.ai-chat-widget-header-actions{display:flex;align-items:center;gap:.15rem}.ai-chat-widget-toggle.ai-chat-widget-draggable{touch-action:none}.ai-chat-widget-container.ai-chat-widget-dragging,.ai-chat-widget-toggle.ai-chat-widget-dragging{transition:none}.ai-chat-widget-toggle.ai-chat-widget-dragging:hover{transform:none}.ai-chat-widget-header .title{font-weight:600;font-size:.95rem}.ai-chat-widget-header button{background:0 0;border:none;color:var(--bs-white,#fff);cursor:pointer;font-size:1rem;padding:0 .25rem;opacity:.8}.ai-chat-widget-header button:hover{opacity:1}.ai-chat-widget-profile-select{padding:.5rem .75rem;border-bottom:1px solid var(--bs-border-color,#dee2e6);background:var(--bs-tertiary-bg,#f8f9fa)}.ai-chat-widget-profile-select select{width:100%;padding:.35rem .5rem;color:var(--bs-body-color,#212529);background-color:var(--bs-body-bg,#fff);border:1px solid var(--bs-border-color,#ced4da);border-radius:6px;font-size:.85rem}.ai-chat-widget-messages{flex:1;overflow-y:auto;padding:.75rem;font-size:.9rem}.ai-chat-widget-messages .ai-chat-messages{padding:.5rem}.ai-chat-widget-messages .ai-chat-message-item{position:relative;padding:.5rem 0}.ai-chat-widget-messages .ai-chat-message-item+.ai-chat-message-item::before{content:'';display:block;width:100%;margin-bottom:.5rem;border-top:1px solid rgba(var(--bs-body-color-rgb,33,37,41),.08)}.ai-chat-widget-messages .ai-chat-message-heading{font-size:.75rem;margin-bottom:.35rem}.ai-chat-widget-messages .ai-chat-message-heading .ai-chat-msg-role{margin-bottom:0;text-transform:uppercase;letter-spacing:.03em}.ai-chat-widget-messages .ai-chat-message-heading-assistant .ai-chat-msg-role{display:inline-flex;align-items:center;gap:.35rem}.ai-chat-widget-messages .ai-chat-message-role-label{display:inline-block;text-transform:inherit}.ai-chat-widget-messages .ai-chat-msg-role{font-weight:600;margin-bottom:2px}.ai-chat-widget-messages .ai-chat-msg-role-user{color:var(--bs-primary,#0d6efd)}.ai-chat-widget-messages .ai-chat-msg-role-assistant{color:var(--ai-widget-primary,#6f42c1)}.ai-chat-widget-messages .ai-bot-icon{color:currentColor;display:inline-block;animation:none}@keyframes ai-widget-streaming-effect{0%{opacity:1;transform:rotate(0)}10%{opacity:.3;transform:rotate(0)}20%{opacity:1;transform:rotate(0)}30%{opacity:.3;transform:rotate(0)}40%{opacity:1;transform:rotate(0)}50%{opacity:.3;transform:rotate(0)}60%{opacity:1;transform:rotate(0)}80%{opacity:1;transform:rotate(360deg)}100%{opacity:1;transform:rotate(360deg)}}.ai-chat-widget-messages .ai-streaming-icon{color:currentColor;display:inline-block;animation:ai-widget-streaming-effect 4s ease-in-out infinite}.ai-chat-widget-messages .ai-chat-message-body{position:relative;padding:0;overflow-wrap:break-word;word-break:break-word;min-width:0}.ai-chat-widget-messages .ai-chat-message-body p:last-child{margin-bottom:0}.ai-chat-widget-messages .message-buttons-container{position:absolute;right:.1rem;bottom:0;display:inline-flex;align-items:center;gap:.35rem;width:auto;max-width:max-content;opacity:0;pointer-events:none;transition:opacity .15s ease-in-out;background:rgba(var(--bs-body-bg-rgb,255,255,255),.96);padding:.2rem .35rem;border-radius:999px;box-shadow:0 1px 3px rgba(var(--bs-body-color-rgb,33,37,41),.08);margin-bottom:1rem;transform:translateY(50%);z-index:1}.ai-chat-widget-messages .ai-chat-message-item:focus-within .message-buttons-container,.ai-chat-widget-messages .ai-chat-message-item:hover .message-buttons-container{opacity:1;pointer-events:auto}.ai-chat-widget-messages .message-buttons-container:has(.tts-playing){opacity:1;pointer-events:auto}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox{display:inline-flex;align-items:center;justify-content:center;flex:0 0 auto;width:1.75rem;height:1.75rem;margin:0!important;padding:0!important;border-radius:50%;color:var(--bs-secondary-color,#6c757d)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container .ai-chat-message-assistant-feedback{display:inline-flex;align-items:center;gap:.35rem}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox:focus,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox:hover{color:var(--bs-body-color,#212529)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success:focus,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success:hover{color:var(--bs-success,#198754)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success svg{color:inherit!important;fill:currentColor}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.tts-playing{color:var(--ai-widget-primary,#6f42c1)!important;opacity:1;pointer-events:auto}.ai-chat-widget-history-panel{position:absolute;top:3.5rem;left:0;right:0;bottom:0;background:var(--bs-body-bg,#fff);padding:1rem;z-index:10;display:none;overflow-y:auto;flex-direction:column}.ai-chat-widget-history-panel.show{display:flex}.ai-chat-widget-history-panel .history-header{display:flex;justify-content:space-between;align-items:center;margin-bottom:.75rem;flex-shrink:0}.ai-chat-widget-history-panel .history-header h5{margin:0;font-size:1rem}.ai-chat-widget-history-list{list-style:none;padding:0;margin:0;overflow-y:auto;flex:1 1 auto}.ai-chat-widget-history-list li{margin-bottom:0}.ai-chat-widget-history-list .chat-session-history-item{display:block;padding:.5rem .75rem;text-decoration:none;color:var(--bs-body-color,#212529);font-size:.85rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:background .15s;cursor:pointer;border-bottom:1px solid var(--bs-border-color,#dee2e6)}.ai-chat-widget-history-list li:last-child .chat-session-history-item{border-bottom:none}.ai-chat-widget-history-list .chat-session-history-item:hover{background:var(--bs-tertiary-bg,#f8f9fa)}.ai-chat-widget-messages .ai-chat-notification{margin:.75rem .5rem 0;padding:.625rem .75rem;border:1px solid rgba(var(--bs-body-color-rgb,33,37,41),.12);border-radius:.5rem;background:rgba(var(--bs-tertiary-bg-rgb,248,249,250),.98)}.ai-chat-widget-messages .ai-chat-notification-content{display:flex;align-items:flex-start;gap:.5rem}.ai-chat-widget-messages .ai-chat-notification-icon{margin-top:.1rem;color:var(--bs-secondary-color,#6c757d)}.ai-chat-widget-messages .ai-chat-notification-text{flex:1 1 auto;min-width:0}.ai-chat-widget-messages .ai-chat-notification-dismiss{margin-left:auto!important;color:var(--bs-secondary-color,#6c757d)!important}.ai-chat-widget-messages .ai-chat-notification-actions{display:flex;gap:.5rem;margin-top:.5rem}.ai-chat-widget-messages .chart-container{width:100%;max-width:100%!important;min-height:280px}.ai-chat-widget-messages .ai-chat-message-body pre{background:var(--bs-dark-bg-subtle,#2d2d2d);color:var(--bs-emphasis-color,#f8f8f2);padding:.5rem;border-radius:4px;overflow-x:auto;font-size:.8rem}.ai-chat-widget-messages .ai-chat-message-body code{font-size:.85em}.ai-chat-widget-input{border-top:1px solid var(--bs-border-color,#dee2e6);padding:.5rem .75rem;display:flex;flex-wrap:wrap;gap:.5rem;align-items:flex-end}.ai-chat-widget-input .ai-chat-doc-bar{flex:0 0 100%;margin:-.5rem -.75rem .5rem;padding:.5rem .75rem;max-height:4.5rem;overflow-y:auto}.ai-chat-widget-input textarea{flex:1;min-width:0;color:var(--bs-body-color,#212529);background-color:var(--bs-body-bg,#fff);border:1px solid var(--bs-border-color,#ced4da);border-radius:8px;padding:.4rem .6rem;font-size:.9rem;resize:none;max-height:80px;min-height:36px;line-height:1.4;outline:0}.ai-chat-widget-input textarea:focus{border-color:var(--ai-widget-primary);box-shadow:0 0 0 2px rgba(var(--ai-widget-primary-rgb,108,117,125),.15)}.ai-chat-widget-input button{background:var(--ai-widget-primary);color:var(--bs-white,#fff);border:none;border-radius:8px;padding:.4rem .75rem;cursor:pointer;font-size:.9rem;white-space:nowrap}.ai-chat-widget-input .btn{background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);border:1px solid var(--bs-border-color,#ced4da)}.ai-chat-widget-input .btn.btn-link{background:0 0;color:var(--bs-secondary-color,#6c757d);border:none;padding:.4rem}.ai-chat-widget-input .btn.btn-outline-dark{background:var(--bs-dark,#212529);color:var(--bs-white,#fff);border-color:var(--bs-dark,#212529)}.ai-chat-widget-input .btn.btn-outline-secondary{color:var(--bs-secondary-color,#6c757d)}.ai-chat-widget-input button:disabled{opacity:.5;cursor:not-allowed}.ai-chat-widget-status{padding:.25rem .75rem;font-size:.8rem;color:var(--bs-secondary-color,#6c757d);display:none}.ai-chat-widget-status.visible{display:block}.ai-chat-widget-welcome{text-align:center;color:var(--bs-secondary-color,#6c757d);padding:2rem 1rem;font-size:.9rem}@media (max-width:576px){.ai-chat-widget-container{width:calc(100vw - 1rem);height:calc(100vh - 6rem);right:.5rem;bottom:4.5rem}} +:root{--ai-widget-primary:var(--bs-secondary, #6c757d);--ai-widget-primary-rgb:var(--bs-secondary-rgb, 108, 117, 125);--ai-widget-width:380px;--ai-widget-height:520px}.ai-chat-widget-toggle{position:fixed;bottom:1.25rem;right:1.25rem;z-index:10000;width:52px;height:52px;border-radius:50%;background:var(--ai-widget-primary);color:var(--bs-white,#fff);border:none;cursor:pointer;display:flex;align-items:center;justify-content:center;font-size:1.5rem;box-shadow:0 4px 12px rgba(var(--bs-body-color-rgb,33,37,41),.2);transition:transform .2s,box-shadow .2s}.ai-chat-widget-toggle:hover{transform:scale(1.1);box-shadow:0 6px 16px rgba(var(--bs-body-color-rgb,33,37,41),.3)}.ai-chat-widget-container{position:fixed;bottom:5rem;right:1.25rem;z-index:10001;width:var(--ai-widget-width);height:var(--ai-widget-height);background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);border:1px solid var(--bs-border-color,#dee2e6);border-radius:12px;box-shadow:0 8px 32px rgba(var(--bs-body-color-rgb,33,37,41),.18);display:none;flex-direction:column;overflow:hidden}.ai-chat-widget-container.ai-chat-widget-resizable{resize:both;min-width:320px;min-height:420px;max-width:min(90vw,960px);max-height:min(90vh,960px)}.ai-chat-widget-container.open{display:flex}.ai-chat-widget-header{display:flex;align-items:center;justify-content:space-between;padding:.75rem 1rem;background:var(--ai-widget-primary);color:var(--bs-white,#fff)}.ai-chat-widget-header.ai-chat-widget-drag-handle{cursor:move;user-select:none;touch-action:none}.ai-chat-widget-header-actions{display:flex;align-items:center;gap:.15rem}.ai-chat-widget-toggle.ai-chat-widget-draggable{touch-action:none}.ai-chat-widget-container.ai-chat-widget-dragging,.ai-chat-widget-toggle.ai-chat-widget-dragging{transition:none}.ai-chat-widget-toggle.ai-chat-widget-dragging:hover{transform:none}.ai-chat-widget-header .title{font-weight:600;font-size:.95rem}.ai-chat-widget-header button{background:0 0;border:none;color:var(--bs-white,#fff);cursor:pointer;font-size:1rem;padding:0 .25rem;opacity:.8}.ai-chat-widget-header button:hover{opacity:1}.ai-chat-widget-profile-select{padding:.5rem .75rem;border-bottom:1px solid var(--bs-border-color,#dee2e6);background:var(--bs-tertiary-bg,#f8f9fa)}.ai-chat-widget-profile-select select{width:100%;padding:.35rem .5rem;color:var(--bs-body-color,#212529);background-color:var(--bs-body-bg,#fff);border:1px solid var(--bs-border-color,#ced4da);border-radius:6px;font-size:.85rem}.ai-chat-widget-messages{flex:1;overflow-y:auto;padding:.75rem;font-size:.9rem}.ai-chat-widget-messages .ai-chat-messages{padding:.5rem}.ai-chat-widget-messages .ai-chat-message-item{position:relative;padding:.5rem 0}.ai-chat-widget-messages .ai-chat-message-item+.ai-chat-message-item::before{content:'';display:block;width:100%;margin-bottom:.5rem;border-top:1px solid rgba(var(--bs-body-color-rgb,33,37,41),.08)}.ai-chat-widget-messages .ai-chat-message-heading{margin-bottom:.35rem}.ai-chat-widget-messages .ai-chat-message-heading .ai-chat-msg-role{margin-bottom:0;text-transform:uppercase;letter-spacing:.03em}.ai-chat-widget-messages .ai-chat-message-heading-assistant .ai-chat-msg-role{display:inline-flex;align-items:center;gap:.35rem}.ai-chat-widget-messages .ai-chat-message-role-label{display:inline-block;text-transform:inherit}.ai-chat-widget-messages .ai-chat-msg-role{font-weight:600;margin-bottom:2px}.ai-chat-widget-messages .ai-chat-msg-role-user{color:var(--bs-primary,#0d6efd)}.ai-chat-widget-messages .ai-chat-msg-role-assistant{color:var(--ai-widget-primary,#6f42c1)}.ai-chat-widget-messages .ai-bot-icon{color:currentColor;display:inline-block;animation:none}@keyframes ai-widget-streaming-effect{0%{opacity:1;transform:rotate(0)}10%{opacity:.3;transform:rotate(0)}20%{opacity:1;transform:rotate(0)}30%{opacity:.3;transform:rotate(0)}40%{opacity:1;transform:rotate(0)}50%{opacity:.3;transform:rotate(0)}60%{opacity:1;transform:rotate(0)}80%{opacity:1;transform:rotate(360deg)}100%{opacity:1;transform:rotate(360deg)}}.ai-chat-widget-messages .ai-streaming-icon{color:currentColor;display:inline-block;animation:ai-widget-streaming-effect 4s ease-in-out infinite}.ai-chat-widget-messages .ai-chat-message-body{position:relative;padding:0;overflow-wrap:break-word;word-break:break-word;min-width:0}.ai-chat-widget-messages .ai-chat-message-body p:last-child{margin-bottom:0}.ai-chat-widget-messages .message-buttons-container{position:absolute;right:.1rem;bottom:0;display:inline-flex;align-items:center;gap:.35rem;width:auto;max-width:max-content;opacity:0;pointer-events:none;transition:opacity .15s ease-in-out;background:rgba(var(--bs-body-bg-rgb,255,255,255),.96);padding:.2rem .35rem;border-radius:999px;box-shadow:0 1px 3px rgba(var(--bs-body-color-rgb,33,37,41),.08);margin-bottom:1rem;transform:translateY(50%);z-index:1}.ai-chat-widget-messages .ai-chat-message-item:focus-within .message-buttons-container,.ai-chat-widget-messages .ai-chat-message-item:hover .message-buttons-container{opacity:1;pointer-events:auto}.ai-chat-widget-messages .message-buttons-container:has(.tts-playing){opacity:1;pointer-events:auto}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox{display:inline-flex;align-items:center;justify-content:center;flex:0 0 auto;width:1.75rem;height:1.75rem;margin:0!important;padding:0!important;border-radius:50%;color:var(--bs-secondary-color,#6c757d)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container .ai-chat-message-assistant-feedback{display:inline-flex;align-items:center;gap:.35rem}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox:focus,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox:hover{color:var(--bs-body-color,#212529)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success:focus,.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success:hover{color:var(--bs-success,#198754)!important;text-decoration:none}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.text-success svg{color:inherit!important;fill:currentColor}.ai-chat-widget-messages .message-buttons-container>.button-message-toolbox.tts-playing{color:var(--ai-widget-primary,#6f42c1)!important;opacity:1;pointer-events:auto}.ai-chat-widget-history-panel{position:absolute;top:3.5rem;left:0;right:0;bottom:0;background:var(--bs-body-bg,#fff);padding:1rem;z-index:10;display:none;overflow-y:auto;flex-direction:column}.ai-chat-widget-history-panel.show{display:flex}.ai-chat-widget-history-panel .history-header{display:flex;justify-content:space-between;align-items:center;margin-bottom:.75rem;flex-shrink:0}.ai-chat-widget-history-panel .history-header h5{margin:0;font-size:1rem}.ai-chat-widget-history-list{list-style:none;padding:0;margin:0;overflow-y:auto;flex:1 1 auto}.ai-chat-widget-history-list li{margin-bottom:0}.ai-chat-widget-history-list .chat-session-history-item{display:block;padding:.5rem .75rem;text-decoration:none;color:var(--bs-body-color,#212529);font-size:.85rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:background .15s;cursor:pointer;border-bottom:1px solid var(--bs-border-color,#dee2e6)}.ai-chat-widget-history-list li:last-child .chat-session-history-item{border-bottom:none}.ai-chat-widget-history-list .chat-session-history-item:hover{background:var(--bs-tertiary-bg,#f8f9fa)}.ai-chat-widget-messages .ai-chat-notification{margin:.75rem .5rem 0;padding:.625rem .75rem;border:1px solid rgba(var(--bs-body-color-rgb,33,37,41),.12);border-radius:.5rem;background:rgba(var(--bs-tertiary-bg-rgb,248,249,250),.98)}.ai-chat-widget-messages .ai-chat-notification-content{display:flex;align-items:flex-start;gap:.5rem}.ai-chat-widget-messages .ai-chat-notification-icon{margin-top:.1rem;color:var(--bs-secondary-color,#6c757d)}.ai-chat-widget-messages .ai-chat-notification-text{flex:1 1 auto;min-width:0}.ai-chat-widget-messages .ai-chat-notification-dismiss{margin-left:auto!important;color:var(--bs-secondary-color,#6c757d)!important}.ai-chat-widget-messages .ai-chat-notification-actions{display:flex;gap:.5rem;margin-top:.5rem}.ai-chat-widget-messages .chart-container{width:100%;max-width:100%!important;min-height:280px}.ai-chat-widget-messages .ai-chat-message-body pre{background:var(--bs-dark-bg-subtle,#2d2d2d);color:var(--bs-emphasis-color,#f8f8f2);padding:.5rem;border-radius:4px;overflow-x:auto;font-size:.8rem}.ai-chat-widget-messages .ai-chat-message-body code{font-size:.85em}.ai-chat-widget-input{border-top:1px solid var(--bs-border-color,#dee2e6);padding:.5rem .75rem;display:flex;flex-wrap:wrap;gap:.5rem;align-items:flex-end}.ai-chat-widget-input .ai-chat-doc-bar{flex:0 0 100%;margin:-.5rem -.75rem .5rem;padding:.5rem .75rem;max-height:4.5rem;overflow-y:auto}.ai-chat-widget-input textarea{flex:1;min-width:0;color:var(--bs-body-color,#212529);background-color:var(--bs-body-bg,#fff);border:1px solid var(--bs-border-color,#ced4da);border-radius:8px;padding:.4rem .6rem;font-size:.9rem;resize:none;max-height:80px;min-height:36px;line-height:1.4;outline:0}.ai-chat-widget-input textarea:focus{border-color:var(--ai-widget-primary);box-shadow:0 0 0 2px rgba(var(--ai-widget-primary-rgb,108,117,125),.15)}.ai-chat-widget-input button{background:var(--ai-widget-primary);color:var(--bs-white,#fff);border:none;border-radius:8px;padding:.4rem .75rem;cursor:pointer;font-size:.9rem;white-space:nowrap}.ai-chat-widget-input .btn{background:var(--bs-body-bg,#fff);color:var(--bs-body-color,#212529);border:1px solid var(--bs-border-color,#ced4da)}.ai-chat-widget-input .btn.btn-link{background:0 0;color:var(--bs-secondary-color,#6c757d);border:none;padding:.4rem}.ai-chat-widget-input .btn.btn-outline-dark{background:var(--bs-dark,#212529);color:var(--bs-white,#fff);border-color:var(--bs-dark,#212529)}.ai-chat-widget-input .btn.btn-outline-secondary{color:var(--bs-secondary-color,#6c757d)}.ai-chat-widget-input button:disabled{opacity:.5;cursor:not-allowed}.ai-chat-widget-status{padding:.25rem .75rem;font-size:.8rem;color:var(--bs-secondary-color,#6c757d);display:none}.ai-chat-widget-status.visible{display:block}.ai-chat-widget-welcome{text-align:center;color:var(--bs-secondary-color,#6c757d);padding:2rem 1rem;font-size:.9rem}@media (max-width:576px){.ai-chat-widget-container{width:calc(100vw - 1rem);height:calc(100vh - 6rem);right:.5rem;bottom:4.5rem}} diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/ChatInteractions/Views/ChatInteraction/Chat.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/ChatInteractions/Views/ChatInteraction/Chat.cshtml index 63b539e7..0b22ed1a 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/ChatInteractions/Views/ChatInteraction/Chat.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/ChatInteractions/Views/ChatInteraction/Chat.cshtml @@ -571,6 +571,7 @@
0%
+
} @@ -1229,12 +1230,15 @@ function showUploadProgress(progress, message, cssClass) { var progressContainer = document.getElementById('chat-doc-upload-progress'); var progressBar = document.getElementById('chat-doc-upload-progress-bar'); - if (!progressContainer || !progressBar) { + var progressText = document.getElementById('chat-doc-upload-progress-text'); + if (!progressContainer || !progressBar || !progressText) { return; } if (progress === null || progress === undefined) { progressContainer.className = 'progress mt-2 d-none'; + progressText.className = 'small text-muted mt-1 d-none'; + progressText.textContent = ''; progressContainer.setAttribute('aria-valuenow', '0'); progressBar.className = 'progress-bar progress-bar-striped progress-bar-animated'; progressBar.style.width = '0%'; @@ -1244,10 +1248,18 @@ var roundedProgress = Math.max(0, Math.min(100, Math.round(progress))); progressContainer.className = 'progress mt-2'; + progressText.className = 'small text-muted mt-1'; progressContainer.setAttribute('aria-valuenow', roundedProgress.toString()); progressBar.className = 'progress-bar ' + (cssClass || 'progress-bar-striped progress-bar-animated'); progressBar.style.width = roundedProgress + '%'; - progressBar.textContent = message || (roundedProgress + '%'); + progressBar.textContent = roundedProgress + '%'; + progressText.textContent = message + ? (message + ' (' + roundedProgress + '%)') + : (roundedProgress + '%'); + + if (message) { + showUploadStatus(progressText.textContent, 'text-warning'); + } } function createUploadItem(file, index) { @@ -1285,6 +1297,11 @@ showUploadProgress(overallProgress, 'Uploading ' + (fileIndex + 1) + ' of ' + totalFiles, 'progress-bar-striped progress-bar-animated'); }); + xhr.upload.addEventListener('load', function () { + var overallProgress = ((fileIndex + 1) / totalFiles) * 100; + showUploadProgress(overallProgress, 'Upload complete. Processing on server...', 'progress-bar-striped progress-bar-animated'); + }); + xhr.addEventListener('load', function () { if (xhr.status >= 200 && xhr.status < 300) { try { diff --git a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularToolContextTests.cs b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularToolContextTests.cs index 443d69b0..55874c7b 100644 --- a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularToolContextTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularToolContextTests.cs @@ -2,11 +2,15 @@ using CrestApps.Core.AI.Documents; using CrestApps.Core.AI.Documents.Generation; using CrestApps.Core.AI.Documents.Models; +using CrestApps.Core.AI.Documents.Services; using CrestApps.Core.AI.Documents.Tabular; using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Orchestration; using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.DataIngestion; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; @@ -56,24 +60,244 @@ public async Task ResolveAsync_ExcludesGeneratedDocumentsFromWorkspaceSources() Assert.Equal("uploaded-1", document.DocumentId); } - private static ServiceProvider BuildServices(IAIDocumentStore documentStore) + [Fact] + public async Task LoadArtifactAsync_LoadsMissingArtifactFromStoredTabularFile() + { + var tempRoot = Path.Combine(Path.GetTempPath(), "tabular-tool-context-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + + try + { + var storedDocument = new AIDocument + { + ItemId = "uploaded-1", + ReferenceId = "interaction-1", + ReferenceType = AIReferenceTypes.Document.ChatInteraction, + FileName = "data.csv", + StoredFilePath = "documents/chat-interaction/interaction-1/data.csv", + ContentType = "text/csv", + }; + + var documentStore = new Mock(); + documentStore + .Setup(store => store.GetDocumentsAsync("interaction-1", AIReferenceTypes.Document.ChatInteraction)) + .ReturnsAsync([storedDocument]); + documentStore + .Setup(store => store.FindByIdAsync("uploaded-1", It.IsAny())) + .ReturnsAsync(storedDocument); + + var chunkStore = new Mock(MockBehavior.Strict); + var artifactStore = new Mock(); + artifactStore + .Setup(store => store.GetAsync("uploaded-1", It.IsAny())) + .ReturnsAsync((TabularDocumentArtifact)null); + artifactStore + .Setup(store => store.SaveAsync("uploaded-1", It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + var services = BuildServices(documentStore.Object, chunkStore.Object, artifactStore.Object, tempRoot); + var fileStore = services.GetRequiredService(); + await using (var stream = new MemoryStream("name,amount\nNorth,100\nSouth,200"u8.ToArray())) + { + await fileStore.SaveFileAsync(storedDocument.StoredFilePath, stream); + } + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolExecutionContext = new AIToolExecutionContext(new ChatInteraction + { + ItemId = "interaction-1", + }); + + var context = await TabularToolContext.ResolveAsync(services, TestContext.Current.CancellationToken); + + Assert.NotNull(context); + + var artifact = await context.LoadArtifactAsync(context.Documents[0], TestContext.Current.CancellationToken); + + Assert.Equal(["name", "amount"], artifact.Header); + Assert.Collection( + artifact.Rows, + row => Assert.Equal(["North", "100"], row), + row => Assert.Equal(["South", "200"], row)); + + chunkStore.Verify(store => store.GetChunksByAIDocumentIdAsync(It.IsAny()), Times.Never); + artifactStore.Verify( + store => store.SaveAsync("uploaded-1", It.IsAny(), It.IsAny()), + Times.Once); + } + finally + { + if (Directory.Exists(tempRoot)) + { + Directory.Delete(tempRoot, recursive: true); + } + } + } + + [Fact] + public async Task LoadArtifactAsync_XlsxFile_UsesSpreadsheetRowsWithoutChunkFallback() + { + var tempRoot = Path.Combine(Path.GetTempPath(), "tabular-tool-context-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + + try + { + var storedDocument = new AIDocument + { + ItemId = "uploaded-1", + ReferenceId = "interaction-1", + ReferenceType = AIReferenceTypes.Document.ChatInteraction, + FileName = "survey.xlsx", + StoredFilePath = "documents/chat-interaction/interaction-1/survey.xlsx", + ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + }; + + var documentStore = new Mock(); + documentStore + .Setup(store => store.GetDocumentsAsync("interaction-1", AIReferenceTypes.Document.ChatInteraction)) + .ReturnsAsync([storedDocument]); + documentStore + .Setup(store => store.FindByIdAsync("uploaded-1", It.IsAny())) + .ReturnsAsync(storedDocument); + + var chunkStore = new Mock(MockBehavior.Strict); + var artifactStore = new Mock(); + artifactStore + .Setup(store => store.GetAsync("uploaded-1", It.IsAny())) + .ReturnsAsync((TabularDocumentArtifact)null); + artifactStore + .Setup(store => store.SaveAsync("uploaded-1", It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + var services = BuildServices( + documentStore.Object, + chunkStore.Object, + artifactStore.Object, + tempRoot, + xlsxArtifactBuilder: new SpreadsheetLikeTabularDocumentArtifactBuilder()); + var fileStore = services.GetRequiredService(); + await using (var stream = new MemoryStream([1, 2, 3])) + { + await fileStore.SaveFileAsync(storedDocument.StoredFilePath, stream); + } + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolExecutionContext = new AIToolExecutionContext(new ChatInteraction + { + ItemId = "interaction-1", + }); + + var context = await TabularToolContext.ResolveAsync(services, TestContext.Current.CancellationToken); + + Assert.NotNull(context); + + var artifact = await context.LoadArtifactAsync(context.Documents[0], TestContext.Current.CancellationToken); + + Assert.Equal(["Name", "Amount"], artifact.Header); + Assert.Collection( + artifact.Rows, + row => Assert.Equal(["North", "100"], row), + row => Assert.Equal(["South", "200"], row)); + + chunkStore.Verify(store => store.GetChunksByAIDocumentIdAsync(It.IsAny()), Times.Never); + artifactStore.Verify( + store => store.SaveAsync("uploaded-1", It.IsAny(), It.IsAny()), + Times.Once); + } + finally + { + if (Directory.Exists(tempRoot)) + { + Directory.Delete(tempRoot, recursive: true); + } + } + } + + private static ServiceProvider BuildServices( + IAIDocumentStore documentStore, + IAIDocumentChunkStore chunkStore = null, + ITabularDocumentArtifactStore artifactStore = null, + string basePath = null, + IngestionDocumentReader xlsxReader = null, + ITabularDocumentArtifactBuilder xlsxArtifactBuilder = null) { var options = new ChatDocumentsOptions(); options.Add(".xlsx", embeddable: false, isTabular: true); + options.Add(".csv", embeddable: false, isTabular: true); var fileStoreOptions = new DocumentFileSystemFileStoreOptions { - BasePath = Path.Combine(Path.GetTempPath(), "tabular-tool-context-tests"), + BasePath = basePath ?? Path.Combine(Path.GetTempPath(), "tabular-tool-context-tests"), }; var services = new ServiceCollection(); services.AddSingleton(documentStore); - services.AddSingleton(new Mock().Object); - services.AddSingleton(new Mock().Object); + services.AddSingleton(chunkStore ?? new Mock().Object); + services.AddSingleton(artifactStore ?? new Mock().Object); services.AddSingleton>(Options.Create(options)); services.AddSingleton>(Options.Create(fileStoreOptions)); + services.AddSingleton(_ => new FileSystemFileStore(fileStoreOptions.BasePath)); + services.AddSingleton(); + services.AddKeyedSingleton( + ".csv", + (sp, _) => sp.GetRequiredService()); + if (xlsxReader != null) + { + services.AddSingleton(xlsxReader); + services.AddKeyedSingleton( + ".xlsx", + (_, _) => xlsxReader); + } + if (xlsxArtifactBuilder != null) + { + services.AddSingleton(xlsxArtifactBuilder); + services.AddKeyedSingleton( + ".xlsx", + (_, _) => xlsxArtifactBuilder); + } + services.AddScoped(); + services.AddSingleton>(NullLogger.Instance); return services.BuildServiceProvider(); } + + private sealed class SpreadsheetLikeIngestionDocumentReader : IngestionDocumentReader + { + public override Task ReadAsync( + Stream source, + string identifier, + string mediaType, + CancellationToken cancellationToken = default) + { + var document = new IngestionDocument(identifier); + var section = new IngestionDocumentSection(); + section.Elements.Add(new IngestionDocumentParagraph("Name\tAmount") { Text = "Name\tAmount" }); + section.Elements.Add(new IngestionDocumentParagraph("North\t100") { Text = "North\t100" }); + section.Elements.Add(new IngestionDocumentParagraph("South\t200") { Text = "South\t200" }); + document.Sections.Add(section); + + return Task.FromResult(document); + } + } + + private sealed class SpreadsheetLikeTabularDocumentArtifactBuilder : ITabularDocumentArtifactBuilder + { + public Task CreateAsync( + Stream source, + string fileName, + string contentType, + CancellationToken cancellationToken = default) + { + return Task.FromResult(new TabularDocumentArtifact + { + Header = ["Name", "Amount"], + Rows = + [ + ["North", "100"], + ["South", "200"], + ], + }); + } + } } diff --git a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceDocumentEventHandlerTests.cs b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceDocumentEventHandlerTests.cs index a13293ad..f10ed8dd 100644 --- a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceDocumentEventHandlerTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceDocumentEventHandlerTests.cs @@ -13,7 +13,7 @@ namespace CrestApps.Core.Tests.Core.Documents.Tabular; public sealed class TabularWorkspaceDocumentEventHandlerTests { [Fact] - public async Task UploadedAsync_TabularDocument_SavesArtifact() + public async Task UploadedAsync_TabularDocument_DoesNotSaveArtifact() { var artifactStore = new Mock(); var handler = CreateHandler(artifactStore); @@ -36,8 +36,8 @@ await handler.UploadedAsync(new AIChatDocumentUploadContext }, TestContext.Current.CancellationToken); artifactStore.Verify( - store => store.SaveAsync("doc-1", It.IsAny(), It.IsAny()), - Times.Once); + store => store.SaveAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); } [Fact] diff --git a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceTests.cs b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceTests.cs index d39fb71f..c002646f 100644 --- a/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Documents/Tabular/TabularWorkspaceTests.cs @@ -21,6 +21,36 @@ public async Task EnsureReadyAsync_LoadsTableWithSchema() Assert.Equal(["region", "amount"], table.Columns.Select(c => c.Name)); } + [Fact] + public async Task EnsureReadyAsync_DirectImporter_BypassesArtifactLoader() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var workspace = CreateWorkspace(); + var tables = await workspace.EnsureReadyAsync( + Documents(), + (_, _) => throw new Xunit.Sdk.XunitException("Artifact loader should not run when direct importer succeeds."), + (document, connection, tableName, _) => + { + var columns = TabularWorkspaceSqliteHelpers.BuildColumns(["region", "amount"]); + TabularWorkspaceSqliteHelpers.CreateTable(connection, tableName, columns); + + using var command = connection.CreateCommand(); + command.CommandText = $"INSERT INTO {TabularWorkspaceSqliteHelpers.QuoteIdentifier(tableName)} (\"region\", \"amount\") VALUES ('North', '100'), ('South', '200')"; + command.ExecuteNonQuery(); + + return Task.FromResult(new TabularWorkspaceImportResult(columns, 2, 1, 1)); + }, + cancellationToken); + + var table = Assert.Single(tables); + Assert.Equal(2, table.RowCount); + + var result = await workspace.QueryAsync("SELECT region, amount FROM sales ORDER BY region", 100, cancellationToken); + Assert.Equal(2, result.Rows.Count); + Assert.Equal("North", result.Rows[0][0]); + Assert.Equal("100", result.Rows[0][1]); + } + [Fact] public async Task QueryAsync_RunsAggregation() { diff --git a/tests/CrestApps.Core.Tests/Core/Documents/Tools/GetDocumentMetadataToolTests.cs b/tests/CrestApps.Core.Tests/Core/Documents/Tools/GetDocumentMetadataToolTests.cs new file mode 100644 index 00000000..7552bb30 --- /dev/null +++ b/tests/CrestApps.Core.Tests/Core/Documents/Tools/GetDocumentMetadataToolTests.cs @@ -0,0 +1,202 @@ +using CrestApps.Core.AI; +using CrestApps.Core.AI.Documents; +using CrestApps.Core.AI.Documents.Models; +using CrestApps.Core.AI.Documents.Tabular; +using CrestApps.Core.AI.Documents.Tools; +using CrestApps.Core.AI.Models; +using CrestApps.Core.AI.Orchestration; +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; + +namespace CrestApps.Core.Tests.Core.Documents.Tools; + +public sealed class GetDocumentMetadataToolTests +{ + [Fact] + public async Task InvokeAsync_HeadersScope_ReturnsTabularHeadersWithoutWorkspaceImport() + { + var storedDocument = new AIDocument + { + ItemId = "uploaded-1", + ReferenceId = "interaction-1", + ReferenceType = AIReferenceTypes.Document.ChatInteraction, + FileName = "SkyLineFull 1.xlsx", + ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + FileSize = 128, + }; + + var documentStore = new Mock(); + documentStore + .Setup(store => store.GetDocumentsAsync("interaction-1", AIReferenceTypes.Document.ChatInteraction)) + .ReturnsAsync([storedDocument]); + + var artifactStore = new Mock(); + artifactStore + .Setup(store => store.GetAsync("uploaded-1", It.IsAny())) + .ReturnsAsync(new TabularDocumentArtifact + { + Header = ["First Name", "Signup Date", "Is Active"], + Rows = + [ + ["Ada", "2026-07-01", "true"], + ["Grace", "2026-07-02", "false"], + ], + }); + + var services = BuildServices(documentStore.Object, artifactStore.Object); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolExecutionContext = new AIToolExecutionContext(new ChatInteraction + { + ItemId = "interaction-1", + }); + + var tool = new GetDocumentMetadataTool(); + var arguments = CreateArguments(services, new Dictionary + { + ["scope"] = "headers", + }); + + var result = await tool.InvokeAsync(arguments, TestContext.Current.CancellationToken); + var text = result.ToString(); + + Assert.Contains("\"SkyLineFull 1.xlsx\" has 3 headers.", text); + Assert.Contains("- First Name (inferred type: text)", text); + Assert.Contains("- Signup Date (inferred type: date)", text); + Assert.Contains("- Is Active (inferred type: boolean)", text); + } + + [Fact] + public async Task InvokeAsync_BasicScope_ReturnsNonTabularMetadata() + { + var storedDocument = new AIDocument + { + ItemId = "uploaded-1", + ReferenceId = "interaction-1", + ReferenceType = AIReferenceTypes.Document.ChatInteraction, + FileName = "notes.txt", + ContentType = "text/plain", + FileSize = 42, + }; + + var documentStore = new Mock(); + documentStore + .Setup(store => store.GetDocumentsAsync("interaction-1", AIReferenceTypes.Document.ChatInteraction)) + .ReturnsAsync([storedDocument]); + + var services = BuildServices(documentStore.Object, Mock.Of()); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolExecutionContext = new AIToolExecutionContext(new ChatInteraction + { + ItemId = "interaction-1", + }); + + var tool = new GetDocumentMetadataTool(); + var arguments = CreateArguments(services, new Dictionary + { + ["scope"] = "basic", + }); + + var result = await tool.InvokeAsync(arguments, TestContext.Current.CancellationToken); + var text = result.ToString(); + + Assert.Contains("\"notes.txt\" metadata:", text); + Assert.Contains("document_id: uploaded-1", text); + Assert.Contains("content_type: text/plain", text); + Assert.Contains("file_size_bytes: 42", text); + } + + [Fact] + public async Task InvokeAsync_ColumnsScope_ReturnsNormalizedColumnsWithInferredTypes() + { + var storedDocument = new AIDocument + { + ItemId = "uploaded-1", + ReferenceId = "interaction-1", + ReferenceType = AIReferenceTypes.Document.ChatInteraction, + FileName = "SkyLineFull 1.xlsx", + ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + FileSize = 128, + }; + + var documentStore = new Mock(); + documentStore + .Setup(store => store.GetDocumentsAsync("interaction-1", AIReferenceTypes.Document.ChatInteraction)) + .ReturnsAsync([storedDocument]); + + var artifactStore = new Mock(); + artifactStore + .Setup(store => store.GetAsync("uploaded-1", It.IsAny())) + .ReturnsAsync(new TabularDocumentArtifact + { + Header = ["Order ID", "Total Amount", "Updated At"], + Rows = + [ + ["1001", "10.25", "2026-07-06T14:48:16Z"], + ["1002", "11.50", "2026-07-06T15:00:00Z"], + ], + }); + + var services = BuildServices(documentStore.Object, artifactStore.Object); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolExecutionContext = new AIToolExecutionContext(new ChatInteraction + { + ItemId = "interaction-1", + }); + + var tool = new GetDocumentMetadataTool(); + var arguments = CreateArguments(services, new Dictionary + { + ["scope"] = "columns", + }); + + var result = await tool.InvokeAsync(arguments, TestContext.Current.CancellationToken); + var text = result.ToString(); + + Assert.Contains("- Order_ID (source header: Order ID) — inferred type: integer", text); + Assert.Contains("- Total_Amount (source header: Total Amount) — inferred type: decimal", text); + Assert.Contains("- Updated_At (source header: Updated At) — inferred type: datetime", text); + } + + private static AIFunctionArguments CreateArguments(IServiceProvider services, Dictionary values) + { + return new AIFunctionArguments(values) + { + Services = services, + }; + } + + private static ServiceProvider BuildServices( + IAIDocumentStore documentStore, + ITabularDocumentArtifactStore artifactStore) + { + var options = new ChatDocumentsOptions(); + options.Add(".xlsx", embeddable: false, isTabular: true); + options.Add(".txt"); + + var fileStoreOptions = new DocumentFileSystemFileStoreOptions + { + BasePath = Path.Combine(Path.GetTempPath(), "get-document-metadata-tool-tests", Guid.NewGuid().ToString("N")), + }; + + var services = new ServiceCollection(); + services.AddSingleton(documentStore); + services.AddSingleton(Mock.Of()); + services.AddSingleton(artifactStore); + services.AddSingleton>(Options.Create(options)); + services.AddSingleton>(Options.Create(fileStoreOptions)); + services.AddSingleton(_ => new FileSystemFileStore(fileStoreOptions.BasePath)); + services.AddScoped(); + services.AddSingleton>(NullLogger.Instance); + services.AddSingleton>(NullLogger.Instance); + + return services.BuildServiceProvider(); + } +} diff --git a/tests/CrestApps.Core.Tests/Core/Services/MultiSourceNamedCatalogTests.cs b/tests/CrestApps.Core.Tests/Core/Services/MultiSourceNamedCatalogTests.cs new file mode 100644 index 00000000..9d6def5a --- /dev/null +++ b/tests/CrestApps.Core.Tests/Core/Services/MultiSourceNamedCatalogTests.cs @@ -0,0 +1,110 @@ +using CrestApps.Core.AI.Models; +using CrestApps.Core.AI.Services; +using CrestApps.Core.Services; + +namespace CrestApps.Core.Tests.Core.Services; + +public sealed class MultiSourceNamedCatalogTests +{ + [Fact] + public async Task GetAllAsync_CachesMergedEntriesWithinStoreInstance() + { + // Arrange + var source = new CountingConnectionSource( + [ + new AIProviderConnection + { + ItemId = "connection-1", + Name = "winnerware-sys", + ClientName = "Azure", + Source = "Azure", + }, + ]); + var store = new DefaultAIProviderConnectionStore([source]); + + // Act + var first = await store.GetAllAsync(TestContext.Current.CancellationToken); + var second = await store.GetAllAsync(TestContext.Current.CancellationToken); + var lookup = await store.FindByNameAsync("winnerware-sys", TestContext.Current.CancellationToken); + + // Assert + Assert.Single(first); + Assert.Single(second); + Assert.NotNull(lookup); + Assert.Equal(1, source.ReadCount); + } + + [Fact] + public async Task CreateAsync_InvalidatesMergedEntryCache() + { + // Arrange + var source = new CountingConnectionSource( + [ + new AIProviderConnection + { + ItemId = "connection-1", + Name = "winnerware-sys", + ClientName = "Azure", + Source = "Azure", + }, + ]); + var store = new DefaultAIProviderConnectionStore([source]); + + _ = await store.GetAllAsync(TestContext.Current.CancellationToken); + + // Act + await store.CreateAsync(new AIProviderConnection + { + ItemId = "connection-2", + Name = "WinnerWare", + ClientName = "Azure", + Source = "Azure", + }, TestContext.Current.CancellationToken); + + var entries = await store.GetAllAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(2, source.ReadCount); + Assert.Equal(2, entries.Count); + } + + private sealed class CountingConnectionSource(List entries) : IWritableNamedSourceCatalogSource + { + public int ReadCount { get; private set; } + + public int Order => 0; + + public ValueTask> GetEntriesAsync( + IReadOnlyCollection knownEntries, + CancellationToken cancellationToken = default) + { + ReadCount++; + + return ValueTask.FromResult>(entries.ToArray()); + } + + public ValueTask CreateAsync(AIProviderConnection entry, CancellationToken cancellationToken = default) + { + entries.Add(entry); + + return ValueTask.CompletedTask; + } + + public ValueTask DeleteAsync(AIProviderConnection entry, CancellationToken cancellationToken = default) + { + return ValueTask.FromResult(entries.Remove(entry)); + } + + public ValueTask UpdateAsync(AIProviderConnection entry, CancellationToken cancellationToken = default) + { + var index = entries.FindIndex(existing => string.Equals(existing.ItemId, entry.ItemId, StringComparison.OrdinalIgnoreCase)); + + if (index >= 0) + { + entries[index] = entry; + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/CrestApps.Core.Tests/Helpers/DocumentReaders/OpenXmlTabularDocumentArtifactBuilderTests.cs b/tests/CrestApps.Core.Tests/Helpers/DocumentReaders/OpenXmlTabularDocumentArtifactBuilderTests.cs new file mode 100644 index 00000000..c6f7de8a --- /dev/null +++ b/tests/CrestApps.Core.Tests/Helpers/DocumentReaders/OpenXmlTabularDocumentArtifactBuilderTests.cs @@ -0,0 +1,301 @@ +using CrestApps.Core.AI.Documents.OpenXml.Services; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.Logging.Abstractions; + +namespace CrestApps.Core.Tests.Helpers.DocumentReaders; + +public sealed class OpenXmlTabularDocumentArtifactBuilderTests +{ + private readonly OpenXmlTabularDocumentArtifactBuilder _builder = new(NullLogger.Instance); + + [Fact] + public async Task CreateAsync_SharedStringsWorkbook_ExtractsHeaderAndRows() + { + await using var stream = CreateExcelWithSharedStrings([["Title", "Question", "Answer"], ["Thor Weapon", "What is Thor's weapon?", "Mjolnir"],]); + + var artifact = await _builder.CreateAsync( + stream, + "test.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + TestContext.Current.CancellationToken); + + Assert.Equal(["Title", "Question", "Answer"], artifact.Header); + Assert.Collection( + artifact.Rows, + row => Assert.Equal(["Thor Weapon", "What is Thor's weapon?", "Mjolnir"], row)); + } + + [Fact] + public async Task CreateAsync_SparseCellsWorkbook_PreservesColumnPositions() + { + await using var stream = CreateExcelWithSparseCells(); + + var artifact = await _builder.CreateAsync( + stream, + "test.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + TestContext.Current.CancellationToken); + + Assert.Equal(34, artifact.Header.Count); + Assert.Equal("Q3_C28/What fast food or quick service restaurants have you visited?", artifact.Header[33]); + Assert.Single(artifact.Rows); + Assert.Equal(34, artifact.Rows[0].Count); + Assert.Equal("1", artifact.Rows[0][33]); + } + + [Fact] + public async Task CreateAsync_BooleanWorkbook_ExtractsBooleanValues() + { + await using var stream = CreateExcelWithBooleans(true, false); + + var artifact = await _builder.CreateAsync( + stream, + "test.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + TestContext.Current.CancellationToken); + + Assert.Equal(["TRUE", "FALSE"], artifact.Header); + Assert.Empty(artifact.Rows); + } + + [Fact] + public async Task CreateAsync_MultipleWorksheets_SkipsSubsequentWorksheetHeaders() + { + await using var stream = CreateExcelWithMultipleSheets(); + + var artifact = await _builder.CreateAsync( + stream, + "test.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + TestContext.Current.CancellationToken); + + Assert.Equal(["Name", "Amount"], artifact.Header); + Assert.Collection( + artifact.Rows, + row => Assert.Equal(["North", "100"], row), + row => Assert.Equal(["South", "200"], row)); + } + + private static MemoryStream CreateExcelWithSharedStrings(string[][] rows) + { + var stream = new MemoryStream(); + using (var doc = SpreadsheetDocument.Create(stream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = doc.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + var allStrings = rows.SelectMany(r => r).Distinct().ToList(); + var sstPart = workbookPart.AddNewPart(); + var sst = new SharedStringTable(); + foreach (var s in allStrings) + { + sst.AppendChild(new SharedStringItem(new DocumentFormat.OpenXml.Spreadsheet.Text(s))); + } + + sstPart.SharedStringTable = sst; + var worksheetPart = workbookPart.AddNewPart(); + var sheetData = new SheetData(); + uint rowIndex = 1; + foreach (var rowData in rows) + { + var row = new Row + { + RowIndex = rowIndex, + }; + var colIndex = 0; + foreach (var cellValue in rowData) + { + var cellRef = $"{(char)('A' + colIndex)}{rowIndex}"; + var cell = new Cell + { + CellReference = cellRef, + DataType = CellValues.SharedString, + CellValue = new CellValue(allStrings.IndexOf(cellValue).ToString()), + }; + row.AppendChild(cell); + colIndex++; + } + + sheetData.AppendChild(row); + rowIndex++; + } + + worksheetPart.Worksheet = new Worksheet(sheetData); + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + sheets.AppendChild(new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Sheet1", + }); + } + + stream.Position = 0; + + return stream; + } + + private static MemoryStream CreateExcelWithSparseCells() + { + var stream = new MemoryStream(); + using (var doc = SpreadsheetDocument.Create(stream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = doc.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + var worksheetPart = workbookPart.AddNewPart(); + var sheetData = new SheetData(); + + var header = new Row { RowIndex = 1 }; + header.AppendChild(new Cell + { + CellReference = "A1", + DataType = CellValues.InlineString, + InlineString = new InlineString(new DocumentFormat.OpenXml.Spreadsheet.Text("Respondent")), + }); + header.AppendChild(new Cell + { + CellReference = "AH1", + DataType = CellValues.InlineString, + InlineString = new InlineString(new DocumentFormat.OpenXml.Spreadsheet.Text("Q3_C28/What fast food or quick service restaurants have you visited?")), + }); + sheetData.AppendChild(header); + + var row = new Row { RowIndex = 2 }; + row.AppendChild(new Cell + { + CellReference = "A2", + CellValue = new CellValue("1001"), + }); + row.AppendChild(new Cell + { + CellReference = "AH2", + CellValue = new CellValue("1"), + }); + sheetData.AppendChild(row); + + worksheetPart.Worksheet = new Worksheet(sheetData); + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + sheets.AppendChild(new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Sheet1", + }); + } + + stream.Position = 0; + + return stream; + } + + private static MemoryStream CreateExcelWithBooleans(params bool[] values) + { + var stream = new MemoryStream(); + using (var doc = SpreadsheetDocument.Create(stream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = doc.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + var worksheetPart = workbookPart.AddNewPart(); + var sheetData = new SheetData(); + var row = new Row + { + RowIndex = 1, + }; + + for (var i = 0; i < values.Length; i++) + { + row.AppendChild(new Cell + { + CellReference = $"{(char)('A' + i)}1", + DataType = CellValues.Boolean, + CellValue = new CellValue(values[i] ? "1" : "0"), + }); + } + + sheetData.AppendChild(row); + worksheetPart.Worksheet = new Worksheet(sheetData); + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + sheets.AppendChild(new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Sheet1", + }); + } + + stream.Position = 0; + + return stream; + } + + private static MemoryStream CreateExcelWithMultipleSheets() + { + var stream = new MemoryStream(); + using (var doc = SpreadsheetDocument.Create(stream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = doc.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + + AppendSheet( + workbookPart, + sheets, + 1, + [ + ["Name", "Amount"], + ["North", "100"], + ]); + AppendSheet( + workbookPart, + sheets, + 2, + [ + ["Name", "Amount"], + ["South", "200"], + ]); + } + + stream.Position = 0; + + return stream; + } + + private static void AppendSheet( + WorkbookPart workbookPart, + Sheets sheets, + uint sheetId, + string[][] rows) + { + var worksheetPart = workbookPart.AddNewPart(); + var sheetData = new SheetData(); + + for (var rowIndex = 0; rowIndex < rows.Length; rowIndex++) + { + var row = new Row + { + RowIndex = (uint)rowIndex + 1, + }; + + for (var columnIndex = 0; columnIndex < rows[rowIndex].Length; columnIndex++) + { + row.AppendChild(new Cell + { + CellReference = $"{(char)('A' + columnIndex)}{rowIndex + 1}", + DataType = CellValues.InlineString, + InlineString = new InlineString(new DocumentFormat.OpenXml.Spreadsheet.Text(rows[rowIndex][columnIndex])), + }); + } + + sheetData.AppendChild(row); + } + + worksheetPart.Worksheet = new Worksheet(sheetData); + sheets.AppendChild(new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = sheetId, + Name = $"Sheet{sheetId}", + }); + } +}