Fix Selectable Tools

src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/AIToolDefinitionEntry.cs

@@ -111,4 +111,13 @@ public bool HasPurpose(string purpose)
 
         return string.Equals(Purpose, purpose, StringComparison.OrdinalIgnoreCase);
     }
+
+    /// <summary>
+    /// Determines whether the tool is selectable in user-facing tool pickers.
+    /// </summary>
+    /// <returns><see langword="true"/> when the tool is neither a system tool nor hidden; otherwise <see langword="false"/>.</returns>
+    public bool IsSelectable()
+    {
+        return !IsSystemTool && !Hidden;
+    }
 }

src/Abstractions/CrestApps.Core.AI.Abstractions/Tooling/AIToolDefinitionOptions.cs

@@ -38,6 +38,17 @@ public IReadOnlyList<string> ExpandToolNames(IEnumerable<string> toolNames)
         return expandedToolNames;
     }
 
+    /// <summary>
+    /// Gets the registered tools that are selectable in user-facing tool pickers.
+    /// </summary>
+    /// <returns>A read-only dictionary of selectable AI tool definitions keyed by tool name.</returns>
+    public IReadOnlyDictionary<string, AIToolDefinitionEntry> GetSelectableTools()
+    {
+        return _tools
+            .Where(tool => tool.Value.IsSelectable())
+            .ToDictionary(tool => tool.Key, tool => tool.Value, StringComparer.OrdinalIgnoreCase);
+    }
+
     internal void SetTool(string name, AIToolDefinitionEntry entry)
     {
         _tools[name] = entry;

src/CrestApps.Core.Docs/docs/a2a/host.md

@@ -171,6 +171,8 @@ Each AI profile of type `Agent` becomes either:
 - An **independent Agent Card** (default behavior), or
 - A **skill on a combined Agent Card** (when `ExposeAgentsAsSkill` is `true`)
 
+This includes **code-defined system agents** contributed through `IAIProfileProvider`. For example, the built-in Tabular Data Agent is hidden from the MVC and Blazor agent pickers because it is always available and system-managed, but it is still published through the A2A host because the host reads the merged `IAIProfileManager.GetAsync(AIProfileType.Agent)` result.
+
 ### Agent Card Structure (A2A Protocol)
 
 A published Agent Card follows the A2A specification:
@@ -229,6 +231,8 @@ This approach is useful when:
 - The client's AI model should choose which skill to invoke based on the task
 - You want to simplify discovery for clients that don't need to manage multiple connections
 
+Because system agents are part of the same merged agent list, they also appear here automatically. That means a hidden agent can remain unavailable for manual UI selection while still being discoverable to remote A2A clients.
+
 ```csharp
 builder.Services.Configure<A2AHostOptions>(options =>
 {

src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md

@@ -81,6 +81,7 @@ description: Initial standalone release notes for the CrestApps.Core repository.
 - adds a defense-in-depth prompt security layer for AI Profile chat experiences with normalized regex-rule evaluation, weighted risk scoring, profile-level overrides, output filtering, audit logging, and documentation for remaining regex-based limitations
 - adds AI tool dependency registration through the fluent `AIToolBuilder`, automatically expands selected tool sets to include registered dependencies during profile/system tool resolution, ignores missing dependencies safely, and adds focused unit coverage for recursive, shared, and circular dependency graphs
 - replaces the always-on `read_tabular_data` system tool with an always-available, system **Tabular Data Agent** that loads uploaded non-embeddable files (such as CSV and Excel) lazily into an in-memory SQLite database and exposes `list_tabular_data`, `query_tabular_data`, `execute_tabular_command`, and `export_tabular_data` SQL tools, so the model analyzes, manipulates, and creates downloadable CSV versions of large tabular files through scoped SQL while only minimal results enter the prompt and the original uploaded file is always preserved; the agent's system prompt is sourced from the embedded `tabular-data-agent` AI template and its SQL tools are hidden from the user-facing tool picker
+- tightens the MVC, Blazor, and shared chat-settings selection flows so only selectable tools and user-selectable agents can be chosen or persisted from the UI, keeping hidden/system tools and framework-managed system agents such as the Tabular Data Agent out of manual pickers while still exposing those system agents through the A2A host
 - caches in-memory tabular databases per active chat scope instead of rebuilding them for every prompt: workspaces are keyed by chat interaction/session/profile document scope, reused while the user remains active, expired after a configurable sliding idle timeout (five minutes by default), cleaned by a hosted background service, and invalidated immediately when tabular documents or related chat interactions/sessions are removed; parsed tabular document artifacts are persisted through `ITabularDocumentArtifactStore` so another app instance can hydrate from shared document storage, and `ITabularWorkspaceInvalidationPublisher` provides the distributed backplane extension point for cross-instance cache clears; tabular files are identified through `ExtractorExtension.IsTabular` and `ChatDocumentsOptions.TabularFileExtensions` instead of a hardcoded extension list
 - improves tabular upload handling by storing raw tabular content chunks without embeddings, preserving sparse XLSX cell positions, using compact survey header codes such as `Q3_C28` as SQL column names while retaining the full source header, exposing typed `object[]` query rows, and adding document-availability guidance that routes row counts, summaries, and calculations to the Tabular Data Agent
 - ensures generated tabular-export files are always downloadable: `AICompletionReference.IsGenerated` flags tool-produced deliverables, the export tool sets it when it creates the CSV `AIDocument`, and the chat UI always renders generated references as a download even when the primary model omits the `[doc:n]` marker after delegating to the Tabular Data Agent
@@ -98,4 +99,5 @@ description: Initial standalone release notes for the CrestApps.Core repository.
 - lets `execute_tabular_command` apply multiple SQL statements in a single call: the tool now accepts one or more semicolon-separated data/schema statements, validates each one independently against the tabular SQL guard (respecting string literals, quoted identifiers, and comments so semicolons inside them never split a statement), and runs the whole batch in one transaction that rolls back together on failure, so the model makes every requested change in one tool call instead of many slow per-cell round-trips that previously hit the tool iteration limit on large files
 - stores AI-generated downloads under a dedicated `generated` subfolder inside each chat session/chat interaction document path, lets `FileSystemFileStore` open download streams with delete sharing so `ClearHistory` can remove generated files even after a user downloaded them, caches identical `export_tabular_data` calls within the same prompt, and rejects status-only `generate_file` calls after an existing export so the model no longer creates an extra bogus file that only says the download is ready
 - adds a dedicated hidden `fill_empty_tabular_cells` tool for the Tabular Data Agent so “replace every empty cell with X” requests run as one set-based update instead of the model composing hundreds of per-column statements, and broadens `generate_file` tabular misuse detection so conversational/question text like “Would you like me to generate…” cannot be written into `.xlsx` downloads
+- keeps hidden tools private to their owning profiles and agents across the shared MCP server handlers, so agent-only helpers such as the Tabular Data Agent SQL tools are no longer listed or callable as direct MCP tools
 - moves tabular workspace storage from in-memory SQLite to a file-based SQLite database stored alongside uploaded documents in a `data` folder, so workspace state persists across process restarts without artifact-store round-trips and reduces peak memory usage under high traffic; removes the singleton workspace cache, invalidation publisher interfaces, and cleanup background service in favor of creating a disposable workspace per tool call that opens and closes its own connection, with the document cleanup service and document event handler deleting the database file directly on session or document removal

src/CrestApps.Core.Docs/docs/core/agents.md

@@ -123,7 +123,7 @@ agent.Put(new AgentMetadata
 });
 ```
 
-When a tool-capable agent is invoked, `AgentProxyTool` runs it through the orchestrator so its configured tools are available. A recursion-depth guard (`AIInvocationContext.AgentInvocationDepth`) suppresses nested agents, so an agent can never invoke another agent — bounding recursion to a single level. This is how the system [Tabular Data Agent](./ai-documents.md#tabular-data-agent) runs its SQL tools.
+When a tool-capable agent is invoked, `AgentProxyTool` runs it through the orchestrator so its configured tools are available. A recursion-depth guard (`AIInvocationContext.AgentInvocationDepth`) suppresses nested agents, so an agent can never invoke another agent — bounding recursion to a single level. This is how the system [Tabular Data Agent](./ai-documents.md#tabular-files) runs its SQL tools.
 
 ### Code-defined profiles and system agents
 
@@ -151,6 +151,19 @@ internal sealed class MyAgentProvider : IAIProfileProvider
 
 Register it with `services.TryAddEnumerable(ServiceDescriptor.Scoped<IAIProfileProvider, MyAgentProvider>())`.
 
+### Pattern: hidden system agents that still participate in A2A
+
+The built-in **Tabular Data Agent** is the reference pattern for a framework-managed system agent:
+
+- it is defined in code through `IAIProfileProvider`
+- it is marked `AlwaysAvailable`
+- it sets `IsSystem = true`
+- it enables `AllowToolInvocation` so it can use its own hidden SQL tools
+- it stays out of the AI Profile and Chat Interaction pickers because system agents are not user-selectable
+- it is still returned by `IAIProfileManager.GetAsync(AIProfileType.Agent)`, so the A2A host exposes it like any other agent
+
+Use the same pattern for additional code-defined system agents when you want a capability to be automatically present for orchestration and remotely invocable over A2A without making it a manual UI choice.
+
 ## Creating Agent Profiles
 
 Agent profiles are standard `AIProfile` objects with `Type = AIProfileType.Agent`. They require a `Name` and `Description` at minimum — the description is what the primary model sees when deciding whether to invoke the agent.

src/CrestApps.Core.Docs/docs/core/ai-documents.md

@@ -59,6 +59,8 @@ This is the recommended path for tasks such as:
 - adding calculated columns
 - exporting an updated spreadsheet for download
 
+Under the hood, tabular workflows are handled by the built-in **Tabular Data Agent**. It is a code-defined, always-available **system agent** that stays hidden from the AI Profile and Chat Interaction agent pickers, yet still participates in orchestration and is exposed through the A2A host for remote clients.
+
 ### Images
 
 When your deployment supports vision, users can upload supported image files alongside standard documents. This enables image-aware chat scenarios such as describing screenshots, extracting visible text, or answering questions about diagrams and photos.

src/CrestApps.Core.Docs/docs/core/ai-profiles.md

@@ -96,6 +96,8 @@ This is why profiles are broader than plain chat presets. They can define how th
 
 If a selected tool has registered dependencies, CrestApps.Core automatically includes those dependent tools at runtime. That lets profiles keep only the top-level tool selection while helper tools remain hidden or system-managed.
 
+The MVC and Blazor editors only surface **selectable** tools and **user-selectable** agents here. Hidden tools, system tools, always-available agents, and system agents such as the Tabular Data Agent stay out of the picker and continue to be managed by the framework.
+
 ### 5. Knowledge and retrieval
 
 Profiles can be linked to:

src/CrestApps.Core.Docs/docs/core/tools.md

@@ -38,6 +38,7 @@ The tool builder pattern provides a fluent API for all of this.
 | Type | Registration | Visibility | Use Case |
 |------|-------------|-----------|----------|
 | **Selectable** | `.Selectable()` | Visible in UI for profile assignment | User-facing tools (calculator, search) |
+| **Hidden** | `.Hidden()` | Not shown in the picker or generic public exports; usable only when a profile or agent names it explicitly | Private helper tools for specialized agents |
 | **System** | Default (no `.Selectable()`) | Hidden, auto-included by orchestrator | Internal tools (RAG search, image gen) |
 
 ## Fluent Builder API
@@ -54,6 +55,7 @@ Returns an `AIToolBuilder<TTool>` for fluent configuration:
 | `.WithPurpose(string)` | Semantic purpose tag (see below) |
 | `.WithDependency(string)` / `.WithDependencies(params string[])` | Registers dependency tools that should be added automatically |
 | `.WithoutDependency(string)` / `.WithoutDependencies(params string[])` | Removes previously registered dependency tools |
+| `.Hidden()` | Makes the tool private to explicitly named profiles or agents and excludes it from picker-style/public export surfaces |
 | `.Selectable()` | Makes the tool visible in UI and assignable to profiles |
 
 ### Tool Purposes

src/CrestApps.Core.Docs/docs/mcp/server.md

@@ -304,7 +304,7 @@ When your application acts as an MCP server, registered AI tools are exposed to
 1. **List tools** — Returns metadata (name, description, JSON schema) for all registered tools
 2. **Call tool** — Resolves the tool by name from the registry and invokes it with the provided arguments
 
-Tools registered via `AddCoreAITool<T>()` (see [Custom Tools](../core/tools.md)) are automatically available to MCP clients.
+Tools registered via `AddCoreAITool<T>()` (see [Custom Tools](../core/tools.md)) are automatically available to MCP clients unless they are marked with `.Hidden()`. Hidden tools remain available to explicitly configured profiles and agents, but the shared MCP handlers do not list or invoke them directly.
 
 ## Server Metadata
 

src/Primitives/CrestApps.Core.AI.Chat/Hubs/ChatInteractionHubBase.cs

@@ -6,8 +6,10 @@
 using CrestApps.Core.AI.Deployments;
 using CrestApps.Core.AI.Models;
 using CrestApps.Core.AI.Orchestration;
+using CrestApps.Core.AI.Profiles;
 using CrestApps.Core.AI.ResponseHandling;
 using CrestApps.Core.AI.Services;
+using CrestApps.Core.AI.Tooling;
 using CrestApps.Core.Services;
 using Cysharp.Text;
 using Microsoft.AspNetCore.SignalR;
@@ -352,6 +354,17 @@ protected virtual Task ApplyCoreSettingsAsync(
         IServiceProvider services,
         ChatInteraction interaction,
         JsonElement settings)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentNullException.ThrowIfNull(interaction);
+
+        return ApplyCoreSettingsInternalAsync(services, interaction, settings);
+    }
+
+    private static async Task ApplyCoreSettingsInternalAsync(
+        IServiceProvider services,
+        ChatInteraction interaction,
+        JsonElement settings)
     {
         interaction.Title = JsonHelper.GetString(settings, "title") ?? "Untitled";
         interaction.OrchestratorName = JsonHelper.GetString(settings, "orchestratorName");
@@ -366,12 +379,31 @@ protected virtual Task ApplyCoreSettingsAsync(
         interaction.PresencePenalty = JsonHelper.GetFloat(settings, "presencePenalty");
         interaction.MaxTokens = JsonHelper.GetInt(settings, "maxTokens");
         interaction.PastMessagesCount = JsonHelper.GetInt(settings, "pastMessagesCount");
-        interaction.ToolNames = JsonHelper.GetStringArray(settings, "toolNames");
-        interaction.AgentNames = JsonHelper.GetStringArray(settings, "agentNames");
+
+        var selectableToolNames = services
+            .GetRequiredService<IOptions<AIToolDefinitionOptions>>()
+            .Value
+            .GetSelectableTools()
+            .Keys
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        interaction.ToolNames = (JsonHelper.GetStringArray(settings, "toolNames") ?? [])
+            .Where(name => !string.IsNullOrWhiteSpace(name) && selectableToolNames.Contains(name))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        var allAgents = await services.GetRequiredService<IAIProfileManager>().GetAsync(AIProfileType.Agent);
+        var userSelectableAgentNames = (allAgents ?? [])
+            .Where(profile => profile.IsUserSelectableAgent())
+            .Select(profile => profile.Name)
+            .Where(name => !string.IsNullOrWhiteSpace(name))
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        interaction.AgentNames = (JsonHelper.GetStringArray(settings, "agentNames") ?? [])
+            .Where(name => !string.IsNullOrWhiteSpace(name) && userSelectableAgentNames.Contains(name))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
         interaction.McpConnectionIds = JsonHelper.GetStringArray(settings, "mcpConnectionIds");
         interaction.A2AConnectionIds = JsonHelper.GetStringArray(settings, "a2aConnectionIds");
-
-        return Task.CompletedTask;
     }
 
     /// <summary>

src/Primitives/CrestApps.Core.AI.Mcp/McpServerBuilderExtensions.cs

@@ -33,7 +33,7 @@ public static IMcpServerBuilder WithCrestAppsHandlers(this IMcpServerBuilder bui
                         ILogger logger = null;
                         var tools = new List<Tool>();
 
-                        foreach (var (name, _) in toolDefinitions.Tools)
+                        foreach (var (name, _) in toolDefinitions.Tools.Where(tool => !tool.Value.Hidden))
                         {
                             try
                             {
@@ -73,7 +73,8 @@ public static IMcpServerBuilder WithCrestAppsHandlers(this IMcpServerBuilder bui
                     {
                         var toolDefinitions = request.Services.GetRequiredService<IOptions<AIToolDefinitionOptions>>().Value;
 
-                        if (toolDefinitions.Tools.ContainsKey(request.Params.Name))
+                        if (toolDefinitions.Tools.TryGetValue(request.Params.Name, out var definition) &&
+                            !definition.Hidden)
                         {
                             if (request.Services.GetKeyedService<AITool>(request.Params.Name) is not AIFunction aiFunction)
                             {

src/Primitives/CrestApps.Core.AI/Extensions/AIAgentProfileExtensions.cs

@@ -34,15 +34,16 @@ public static bool IsSystemAgent(this AIProfile profile)
 
     /// <summary>
     /// Determines whether the agent profile should appear in the user-facing agent selection
-    /// list. Always-available (system) agents are excluded because they are included
-    /// automatically and never need to be selected manually.
+    /// list. Always-available agents and system agents are excluded because they are
+    /// included automatically or managed in code and never need to be selected manually.
     /// </summary>
     /// <param name="profile">The agent profile.</param>
     /// <returns><see langword="true"/> when the agent is user-selectable; otherwise <see langword="false"/>.</returns>
     public static bool IsUserSelectableAgent(this AIProfile profile)
     {
         return profile is not null
             && !string.IsNullOrEmpty(profile.Description)
-            && !profile.IsAlwaysAvailableAgent();
+            && !profile.IsAlwaysAvailableAgent()
+            && !profile.IsSystemAgent();
     }
 }