Add More Elasticsearch Authentication Type

src/Abstractions/CrestApps.Core.AI.Abstractions/Models/ElasticsearchSourceMetadata.cs

@@ -5,6 +5,16 @@ namespace CrestApps.Core.AI.Models;
 /// </summary>
 public sealed class ElasticsearchSourceMetadata
 {
+    /// <summary>
+    /// The self-managed Elasticsearch environment type value.
+    /// </summary>
+    public const string SelfManagedEnvironmentType = "SelfManaged";
+
+    /// <summary>
+    /// The Elastic Cloud hosted environment type value.
+    /// </summary>
+    public const string CloudHostedEnvironmentType = "CloudHosted";
+
     /// <summary>
     /// The anonymous Elasticsearch authentication type value.
     /// </summary>
@@ -15,11 +25,36 @@ public sealed class ElasticsearchSourceMetadata
     /// </summary>
     public const string BasicAuthenticationType = "Basic";
 
+    /// <summary>
+    /// The Elasticsearch API key authentication type value.
+    /// </summary>
+    public const string ApiKeyAuthenticationType = "ApiKey";
+
+    /// <summary>
+    /// The Elasticsearch base64-encoded API key authentication type value.
+    /// </summary>
+    public const string Base64ApiKeyAuthenticationType = "Base64ApiKey";
+
+    /// <summary>
+    /// The Elasticsearch key identifier plus API key authentication type value.
+    /// </summary>
+    public const string KeyIdAndKeyAuthenticationType = "KeyIdAndKey";
+
+    /// <summary>
+    /// Gets or sets the Elasticsearch environment type.
+    /// </summary>
+    public string EnvironmentType { get; set; }
+
     /// <summary>
     /// Gets or sets the Elasticsearch endpoint URL.
     /// </summary>
     public string Url { get; set; }
 
+    /// <summary>
+    /// Gets or sets the Elastic Cloud deployment identifier.
+    /// </summary>
+    public string CloudId { get; set; }
+
     /// <summary>
     /// Gets or sets the authentication type.
     /// </summary>
@@ -40,25 +75,97 @@ public sealed class ElasticsearchSourceMetadata
     /// </summary>
     public string Password { get; set; }
 
+    /// <summary>
+    /// Gets or sets the optional protected Elasticsearch API key value.
+    /// </summary>
+    public string ApiKey { get; set; }
+
+    /// <summary>
+    /// Gets or sets the optional protected base64-encoded Elasticsearch API key value.
+    /// </summary>
+    public string Base64ApiKey { get; set; }
+
+    /// <summary>
+    /// Gets or sets the optional Elasticsearch API key identifier.
+    /// </summary>
+    public string ApiKeyId { get; set; }
+
     /// <summary>
     /// Gets or sets the optional TLS certificate fingerprint.
     /// </summary>
     public string CertificateFingerprint { get; set; }
 
+    /// <summary>
+    /// Gets the normalized environment type.
+    /// </summary>
+    public string GetEnvironmentType()
+    {
+        if (string.IsNullOrWhiteSpace(EnvironmentType))
+        {
+            return string.IsNullOrWhiteSpace(CloudId)
+                ? SelfManagedEnvironmentType
+                : CloudHostedEnvironmentType;
+        }
+
+        var environmentType = EnvironmentType.Trim();
+
+        if (string.Equals(environmentType, CloudHostedEnvironmentType, StringComparison.OrdinalIgnoreCase))
+        {
+            return CloudHostedEnvironmentType;
+        }
+
+        return SelfManagedEnvironmentType;
+    }
+
     /// <summary>
     /// Gets the normalized authentication type.
     /// </summary>
     public string GetAuthenticationType()
     {
         if (string.IsNullOrWhiteSpace(AuthenticationType))
         {
-            return string.IsNullOrWhiteSpace(Username) && string.IsNullOrWhiteSpace(Password)
-                ? NoneAuthenticationType
-                : BasicAuthenticationType;
+            if (!string.IsNullOrWhiteSpace(Username) || !string.IsNullOrWhiteSpace(Password))
+            {
+                return BasicAuthenticationType;
+            }
+
+            if (!string.IsNullOrWhiteSpace(ApiKeyId) || !string.IsNullOrWhiteSpace(ApiKey))
+            {
+                return string.IsNullOrWhiteSpace(ApiKeyId)
+                    ? ApiKeyAuthenticationType
+                    : KeyIdAndKeyAuthenticationType;
+            }
+
+            if (!string.IsNullOrWhiteSpace(Base64ApiKey))
+            {
+                return Base64ApiKeyAuthenticationType;
+            }
+
+            return NoneAuthenticationType;
+        }
+
+        var authenticationType = AuthenticationType.Trim();
+
+        if (string.Equals(authenticationType, BasicAuthenticationType, StringComparison.OrdinalIgnoreCase))
+        {
+            return BasicAuthenticationType;
+        }
+
+        if (string.Equals(authenticationType, ApiKeyAuthenticationType, StringComparison.OrdinalIgnoreCase))
+        {
+            return ApiKeyAuthenticationType;
+        }
+
+        if (string.Equals(authenticationType, Base64ApiKeyAuthenticationType, StringComparison.OrdinalIgnoreCase))
+        {
+            return Base64ApiKeyAuthenticationType;
+        }
+
+        if (string.Equals(authenticationType, KeyIdAndKeyAuthenticationType, StringComparison.OrdinalIgnoreCase))
+        {
+            return KeyIdAndKeyAuthenticationType;
         }
 
-        return string.Equals(AuthenticationType.Trim(), BasicAuthenticationType, StringComparison.OrdinalIgnoreCase)
-            ? BasicAuthenticationType
-            : NoneAuthenticationType;
+        return NoneAuthenticationType;
     }
 }

src/Abstractions/CrestApps.Core.Infrastructure.Abstractions/Indexing/IndexProfileSourceDescriptor.cs

@@ -1,5 +1,3 @@
-using Microsoft.Extensions.Localization;
-
 namespace CrestApps.Core.Infrastructure.Indexing;
 
 /// <summary>
@@ -22,15 +20,15 @@ public sealed class IndexProfileSourceDescriptor
     /// <summary>
     /// Gets or sets the human-readable display name of the search provider.
     /// </summary>
-    public LocalizedString ProviderDisplayName { get; set; }
+    public string ProviderDisplayName { get; set; }
 
     /// <summary>
     /// Gets or sets the human-readable display name shown in the UI for this source descriptor.
     /// </summary>
-    public LocalizedString DisplayName { get; set; }
+    public string DisplayName { get; set; }
 
     /// <summary>
     /// Gets or sets a short description of this source descriptor shown in the UI.
     /// </summary>
-    public LocalizedString Description { get; set; }
+    public string Description { get; set; }
 }

src/Abstractions/CrestApps.Core.Infrastructure.Abstractions/Indexing/IndexProfileSourceOptions.cs

@@ -14,6 +14,22 @@ public sealed class IndexProfileSourceOptions
     /// </summary>
     public List<IndexProfileSourceDescriptor> Sources { get; } = [];
 
+    /// <summary>
+    /// Adds or update.
+    /// </summary>
+    /// <param name="providerName">The provider name.</param>
+    /// <param name="providerDisplayName">The provider display name.</param>
+    /// <param name="type">The type.</param>
+    /// <param name="configure">The action used to configure.</param>
+    public void AddOrUpdate(
+        string providerName,
+        string providerDisplayName,
+        string type,
+        Action<IndexProfileSourceDescriptor> configure = null)
+    {
+        AddOrUpdate(providerName, new LocalizedString(providerDisplayName, providerDisplayName), type, configure);
+    }
+
     /// <summary>
     /// Adds or update.
     /// </summary>
@@ -38,10 +54,10 @@ public void AddOrUpdate(
         descriptor ??= new IndexProfileSourceDescriptor
         {
             ProviderName = providerName,
-            ProviderDisplayName = providerDisplayName,
+            ProviderDisplayName = providerDisplayName.Value,
             Type = type,
-            DisplayName = new LocalizedString(type, type),
-            Description = new LocalizedString(type, type),
+            DisplayName = type,
+            Description = type,
         };
 
         configure?.Invoke(descriptor);

src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md

@@ -77,9 +77,11 @@ description: Initial standalone release notes for the CrestApps.Core repository.
 - caps the total uploaded vision-image bytes loaded into a single multimodal request through `ChatDocumentsOptions.MaxVisionInputBytesPerRequest`, removes the extra `MemoryStream` copy when attaching those images, and documents how to resolve a vision-capable chat client for direct image-description requests
 - adds the standalone `CrestApps.Core.AI.Resilience` package with opt-in Microsoft.Extensions.AI builder resilience extensions for chat, embeddings, image generation, speech-to-text, and text-to-speech clients, including `UseDefaultResilience()` for provider `429 Too Many Requests` retries and `UseResilience(...)` for custom Polly/Microsoft resilience pipelines; the docs now include a dedicated AI Resilience page, the default retry schedule uses exponential backoff with jitter (about 1-2, 2-4, 4-8, 8-16, and 16-32 seconds across five retries), framework-owned completion clients and utility-deployment chat flows apply the default retry policy automatically, host-created clients remain opt-in, builder examples require `Build(serviceProvider)` instead of `Build(null)`, and Azure OpenAI exposes shared SDK retry settings through `CrestApps:AI:AzureClient` with matching five-retry exponential defaults
 - adds `IAIClientFactory` overloads that accept builder-configuration delegates for chat, embeddings, image generation, speech-to-text, and text-to-speech clients, so callers can apply middleware such as `UseDefaultResilience()` while the factory owns the final `Build(serviceProvider)` step
+- expands Elasticsearch AI data source authentication beyond Basic by adding Elastic Cloud ID support plus `ApiKey`, `Base64ApiKey`, and `KeyIdAndKey` modes with protected per-source secrets in the MVC and Blazor editors and the shared Elasticsearch client factory
 - adds `CrestApps.Core.PostgreSQL` and `CrestApps.Core.AI.PostgreSQL` packages providing a lightweight PostgreSQL + pgvector vector search backend as an alternative to Elasticsearch and Azure AI Search, registers the same keyed services (`ISearchIndexManager`, `ISearchDocumentManager`, `IDataSourceContentManager`, `IDataSourceDocumentReader`, `IODataFilterTranslator`) under the `"PostgreSQL"` provider name, supports `AddAIDocuments()`, `AddAIDataSources()`, and `AddAIMemory()` builder extensions, and integrates into both MVC and Blazor sample hosts
 - fixes hosted document and data-source indexing flows so background workers create a scoped service provider before resolving scoped indexing services, preventing upload-triggered failures and similar nightly alignment lifetime issues
 - standardizes Azure AI Search configuration on top-level `AuthenticationType`, `ApiKey`, `IdentityClientId`, and `IndexPrefix` settings under `CrestApps:AzureAISearch`, and refreshes the sample host / docs examples to list the full supported option set in one place
+- adds an explicit Elasticsearch data-source environment selector (`SelfManaged` vs `CloudHosted`) so the MVC and Blazor editors show either `Url` or `CloudId` as appropriate and validation now requires the matching field for the chosen environment
 - replaces per-turn raw image byte injection with an analyze-once-at-upload strategy: `IImageAnalysisService` calls a vision model to extract caption, OCR text, and detected entities when images are uploaded, stores the results as `AIDocumentChunk` records searchable via `read_document` and `search_documents`, adds `inspect_image` as an on-demand tool for pixel-level inspection when the text analysis is insufficient, removes `BuildVisionUserContentsAsync` from `DocumentOrchestrationHandler` so image bytes are never attached to every user message, and updates the document-availability prompt to guide the model toward text-based tools first
 - adds a defense-in-depth prompt security layer for AI Profile chat experiences with normalized regex-rule evaluation, weighted risk scoring, profile-level overrides, output filtering, audit logging, and documentation for remaining regex-based limitations
 - adds AI tool dependency registration through the fluent `AIToolBuilder`, automatically expands selected tool sets to include registered dependencies during profile/system tool resolution, ignores missing dependencies safely, and adds focused unit coverage for recursive, shared, and circular dependency graphs

src/CrestApps.Core.Docs/docs/data-sources/elasticsearch.md

@@ -32,8 +32,13 @@ builder.Services.AddCoreElasticsearchServices();
     "Search": {
       "Elasticsearch": {
         "Url": "https://localhost:9200",
+        "CloudId": "",
+        "AuthenticationType": "Basic",
         "Username": "elastic",
         "Password": "your-password",
+        "ApiKey": "",
+        "Base64ApiKey": "",
+        "ApiKeyId": "",
         "CertificateFingerprint": "AA:BB:CC:..."
       }
     }
@@ -46,8 +51,13 @@ builder.Services.AddCoreElasticsearchServices();
 | Property | Type | Description |
 |----------|------|-------------|
 | `Url` | `string` | Elasticsearch endpoint URL |
+| `CloudId` | `string` | Elastic Cloud deployment identifier (optional alternative to `Url`) |
+| `AuthenticationType` | `string` | `None`, `Basic`, `ApiKey`, `Base64ApiKey`, or `KeyIdAndKey` |
 | `Username` | `string` | Basic auth username (optional) |
 | `Password` | `string` | Basic auth password (optional) |
+| `ApiKey` | `string` | Raw API key value for `ApiKey` auth, or the key portion for `KeyIdAndKey` |
+| `Base64ApiKey` | `string` | Base64-encoded API key value for `Base64ApiKey` auth |
+| `ApiKeyId` | `string` | API key identifier for `KeyIdAndKey` auth |
 | `CertificateFingerprint` | `string` | TLS certificate fingerprint for verification (optional) |
 
 ## Services Registered (Keyed by `"Elasticsearch"`)
@@ -81,12 +91,19 @@ Override `IAIDataSourceIndexingQueue` when you need a durable or distributed que
 When an `AIDataSource` uses `SourceType = "Elasticsearch"`, the mapping reads documents from a remote Elasticsearch index using source-specific settings stored on the `AIDataSource` itself:
 
 - `Url`
-- `AuthenticationType` (`None` or `Basic`)
+- `CloudId`
+- `EnvironmentType` (`SelfManaged` or `CloudHosted`)
+- `AuthenticationType` (`None`, `Basic`, `ApiKey`, `Base64ApiKey`, or `KeyIdAndKey`)
 - `IndexName`
 - `Username` (when `AuthenticationType = "Basic"`)
 - `Password` (protected at rest when `AuthenticationType = "Basic"`)
+- `ApiKey` (protected at rest when `AuthenticationType = "ApiKey"` or `KeyIdAndKey`)
+- `Base64ApiKey` (protected at rest when `AuthenticationType = "Base64ApiKey"`)
+- `ApiKeyId` (when `AuthenticationType = "KeyIdAndKey"`)
 - `CertificateFingerprint`
 
+Use `EnvironmentType = "SelfManaged"` with `Url` for self-managed clusters or endpoint-based hosted deployments. Use `EnvironmentType = "CloudHosted"` with `CloudId` for Elastic Cloud hosted deployments. Older records that only store `CloudId` are still inferred as cloud-hosted. Elastic Cloud connections require one of the authenticated modes.
+
 This is different from the Elasticsearch knowledge-base backend configuration. The backend settings under `CrestApps:Search:Elasticsearch` define where the embedded knowledge-base chunks are written. The source mapping settings define where the raw source documents are read from.
 
 Because the remote source index is externally managed, document changes must be pushed into the sync pipeline through `IAIDataSourceChangeNotifier`. See [Custom Sources](./custom-sources.md) for the notification pattern.
@@ -152,8 +169,13 @@ Then configure your `appsettings.Development.json`:
     "Search": {
       "Elasticsearch": {
         "Url": "https://my-cluster.es.us-east-1.aws.found.io:9243",
+        "CloudId": "",
+        "AuthenticationType": "Basic",
         "Username": "elastic",
         "Password": "your-secure-password",
+        "ApiKey": "",
+        "Base64ApiKey": "",
+        "ApiKeyId": "",
         "CertificateFingerprint": "AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99"
       }
     }
@@ -165,9 +187,14 @@ Then configure your `appsettings.Development.json`:
 
 | Property | Type | Required | Default | Description |
 |----------|------|----------|---------|-------------|
-| `Url` | `string` | Yes | — | Elasticsearch endpoint URL. Include the port if non-standard (e.g., `https://localhost:9200`). |
+| `Url` | `string` | Conditionally | — | Elasticsearch endpoint URL. Include the port if non-standard (e.g., `https://localhost:9200`). Required when `CloudId` is empty. |
+| `CloudId` | `string` | Conditionally | — | Elastic Cloud deployment identifier. Required when `Url` is empty. |
+| `AuthenticationType` | `string` | No | `None` | Selects `None`, `Basic`, `ApiKey`, `Base64ApiKey`, or `KeyIdAndKey`. Elastic Cloud requires one of the authenticated modes. |
 | `Username` | `string` | No | — | Username for basic authentication. Typically `"elastic"` for the built-in superuser. |
 | `Password` | `string` | No | — | Password for basic authentication. |
+| `ApiKey` | `string` | No | — | Raw API key for `ApiKey` auth, or the key portion for `KeyIdAndKey`. |
+| `Base64ApiKey` | `string` | No | — | Base64-encoded API key value for `Base64ApiKey` auth. |
+| `ApiKeyId` | `string` | No | — | API key identifier for `KeyIdAndKey` auth. |
 | `CertificateFingerprint` | `string` | No | — | SHA-256 fingerprint of the Elasticsearch TLS certificate. Required when using self-signed certificates. Format: `AA:BB:CC:...` |
 
 :::info
@@ -250,10 +277,10 @@ Deleting an index removes all indexed documents permanently. Re-indexing from th
 
 **Error:** `Elasticsearch.Net.ElasticsearchClientException: 401 Unauthorized`
 
-**Cause:** Invalid username or password.
+**Cause:** Invalid credentials for the selected authentication type.
 
 **Fix:**
-- Verify credentials in `appsettings.json`
+- Verify the selected `AuthenticationType` and its matching credentials in `appsettings.json`
 - Reset the elastic user password: `docker exec -it elasticsearch bin/elasticsearch-reset-password -u elastic`
 
 ### Certificate Error

src/Primitives/CrestApps.Core.AI.Azure.AISearch/ServiceCollectionExtensions.cs

@@ -32,8 +32,8 @@ public static IServiceCollection AddCoreAzureAISearchAIDocumentSource(this IServ
 
         return services.AddCoreAzureAISearchSource(IndexProfileTypes.AIDocuments, descriptor =>
                 {
-                    descriptor.DisplayName = new LocalizedString("AI Documents", "AI Documents");
-                    descriptor.Description = new LocalizedString("Azure AI Search AI Documents Description", "Create an Azure AI Search index for uploaded and embedded AI document chunks.");
+                    descriptor.DisplayName = "AI Documents";
+                    descriptor.Description = "Create an Azure AI Search index for uploaded and embedded AI document chunks.";
                 }).AddCoreAIDocumentIndexProfileHandler();
     }
 
@@ -47,8 +47,8 @@ public static IServiceCollection AddCoreAzureAISearchAIDataSource(this IServiceC
 
         return services.AddCoreAzureAISearchSource(IndexProfileTypes.DataSource, descriptor =>
                 {
-                    descriptor.DisplayName = new LocalizedString("Data Source", "Data Source");
-                    descriptor.Description = new LocalizedString("Azure AI Search Data Source Description", "Create an Azure AI Search index for AI knowledge base data source documents.");
+                    descriptor.DisplayName = "Data Source";
+                    descriptor.Description = "Create an Azure AI Search index for AI knowledge base data source documents.";
                 }).AddCoreAIDataSourceRag()
                 .Configure<AIDataSourceSourceOptions>(options => options.AddOrUpdate(
                     AIDataSourceSourceTypes.AzureAISearch,
@@ -74,8 +74,8 @@ public static IServiceCollection AddCoreAzureAISearchAIMemorySource(this IServic
 
         return services.AddCoreAzureAISearchSource(IndexProfileTypes.AIMemory, descriptor =>
                 {
-                    descriptor.DisplayName = new LocalizedString("AI Memory", "AI Memory");
-                    descriptor.Description = new LocalizedString("Azure AI Search AI Memory Description", "Create an Azure AI Search index for user and system memory records.");
+                    descriptor.DisplayName = "AI Memory";
+                    descriptor.Description = "Create an Azure AI Search index for user and system memory records.";
                 }).AddCoreAIMemoryIndexProfileHandler();
     }