diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 04a83544d0b71..447008ec848ec 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -905,6 +905,7 @@ x-pack/platform/plugins/private/global_search_providers @elastic/appex-sharedux x-pack/platform/plugins/private/graph @elastic/kibana-visualizations x-pack/platform/plugins/private/grokdebugger @elastic/kibana-management x-pack/platform/plugins/private/index_lifecycle_management @elastic/kibana-management +x-pack/platform/plugins/private/indices_metadata @elastic/security-solution x-pack/platform/plugins/private/intercepts @elastic/appex-sharedux x-pack/platform/plugins/private/license_api_guard @elastic/kibana-management x-pack/platform/plugins/private/logstash @elastic/logstash diff --git a/docs/extend/plugin-list.md b/docs/extend/plugin-list.md index 820de855f7e46..aee25a482204c 100644 --- a/docs/extend/plugin-list.md +++ b/docs/extend/plugin-list.md @@ -151,6 +151,7 @@ mapped_pages: | [grokdebugger](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/grokdebugger/README.md) | This plugin helps users define Grok patterns, which are particularly useful for ingesting logs. | | [indexLifecycleManagement](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/index_lifecycle_management/README.md) | You can test that the Frozen badge, phase filtering, and lifecycle information is surfaced in Index Management by running this series of requests in Console: | | [indexManagement](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/index_management/README.md) | This service is exposed from the Index Management setup contract and can be used to add content to the indices list and the index details page. | +| [indicesMetadata](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/private/indices_metadata/README.md) | Plugin for managing and retrieving metadata about indices in Kibana. This plugin collects and processes metadata from Elasticsearch indices, data streams, ILM policies, and index templates. | | [inference](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/inference/README.md) | The inference plugin is a central place to handle all interactions with the Elasticsearch Inference API and external LLM APIs. Its goals are: | | [inferenceEndpoint](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/inference_endpoint/README.md) | A Kibana plugin | | [infra](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/infra/README.md) | This is the home of the infra plugin, which aims to provide a solution for the infrastructure monitoring use-case within Kibana. | diff --git a/package.json b/package.json index 88b1b178f71d5..3176b4f277dbd 100644 --- a/package.json +++ b/package.json @@ -598,6 +598,7 @@ "@kbn/index-management-plugin": "link:x-pack/platform/plugins/shared/index_management", "@kbn/index-management-shared-types": "link:x-pack/platform/packages/shared/index-management/index_management_shared_types", "@kbn/index-patterns-test-plugin": "link:src/platform/test/plugin_functional/plugins/index_patterns", + "@kbn/indices-metadata-plugin": "link:x-pack/platform/plugins/private/indices_metadata", "@kbn/inference-common": "link:x-pack/platform/packages/shared/ai-infra/inference-common", "@kbn/inference-endpoint-plugin": "link:x-pack/platform/plugins/shared/inference_endpoint", "@kbn/inference-endpoint-ui-common": "link:x-pack/platform/packages/shared/kbn-inference-endpoint-ui-common", diff --git a/tsconfig.base.json b/tsconfig.base.json index 3654cdb450570..105add7a49c98 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1106,6 +1106,8 @@ "@kbn/index-management-shared-types/*": ["x-pack/platform/packages/shared/index-management/index_management_shared_types/*"], "@kbn/index-patterns-test-plugin": ["src/platform/test/plugin_functional/plugins/index_patterns"], "@kbn/index-patterns-test-plugin/*": ["src/platform/test/plugin_functional/plugins/index_patterns/*"], + "@kbn/indices-metadata-plugin": ["x-pack/platform/plugins/private/indices_metadata"], + "@kbn/indices-metadata-plugin/*": ["x-pack/platform/plugins/private/indices_metadata/*"], "@kbn/inference-cli": ["x-pack/platform/packages/shared/kbn-inference-cli"], "@kbn/inference-cli/*": ["x-pack/platform/packages/shared/kbn-inference-cli/*"], "@kbn/inference-common": ["x-pack/platform/packages/shared/ai-infra/inference-common"], @@ -2324,4 +2326,4 @@ "@kbn/ambient-storybook-types" ] } -} +} \ No newline at end of file diff --git a/x-pack/platform/plugins/private/indices_metadata/README.md b/x-pack/platform/plugins/private/indices_metadata/README.md new file mode 100644 index 0000000000000..150a6056e7b99 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/README.md @@ -0,0 +1,66 @@ +# Indices Metadata Plugin + +Plugin for managing and retrieving metadata about indices in +Kibana. This plugin collects and processes metadata from Elasticsearch indices, +data streams, ILM policies, and index templates. + +## Overview + +The Indices Metadata plugin is owned by the Security Solution team and provides: + +1. **Automated metadata collection** — Runs scheduled tasks to gather indices + metadata +2. **Analytics integration** — Sends telemetry events for indices statistics + and configurations +3. **Data stream monitoring** — Tracks data streams and their associated + metadata + +## Features + +- **Index Statistics Collection**: Gathers comprehensive statistics about indices +- **Data Stream Monitoring**: Tracks data streams and their metadata +- **ILM Policy Tracking**: Monitors Index Lifecycle Management policies and statistics +- **Index Template Management**: Collects information about index templates +- **Settings Monitoring**: Retrieves and tracks index settings +- **Task Scheduling**: Runs collection tasks every 24 hours via Task Manager + +## Configuration + +The plugin's configuration prefix is `xpack.indicesMetadata` + +### Plugin Configuration Options + +- `enabled`: Whether the plugin is enabled (default: `true`) +- `cdn.url` — URL for artifact downloads +- `cdn.publicKey` — Public key for artifact verification +- `cdn.requestTimeout` — Timeout for CDN requests + +### Configuration Example + +`kibana.yml` + +```yaml +xpack.indicesMetadata: + enabled: false + cdn: + url: 'https://artifacts.elastic.co' + publicKey: '...' + requestTimeout: 30000 +``` + +## Dependencies + +- **Required Plugins**: `taskManager` +- **Owner**: `@elastic/security-solution` + +## Development + +This plugin is server-side only and depends on: + +- Task Manager for scheduled metadata collection +- Analytics service for telemetry +- Elasticsearch client for metadata retrieval + +See the [Kibana contributing +guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for +instructions setting up your development environment. \ No newline at end of file diff --git a/x-pack/platform/plugins/private/indices_metadata/jest.config.js b/x-pack/platform/plugins/private/indices_metadata/jest.config.js new file mode 100644 index 0000000000000..41c22b3315dce --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/jest.config.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: ['/x-pack/platform/plugins/private/indices_metadata'], + coverageDirectory: + '/target/kibana-coverage/jest/x-pack/platform/plugins/private/indices_metadata', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/platform/plugins/private/indices_metadata/{common,public,server}/**/*.{ts,tsx}', + ], + globals: { + Uint8Array: Uint8Array, + }, +}; diff --git a/x-pack/platform/plugins/private/indices_metadata/kibana.jsonc b/x-pack/platform/plugins/private/indices_metadata/kibana.jsonc new file mode 100644 index 0000000000000..46266091eb4a3 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/kibana.jsonc @@ -0,0 +1,25 @@ +{ + "type": "plugin", + "id": "@kbn/indices-metadata-plugin", + "owner": [ + "@elastic/security-solution" + ], + "group": "platform", + "visibility": "private", + "description": "Indices metadata plugin for managing and retrieving metadata about indices in Kibana.", + "plugin": { + "id": "indicesMetadata", + "server": true, + "browser": false, + "configPath": [ + "xpack", + "indicesMetadata" + ], + "requiredPlugins": [ + "taskManager" + ], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/config.ts b/x-pack/platform/plugins/private/indices_metadata/server/config.ts new file mode 100644 index 0000000000000..b2fa94c057528 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/config.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginConfigDescriptor } from '@kbn/core/server'; + +const pluginConfigSchema = schema.object({ + enabled: schema.boolean({ defaultValue: true }), + cdn: schema.maybe( + schema.object({ + url: schema.maybe(schema.string()), + publicKey: schema.maybe(schema.string()), + requestTimeout: schema.maybe(schema.number()), + }) + ), +}); + +export type PluginConfig = TypeOf; + +export const config: PluginConfigDescriptor = { + schema: pluginConfigSchema, +}; diff --git a/x-pack/platform/plugins/private/indices_metadata/server/index.ts b/x-pack/platform/plugins/private/indices_metadata/server/index.ts new file mode 100644 index 0000000000000..02dbdda7fc6a6 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '@kbn/core/server'; +import { IndicesMetadataPlugin } from './plugin'; + +export { config } from './config'; +export async function plugin(context: PluginInitializerContext) { + return new IndicesMetadataPlugin(context); +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/constants.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/constants.ts new file mode 100644 index 0000000000000..7a32d47208cb5 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/constants.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesMetadataConfiguration } from './services/indices_metadata.types'; +import type { CdnConfig } from './services/artifact.types'; + +export const DEFAULT_CDN_CONFIG: CdnConfig = { + url: 'https://artifacts.security.elastic.co', + pubKey: ` +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA6AB2sJ5M1ImN/bkQ7Te6 +uI7vMXjN2yupEmh2rYz4gNzWS351d4JOhuQH3nzxfKdayHgusP/Kq2MXVqALH8Ru +Yu2AF08GdvYlQXPgEVI+tB/riekwU7PXZHdA1dY5/mEZ8SUSM25kcDJ3vTCzFTlL +gl2RNAdkR80d9nhvNSWlhWMwr8coQkr6NmujVU/Wa0w0EXbN1arjcG4qzbOCaR+b +cgQ9LRUoFfK9w+JJHDNjOI7rOmaIDA6Ep4oeDLy5AcGCE8bNmQzxZhRW7NvlNUGS +NTgU0CZTatVsL9AyP15W3k635Cpmy2SMPX+d/CFgvr8QPxtqdrz3q9iOeU3a1LMY +gDcFVmSzn5zieQEPfo/FcQID/gnCmkX0ADVMf1Q20ew66H7UCOejGaerbFZXYnTz +5AgQBWF2taOSSE7gDjGAHereeKp+1PR+tCkoDZIrPEjo0V6+KaTMuYS3oZj1/RZN +oTjQrdfeDj02mEIL+XkcWKAp03PYlWylVwgTMa178DDVuTWtS5lZL8j5LijlH9+6 +xH8o++ghwfxp6ENLKDZPV5IvHHG7Vth9HScoPTQWQ+s8Bt26QENPUV2AbyxbJykY +mJfTDke3bEemHZzRbAmwiQ7VpJjJ4OfLGRy8Pp2AHo8kYIvWyM5+aLMxcxUaYdA9 +5SxoDOgcDBA4lLb6XFLYiDUCAwEAAQ== +-----END PUBLIC KEY----- +`, + requestTimeout: 10_000, +}; + +export const DEFAULT_INDICES_METADATA_CONFIGURATION: IndicesMetadataConfiguration = { + datastreams_threshold: 0, + ilm_policy_query_size: 0, + ilm_stats_query_size: 0, + index_query_size: 0, + indices_settings_threshold: 0, + indices_threshold: 0, +}; diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts new file mode 100644 index 0000000000000..f09863c13ff69 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts @@ -0,0 +1,454 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EventTypeOpts } from '@elastic/ebt/client'; +import type { AnalyticsServiceSetup } from '@kbn/core/server'; +import type { + DataStreams, + IlmPolicies, + IlmsStats, + IndexTemplatesStats, + IndicesSettings, + IndicesStats, +} from '../services/indices_metadata.types'; + +export const DATA_STREAM_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-data-stream-event', + schema: { + items: { + type: 'array', + items: { + properties: { + datastream_name: { + type: 'keyword', + _meta: { description: 'Name of the data stream' }, + }, + ilm_policy: { + type: 'keyword', + _meta: { optional: true, description: 'ILM policy associated to the datastream' }, + }, + template: { + type: 'keyword', + _meta: { optional: true, description: 'Template associated to the datastream' }, + }, + indices: { + type: 'array', + items: { + properties: { + index_name: { type: 'date', _meta: { description: 'Index name' } }, + ilm_policy: { type: 'date', _meta: { optional: true, description: 'ILM policy' } }, + }, + }, + _meta: { optional: true, description: 'Indices associated with the data stream' }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const INDEX_STATS_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-index-stats-event', + schema: { + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index being monitored.' }, + }, + query_total: { + type: 'long', + _meta: { + optional: true, + description: 'The total number of search queries executed on the index.', + }, + }, + query_time_in_millis: { + type: 'long', + _meta: { + optional: true, + description: + 'The total time spent on query execution across all search requests, measured in milliseconds.', + }, + }, + + docs_count_primaries: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents currently stored in the index (primary shards).', + }, + }, + docs_deleted_primaries: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents that have been marked as deleted in the index (primary shards).', + }, + }, + docs_total_size_in_bytes_primaries: { + type: 'long', + _meta: { + optional: true, + description: + 'The total size, in bytes, of all documents stored in the index, including storage overhead (primary shards).', + }, + }, + docs_count: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents currently stored in the index (primary and replica shards).', + }, + }, + docs_deleted: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents that have been marked as deleted in the index (primary and replica shards).', + }, + }, + docs_total_size_in_bytes: { + type: 'long', + _meta: { + optional: true, + description: + 'The total size, in bytes, of all documents stored in the index, including storage overhead (primary and replica shards).', + }, + }, + + index_failed: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents failed to index (primary and replica shards).', + }, + }, + index_failed_due_to_version_conflict: { + type: 'long', + _meta: { + optional: true, + description: + 'The total number of documents failed to index due to version conflict (primary and replica shards).', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const ILM_STATS_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-ilm-stats-event', + schema: { + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index currently managed by the ILM policy.' }, + }, + phase: { + type: 'keyword', + _meta: { + optional: true, + description: + 'The current phase of the ILM policy that the index is in (e.g., hot, warm, cold, frozen, or delete).', + }, + }, + age: { + type: 'text', + _meta: { + optional: true, + description: + 'The age of the index since its creation, indicating how long it has existed.', + }, + }, + policy_name: { + type: 'keyword', + _meta: { + optional: true, + description: 'The name of the ILM policy applied to this index.', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const INDEX_SETTINGS_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-index-settings-event', + schema: { + items: { + type: 'array', + items: { + properties: { + index_name: { + type: 'keyword', + _meta: { description: 'The name of the index.' }, + }, + index_mode: { + type: 'keyword', + _meta: { optional: true, description: 'Index mode.' }, + }, + source_mode: { + type: 'keyword', + _meta: { optional: true, description: 'Source mode.' }, + }, + default_pipeline: { + type: 'keyword', + _meta: { + optional: true, + description: 'Pipeline applied if no pipeline parameter specified when indexing.', + }, + }, + final_pipeline: { + type: 'keyword', + _meta: { + optional: true, + description: + 'Pipeline applied to the document at the end of the indexing process, after the document has been indexed.', + }, + }, + }, + }, + _meta: { description: 'Index settings' }, + }, + }, +}; + +export const ILM_POLICY_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-ilm-policy-event', + schema: { + items: { + type: 'array', + items: { + properties: { + policy_name: { + type: 'keyword', + _meta: { description: 'The name of the ILM policy.' }, + }, + modified_date: { + type: 'date', + _meta: { description: 'The date when the ILM policy was last modified.' }, + }, + phases: { + properties: { + cold: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "cold" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "cold" phase of the ILM policy, applied when data is infrequently accessed.', + }, + }, + delete: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "delete" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "delete" phase of the ILM policy, specifying when the index should be removed.', + }, + }, + frozen: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "frozen" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "frozen" phase of the ILM policy, where data is fully searchable but stored with a reduced resource footprint.', + }, + }, + hot: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "hot" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "hot" phase of the ILM policy, applied to actively written and queried data.', + }, + }, + warm: { + properties: { + min_age: { + type: 'text', + _meta: { + description: + 'The minimum age before the index transitions to the "warm" phase.', + }, + }, + }, + _meta: { + optional: true, + description: + 'Configuration settings for the "warm" phase of the ILM policy, used for read-only data that is less frequently accessed.', + }, + }, + }, + _meta: { + description: + 'The different phases of the ILM policy that define how the index is managed over time.', + }, + }, + }, + }, + _meta: { description: 'Datastreams' }, + }, + }, +}; + +export const INDEX_TEMPLATES_EVENT: EventTypeOpts = { + eventType: 'indices-metadata-index-templates-event', + schema: { + items: { + type: 'array', + items: { + properties: { + template_name: { + type: 'keyword', + _meta: { description: 'The name of the template.' }, + }, + index_mode: { + type: 'keyword', + _meta: { + optional: true, + description: 'The index mode.', + }, + }, + datastream: { + type: 'boolean', + _meta: { + description: 'Datastream dataset', + }, + }, + package_name: { + type: 'keyword', + _meta: { + optional: true, + description: 'The package name', + }, + }, + managed_by: { + type: 'keyword', + _meta: { + optional: true, + description: 'Managed by', + }, + }, + beat: { + type: 'keyword', + _meta: { + optional: true, + description: 'Shipper name', + }, + }, + is_managed: { + type: 'boolean', + _meta: { + optional: true, + description: 'Whether the template is managed', + }, + }, + composed_of: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: 'List of template components', + }, + }, + _meta: { description: '' }, + }, + source_enabled: { + type: 'boolean', + _meta: { + optional: true, + description: + 'The _source field contains the original JSON document body that was provided at index time', + }, + }, + source_includes: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: 'Fields included in _source, if enabled', + }, + }, + _meta: { description: '' }, + }, + source_excludes: { + type: 'array', + items: { + type: 'keyword', + _meta: { + description: '', + }, + }, + _meta: { description: 'Fields excludes from _source, if enabled' }, + }, + }, + }, + _meta: { description: 'Index templates info' }, + }, + }, +}; + +export const registerEbtEvents = (analytics: AnalyticsServiceSetup) => { + const events: Array> = [ + DATA_STREAM_EVENT, + INDEX_STATS_EVENT, + ILM_STATS_EVENT, + ILM_POLICY_EVENT, + INDEX_TEMPLATES_EVENT, + INDEX_SETTINGS_EVENT, + ]; + + events.forEach((eventConfig) => analytics.registerEventType(eventConfig)); +}; diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.errors.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.errors.ts new file mode 100644 index 0000000000000..58d7d26fd6d5d --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.errors.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable max-classes-per-file */ + +/** + * Error thrown when a requested artifact is not found in the manifest + */ +export class ArtifactNotFoundError extends Error { + constructor(artifactName: string) { + super(`No artifact for name ${artifactName}`); + this.name = 'ArtifactNotFoundError'; + Object.setPrototypeOf(this, ArtifactNotFoundError.prototype); + } +} + +/** + * Error thrown when the manifest file is not found in the CDN + */ +export class ManifestNotFoundError extends Error { + constructor(manifestUrl: string) { + super(`No manifest resource found at url: ${manifestUrl}`); + this.name = 'ManifestNotFoundError'; + Object.setPrototypeOf(this, ManifestNotFoundError.prototype); + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.test.ts new file mode 100644 index 0000000000000..3c5fc23f83f3f --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.test.ts @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import axios from 'axios'; +import { ArtifactNotFoundError, ManifestNotFoundError } from './artifact.errors'; +import { generateKeyPairSync, createSign } from 'crypto'; +import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; +import AdmZip from 'adm-zip'; +import { ArtifactService } from './artifact'; + +jest.mock('axios'); + +describe('ArtifactService', () => { + const url = 'http://localhost:3000'; + const requestTimeout = 10_000; + const mockedAxios = axios as jest.Mocked; + const logger = loggingSystemMock.createLogger(); + const defaultClusterInfo: InfoResponse = { + name: 'elasticsearch', + cluster_name: 'elasticsearch', + cluster_uuid: 'fiNVFADnQsepL3HXYMs-qg', + version: { + number: '9.2.0-SNAPSHOT', + build_flavor: 'default', + build_type: 'tar', + build_hash: '560464e544b7e37e581874f44c19c7eac930f901', + build_date: '2025-07-08T02:09:11.988781060Z', + build_snapshot: true, + lucene_version: '10.2.2', + minimum_wire_compatibility_version: '8.19.0', + minimum_index_compatibility_version: '8.0.0', + }, + tagline: 'You Know, for Search', + }; + const artifactName = 'telemetry-buffer-and-batch-sizes-v1'; + + let privKey: string; + let pubKey: string; + + beforeAll(() => { + ({ publicKey: pubKey, privateKey: privKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { type: 'spki', format: 'pem' }, + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + })); + }); + + beforeEach(() => { + mockedAxios.get.mockReset(); + }); + + it('should fail when manifest is not found', async () => { + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(), { + url, + pubKey, + requestTimeout, + }); + + mockedAxios.get.mockImplementationOnce(() => Promise.resolve({ status: 404 })); + + await expect(artifactService.getArtifact(artifactName)).rejects.toThrow(ManifestNotFoundError); + }); + + it('should construct manifest URL by removing -SNAPSHOT suffix from version number', async () => { + const version = '9.2.0'; + const artifactService = new ArtifactService( + logger, + createClusterInfoWithVersion(`${version}-SNAPSHOT`), + { url, pubKey, requestTimeout } + ); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + setupMockResponses(zip.toBuffer()); + + const result = await artifactService.getArtifact(artifactName); + expect(result).toBeDefined(); + expect(mockedAxios.get.mock.calls[0][0]).toBe( + `${url}/downloads/kibana/manifest/artifacts-${version}.zip` + ); + }); + + it('should use exact version number in manifest URL for non-snapshot versions', async () => { + const version = '9.1.1'; + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(version), { + url, + pubKey, + requestTimeout, + }); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + setupMockResponses(zip.toBuffer()); + + const result = await artifactService.getArtifact(artifactName); + expect(result).toBeDefined(); + expect(mockedAxios.get.mock.calls[0][0]).toBe( + `${url}/downloads/kibana/manifest/artifacts-${version}.zip` + ); + }); + + it('should throw an error when requesting an artifact that does not exist in the manifest', async () => { + const invalidArtifactName = 'invalid-artifact-name'; + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(), { + url, + pubKey, + requestTimeout, + }); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + setupMockResponses(zip.toBuffer()); + + await expect(artifactService.getArtifact(invalidArtifactName)).rejects.toThrow( + ArtifactNotFoundError + ); + }); + + it('should retrieve and return artifact content when the artifact exists in the manifest', async () => { + const content = 'artifact content'; + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(), { + url, + pubKey, + requestTimeout, + }); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + setupMockResponses(zip.toBuffer(), content); + + const result = await artifactService.getArtifact(artifactName); + expect(result).toBeDefined(); + expect(result.data as string).toBe(content); + }); + + it('should cache manifest and use If-None-Match header for subsequent requests to avoid redundant downloads', async () => { + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(), { + url, + pubKey, + requestTimeout, + }); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + const fakeEtag = '123'; + const axiosResponse = { + status: 200, + data: zip.toBuffer(), + headers: { etag: fakeEtag }, + }; + + // first request: download the .zip, second request: get the artifact, third request: check if the artifact is modified + // and since the status is 304, it shouldn't download the artifact again. + mockedAxios.get + .mockImplementationOnce(() => Promise.resolve(axiosResponse)) + .mockImplementationOnce(() => Promise.resolve({ status: 200, data: {} })) + .mockImplementationOnce(() => Promise.resolve({ status: 304 })); + + let manifest = await artifactService.getArtifact(artifactName); + expect(manifest).not.toBeFalsy(); + expect(manifest.modified).toEqual(true); + expect(mockedAxios.get.mock.calls.length).toBe(2); + + manifest = await artifactService.getArtifact(artifactName); + expect(manifest).not.toBeFalsy(); + expect(manifest.modified).toEqual(false); + expect(mockedAxios.get.mock.calls.length).toBe(3); + + const [_url, config] = mockedAxios.get.mock.calls[2]; + const headers = config?.headers ?? {}; + expect(headers).not.toBeFalsy(); + expect(headers['If-None-Match']).toEqual(fakeEtag); + }); + + it('should throw an error when manifest signature verification fails with mismatched public key', async () => { + const { publicKey: altPubKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { type: 'spki', format: 'pem' }, + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + }); + + const version = '9.2.0'; + const artifactService = new ArtifactService(logger, createClusterInfoWithVersion(version), { + url, + pubKey: altPubKey, + requestTimeout, + }); + + const zip = createManifestZipPackage( + JSON.stringify({ + artifacts: { + [artifactName]: { + relative_url: '/downloads/artifacts/telemetry-buffer-and-batch-sizes-v1.json', + }, + }, + }) + ); + + setupMockResponses(zip.toBuffer()); + + await expect(artifactService.getArtifact(artifactName)).rejects.toThrowError( + 'Invalid manifest signature' + ); + }); + + function createClusterInfoWithVersion(version: string = '9.2.0'): InfoResponse { + return { + ...defaultClusterInfo, + version: { + ...defaultClusterInfo.version, + number: version, + }, + }; + } + + function setupMockResponses(manifestZipContent: Buffer, artifactContent: string = '') { + mockedAxios.get + .mockImplementationOnce(() => { + return Promise.resolve({ + status: 200, + data: manifestZipContent, + headers: {}, + config: { responseType: 'arraybuffer' }, + }); + }) + .mockImplementationOnce(() => { + return Promise.resolve({ + status: 200, + data: artifactContent, + headers: {}, + }); + }); + } + + function signManifestContent(manifestJson: string): Buffer { + const sign = createSign('RSA-SHA256'); + sign.update(manifestJson); + sign.end(); + return sign.sign(privKey); + } + + function createManifestZipPackage(manifestJson: string): AdmZip { + const zip = new AdmZip(); + zip.addFile('manifest.json', Buffer.from(manifestJson)); + zip.addFile('manifest.sig', signManifestContent(manifestJson)); + return zip; + } +}); diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.ts new file mode 100644 index 0000000000000..f3acb276976f9 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, LogMeta } from '@kbn/core/server'; +import { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; +import axios from 'axios'; +import { createVerify } from 'crypto'; +import AdmZip from 'adm-zip'; +import { cloneDeep } from 'lodash'; + +import type { Manifest, CdnConfig } from './artifact.types'; +import { ArtifactNotFoundError, ManifestNotFoundError } from './artifact.errors'; + +export class ArtifactService { + private readonly logger: Logger; + private readonly cache: Map; + + private cdnConfig?: CdnConfig; + private clusterInfo?: InfoResponse; + private manifestUrl?: string; + + constructor(logger: Logger, clusterInfo: InfoResponse, cdnConfig: CdnConfig) { + this.logger = logger.get(ArtifactService.name); + this.cache = new Map(); + this.configure(clusterInfo, cdnConfig); + } + + public configure(clusterInfo: InfoResponse, cdnConfig: CdnConfig) { + this.logger.debug('Configuring artifact service with cluster info', { clusterInfo } as LogMeta); + + if (!clusterInfo.version?.number) { + throw new Error( + 'Cluster info must include version number, impossible to calculate the manifest url' + ); + } + + this.clusterInfo = clusterInfo; + this.cdnConfig = cdnConfig; + + const version = + clusterInfo.version.number.substring(0, clusterInfo.version.number.indexOf('-')) || + clusterInfo.version.number; + this.manifestUrl = `${this.cdnConfig?.url}/downloads/kibana/manifest/artifacts-${version}.zip`; + + this.logger.debug('Artifact service started', { manifestUrl: this.manifestUrl } as LogMeta); + } + + public async getArtifact(name: string): Promise { + this.logger.debug('Getting artifact', { name } as LogMeta); + + return axios + .get(this.getManifestUrl(), { + headers: this.headers(name), + timeout: this.cdnConfig?.requestTimeout, + validateStatus: (status) => status < 500, + responseType: 'arraybuffer', + }) + .then(async (response) => { + switch (response.status) { + case 200: + const manifest = { + data: await this.getManifest(name, response.data), + modified: true, + }; + // only update etag if we got a valid manifest + if (response.headers && response.headers.etag) { + const cacheEntry = { + manifest: { ...manifest, modified: false }, + etag: response.headers?.etag ?? '', + }; + this.cache.set(name, cacheEntry); + } + return cloneDeep(manifest); + case 304: + return cloneDeep(this.getCachedManifest(name)); + case 404: + // just in case, remove the entry + this.cache.delete(name); + throw new ManifestNotFoundError(this.manifestUrl!); + default: + throw Error(`Failed to download manifest, unexpected status code: ${response.status}`); + } + }); + } + + private getManifestUrl() { + if (!this.manifestUrl) { + throw Error(`No manifest url for version ${this.clusterInfo?.version?.number}`); + } + return this.manifestUrl; + } + + private getCachedManifest(name: string): Manifest { + const entry = this.cache.get(name); + if (!entry) { + throw new ArtifactNotFoundError(name); + } + return entry.manifest; + } + + private async getManifest(name: string, data: Buffer): Promise { + const zip = new AdmZip(data); + + const manifestFile = zip.getEntry('manifest.json'); + const signatureFile = zip.getEntry('manifest.sig'); + + if (!manifestFile) { + throw Error('No manifest.json in artifact zip'); + } + + if (!signatureFile) { + throw Error('No manifest.sig in artifact zip'); + } + + if (!this.isSignatureValid(manifestFile.getData(), signatureFile.getData())) { + throw Error('Invalid manifest signature'); + } + + const manifest = JSON.parse(manifestFile.getData().toString()); + const artifact = manifest.artifacts[name]; + if (artifact) { + const url = `${this.cdnConfig?.url}${artifact.relative_url}`; + const artifactResponse = await axios.get(url, { timeout: this.cdnConfig?.requestTimeout }); + return artifactResponse.data; + } else { + throw new ArtifactNotFoundError(name); + } + } + + private headers(name: string): Record { + const cacheEntry = this.cache.get(name); + if (cacheEntry?.etag) { + return { 'If-None-Match': cacheEntry.etag }; + } + return {}; + } + + private isSignatureValid(data: Buffer, signature: Buffer): boolean { + if (!this.cdnConfig) { + throw Error('No CDN configuration provided'); + } + + const verifier = createVerify('RSA-SHA256'); + verifier.update(data); + verifier.end(); + + return verifier.verify(this.cdnConfig.pubKey, signature); + } +} + +interface CacheEntry { + manifest: Manifest; + etag: string; +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.types.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.types.ts new file mode 100644 index 0000000000000..b683f765d664e --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/artifact.types.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Represents an artifact retrieved from the CDN. + * + * @property data - The actual content of the artifact. The type is unknown as it depends on the specific artifact. + * @property modified - Indicates whether the artifact has been modified since the last retrieval. + * When true, it means this is a fresh download. + * When false, it means this is a cached version that hasn't changed. + */ +export interface Manifest { + data: unknown; + modified: boolean; +} + +/** + * Configuration details for the CDN used to fetch artifacts. + * + * @property url - The base URL of the CDN for artifact downloads. + * @property pubKey - The public key string used to verify artifact signatures. + */ +export interface CdnConfig { + url: string; + pubKey: string; + requestTimeout: number; +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.test.ts new file mode 100644 index 0000000000000..2d04647231189 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.test.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { firstValueFrom } from 'rxjs'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { ArtifactService } from './artifact'; +import { ConfigurationService, REFRESH_CONFIG_INTERVAL_MS } from './configuration'; +import { IndicesMetadataConfiguration } from './indices_metadata.types'; +import { ArtifactNotFoundError, ManifestNotFoundError } from './artifact.errors'; +import { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; +import { CdnConfig } from './artifact.types'; + +jest.mock('./artifact'); + +describe('ConfigurationService', () => { + let logger: ReturnType; + let configurationService: ConfigurationService; + let artifactService: jest.Mocked; + + const defaultConfiguration: IndicesMetadataConfiguration = { + indices_threshold: 100, + datastreams_threshold: 100, + indices_settings_threshold: 100, + index_query_size: 100, + ilm_stats_query_size: 100, + ilm_policy_query_size: 100, + }; + const fakeCdnConfig: CdnConfig = { + url: 'http://localhost:3000', + pubKey: '..', + requestTimeout: 10, + }; + const fakeClusterInfo: InfoResponse = { + name: 'elasticsearch', + cluster_name: 'elasticsearch', + cluster_uuid: 'fiNVFADnQsepL3HXYMs-qg', + version: { + number: '9.2.0-SNAPSHOT', + build_flavor: 'default', + build_type: 'tar', + build_hash: '560464e544b7e37e581874f44c19c7eac930f901', + build_date: '2025-07-08T02:09:11.988781060Z', + build_snapshot: true, + lucene_version: '10.2.2', + minimum_wire_compatibility_version: '8.19.0', + minimum_index_compatibility_version: '8.0.0', + }, + tagline: 'You Know, for Search', + }; + + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + + logger = loggingSystemMock.createLogger(); + configurationService = new ConfigurationService(logger); + artifactService = new ArtifactService( + logger, + fakeClusterInfo, + fakeCdnConfig + ) as jest.Mocked; + }); + + afterEach(() => { + configurationService.stop(); + jest.useRealTimers(); + }); + + describe('initialization', () => { + it('should throw an error when trying to use the service before starting it', () => { + expect(() => configurationService.getIndicesMetadataConfiguration$()).toThrow( + 'Configuration service not started' + ); + }); + + it('should initialize with default configuration', () => { + configurationService.start(artifactService, defaultConfiguration); + expect(() => configurationService.getIndicesMetadataConfiguration$()).not.toThrow(); + }); + }); + + describe('getIndicesMetadataConfiguration$', () => { + it('should return an observable with the default configuration initially', async () => { + configurationService.start(artifactService, defaultConfiguration); + const config$ = configurationService.getIndicesMetadataConfiguration$(); + const config = await firstValueFrom(config$); + expect(config).toEqual(defaultConfiguration); + }); + + it('should emit updated configuration when artifact service returns new configuration', async () => { + const updatedConfig: IndicesMetadataConfiguration = { + ...defaultConfiguration, + indices_threshold: 200, + }; + const updatedConfigTwo: IndicesMetadataConfiguration = { + ...defaultConfiguration, + indices_threshold: 300, + }; + + jest + .spyOn(artifactService, 'getArtifact') + .mockResolvedValueOnce({ + data: defaultConfiguration, + modified: false, + }) + .mockResolvedValueOnce({ + data: updatedConfig, + modified: true, + }) + .mockResolvedValueOnce({ + data: updatedConfigTwo, + modified: true, + }); + + configurationService.start(artifactService, defaultConfiguration); + + let config: IndicesMetadataConfiguration | undefined; + configurationService.getIndicesMetadataConfiguration$().subscribe((c) => { + config = c; + }); + + expect(config).toEqual(defaultConfiguration); + + await jest.advanceTimersByTimeAsync(REFRESH_CONFIG_INTERVAL_MS * 1.1); + expect(config).toEqual(updatedConfig); + + await jest.advanceTimersByTimeAsync(REFRESH_CONFIG_INTERVAL_MS * 1.1); + expect(config).toEqual(updatedConfigTwo); + + await jest.advanceTimersByTimeAsync(REFRESH_CONFIG_INTERVAL_MS * 1.1); + expect(config).toEqual(updatedConfigTwo); + }); + }); + + describe('error handling during configuration refresh', () => { + const errorCases = [ + { name: 'ManifestNotFoundError', error: new ManifestNotFoundError('test-manifest') }, + { name: 'ArtifactNotFoundError', error: new ArtifactNotFoundError('test-artifact') }, + { name: 'UnexpectedError', error: new Error('unexpected error during artifact retrieval') }, + ]; + + errorCases.forEach(({ name, error }) => { + it(`should maintain last valid configuration when ${name} occurs`, async () => { + jest + .spyOn(artifactService, 'getArtifact') + .mockResolvedValueOnce({ + data: defaultConfiguration, + modified: true, + }) + .mockRejectedValue(error); + + configurationService.start(artifactService, defaultConfiguration); + const config$ = configurationService.getIndicesMetadataConfiguration$(); + + let config: IndicesMetadataConfiguration | undefined; + let updatedCount = 0; + config$.subscribe((c) => { + updatedCount++; + config = c; + }); + + expect(config).toEqual(defaultConfiguration); + + for (let i = 0; i < 10; i++) { + await jest.advanceTimersByTimeAsync(REFRESH_CONFIG_INTERVAL_MS * 1.1); + expect(config).toEqual(defaultConfiguration); + } + + // Verify the observable emitted exactly twice: + // 1. Initial emission with default configuration (from startWith) + // 2. First successful artifact retrieval + // After errors occur, no new emissions should happen as the configuration remains unchanged + expect(updatedCount).toBe(2); + }); + }); + }); +}); diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.ts new file mode 100644 index 0000000000000..f97eeaec1f4bf --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/configuration.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + Observable, + ReplaySubject, + timer, + exhaustMap, + takeUntil, + startWith, + filter, + distinctUntilChanged, + shareReplay, +} from 'rxjs'; +import type { Logger, LogMeta } from '@kbn/core/server'; +import { ArtifactService } from './artifact'; +import { ArtifactNotFoundError, ManifestNotFoundError } from './artifact.errors'; +import { + IndicesMetadataConfiguration, + IndicesMetadataConfigurationSchema, +} from './indices_metadata.types'; +export const REFRESH_CONFIG_INTERVAL_MS = 60 * 60 * 1000; +const CONFIGURATION_ARTIFACT_NAME = 'indices-metadata-configuration-v1'; + +export class ConfigurationService { + private readonly logger: Logger; + private artifactService!: ArtifactService; + private indicesMetadataConfiguration$!: Observable; + + private readonly stop$ = new ReplaySubject(1); + + constructor(logger: Logger) { + this.logger = logger.get(ConfigurationService.name); + } + + public start( + artifactService: ArtifactService, + defaultConfiguration: IndicesMetadataConfiguration + ) { + this.artifactService = artifactService; + this.indicesMetadataConfiguration$ = timer(0, REFRESH_CONFIG_INTERVAL_MS).pipe( + exhaustMap(() => this.getConfiguration()), + takeUntil(this.stop$), + startWith(defaultConfiguration), + filter((config) => config !== undefined), + distinctUntilChanged(), + shareReplay(1) + ); + } + + public stop() { + this.stop$.next(); + this.stop$.complete(); + } + + public getIndicesMetadataConfiguration$(): Observable { + this.ensureStarted(); + + return this.indicesMetadataConfiguration$; + } + + private async getConfiguration(): Promise { + this.ensureStarted(); + + try { + this.logger.debug('Getting indices metadata configuration'); + const artifact = await this.artifactService.getArtifact(CONFIGURATION_ARTIFACT_NAME); + if (!artifact.modified) { + this.logger.debug('Indices metadata configuration has not been modified'); + return undefined; + } + + this.logger.debug('Indices metadata configuration has been modified', { + artifact, + } as LogMeta); + return IndicesMetadataConfigurationSchema.validate(artifact.data); + } catch (error) { + if (error instanceof ManifestNotFoundError) { + this.logger.warn('Indices metadata configuration manifest not found'); + } else if (error instanceof ArtifactNotFoundError) { + this.logger.warn('Indices metadata configuration artifact not found'); + } else { + this.logger.error(`Failed to get indices metadata configuration: ${error}`, { + error, + } as LogMeta); + } + return undefined; + } + } + + private ensureStarted() { + if (!this.artifactService || !this.indicesMetadataConfiguration$) { + throw new Error('Configuration service not started'); + } + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts new file mode 100644 index 0000000000000..ad5fa8dfec233 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts @@ -0,0 +1,596 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Subscription } from 'rxjs'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import type { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { AnalyticsServiceStart, ElasticsearchClient } from '@kbn/core/server'; +import type { + DataStream, + IndicesMetadataConfiguration, + IndexSettings, + IndexStats, + IndexTemplateInfo, +} from './indices_metadata.types'; + +import { IndicesMetadataService } from './indices_metadata'; +import { MetadataReceiver } from './receiver'; +import { MetadataSender } from './sender'; +import { ConfigurationService } from './configuration'; +import { + DATA_STREAM_EVENT, + ILM_POLICY_EVENT, + ILM_STATS_EVENT, + INDEX_SETTINGS_EVENT, + INDEX_STATS_EVENT, + INDEX_TEMPLATES_EVENT, +} from '../ebt/events'; + +jest.mock('./receiver'); +jest.mock('./sender'); + +describe('Indices Metadata - IndicesMetadataService', () => { + let logger: ReturnType; + let configurationService: jest.Mocked; + let service: IndicesMetadataService; + let taskManager: jest.Mocked; + let taskManagerStart: jest.Mocked; + let analytics: jest.Mocked; + let esClient: jest.Mocked; + let receiver: jest.Mocked; + let sender: jest.Mocked; + let subscription: jest.Mocked; + + const mockConfiguration: IndicesMetadataConfiguration = { + indices_threshold: 100, + datastreams_threshold: 50, + indices_settings_threshold: 75, + index_query_size: 10, + ilm_stats_query_size: 20, + ilm_policy_query_size: 30, + }; + + const mockIndexSettings: IndexSettings[] = [ + { + index_name: 'test-index-1', + default_pipeline: 'default', + final_pipeline: 'final', + index_mode: 'standard', + source_mode: 'stored', + }, + ]; + + const mockDataStreams: DataStream[] = [ + { + datastream_name: 'test-datastream', + indices: [{ index_name: 'test-index-1', ilm_policy: 'policy1' }], + }, + ]; + + const mockIndexTemplates: IndexTemplateInfo[] = [ + { + template_name: 'test-template', + index_mode: 'standard', + package_name: 'test-package', + datastream: true, + managed_by: 'elasticsearch', + beat: undefined, + is_managed: true, + composed_of: ['component1'], + source_enabled: true, + source_includes: [], + source_excludes: [], + }, + ]; + + const mockIndexStats: IndexStats[] = [ + { + index_name: 'test-index-1', + query_total: 100, + query_time_in_millis: 1000, + docs_count: 500, + docs_deleted: 10, + docs_total_size_in_bytes: 1024000, + }, + ]; + + beforeEach(() => { + jest.clearAllMocks(); + + logger = loggingSystemMock.createLogger(); + configurationService = { + getIndicesMetadataConfiguration$: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + } as unknown as jest.Mocked; + + taskManager = { + registerTaskDefinitions: jest.fn(), + } as unknown as jest.Mocked; + + taskManagerStart = { + ensureScheduled: jest.fn(), + } as unknown as jest.Mocked; + + analytics = { + reportEvent: jest.fn(), + } as unknown as jest.Mocked; + + esClient = {} as jest.Mocked; + + receiver = { + getIndices: jest.fn(), + getDataStreams: jest.fn(), + getIndexTemplatesStats: jest.fn(), + getIndicesStats: jest.fn(), + isIlmStatsAvailable: jest.fn(), + getIlmsStats: jest.fn(), + getIlmsPolicies: jest.fn(), + } as unknown as jest.Mocked; + + sender = { + reportEBT: jest.fn(), + } as unknown as jest.Mocked; + + subscription = { + unsubscribe: jest.fn(), + } as unknown as jest.Mocked; + + (MetadataReceiver as jest.Mock).mockImplementation(() => receiver); + (MetadataSender as jest.Mock).mockImplementation(() => sender); + + service = new IndicesMetadataService(logger, configurationService); + }); + + describe('constructor', () => { + it('should create service with proper logger namespace', () => { + expect(logger.get).toHaveBeenCalledWith('IndicesMetadataService'); + }); + }); + + describe('setup', () => { + it('should register task definitions', () => { + service.setup(taskManager); + + expect(taskManager.registerTaskDefinitions).toHaveBeenCalledWith({ + 'IndicesMetadata:IndicesMetadataTask': expect.objectContaining({ + title: 'Metrics Data Access - Indices Metadata Task', + description: 'This task periodically pushes indices metadata to the telemetry service.', + maxAttempts: 1, + createTaskRunner: expect.any(Function), + }), + }); + }); + + it('should log debug messages', () => { + service.setup(taskManager); + + expect(logger.debug).toHaveBeenCalledWith('Setting up indices metadata service'); + expect(logger.debug).toHaveBeenCalledWith('About to register task', { + task: 'indices-metadata:indices-metadata-task:1.0.0', + }); + expect(logger.debug).toHaveBeenCalledWith('Task registered', { + task: 'indices-metadata:indices-metadata-task:1.0.0', + type: 'IndicesMetadata:IndicesMetadataTask', + }); + }); + }); + + describe('start', () => { + beforeEach(() => { + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockReturnValue(subscription), + } as any); + + taskManagerStart.ensureScheduled.mockResolvedValue({ + id: 'test-task', + schedule: { interval: '24h' }, + } as any); + }); + + it('should initialize receiver and sender', () => { + service.start(taskManagerStart, analytics, esClient); + + expect(MetadataReceiver).toHaveBeenCalledWith(expect.any(Object), esClient); + expect(MetadataSender).toHaveBeenCalledWith(expect.any(Object), analytics); + }); + + it('should subscribe to configuration updates', () => { + const mockSubscribe = jest.fn().mockReturnValue(subscription); + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: mockSubscribe, + } as any); + + service.start(taskManagerStart, analytics, esClient); + + expect(configurationService.getIndicesMetadataConfiguration$).toHaveBeenCalled(); + expect(mockSubscribe).toHaveBeenCalledWith(expect.any(Function)); + }); + + it('should schedule indices metadata task', async () => { + service.start(taskManagerStart, analytics, esClient); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(taskManagerStart.ensureScheduled).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'indices-metadata:indices-metadata-task:1.0.0', + taskType: 'IndicesMetadata:IndicesMetadataTask', + params: {}, + state: {}, + scope: ['uptime'], + }) + ); + }); + + it('should handle task scheduling errors', async () => { + const error = new Error('Failed to schedule task'); + taskManagerStart.ensureScheduled.mockRejectedValue(error); + + service.start(taskManagerStart, analytics, esClient); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(logger.error).toHaveBeenCalledWith('Failed to schedule Indices Metadata Task', { + error, + }); + }); + + it('should handle configuration updates', () => { + const mockSubscribe = jest.fn(); + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: mockSubscribe, + } as any); + + service.start(taskManagerStart, analytics, esClient); + + const configurationCallback = mockSubscribe.mock.calls[0][0]; + configurationCallback(mockConfiguration); + + expect(logger.debug).toHaveBeenCalledWith('Indices metadata configuration updated', { + configuration: mockConfiguration, + }); + }); + }); + + describe('stop', () => { + it('should unsubscribe from configuration updates', () => { + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockReturnValue(subscription), + } as any); + + service.start(taskManagerStart, analytics, esClient); + service.stop(); + + expect(subscription.unsubscribe).toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith('Stopping indices metadata service'); + }); + + it('should handle stop when subscription is undefined', () => { + expect(() => service.stop()).not.toThrow(); + }); + }); + + describe('publishIndicesMetadata', () => { + beforeEach(() => { + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockImplementation((callback) => { + callback(mockConfiguration); + return subscription; + }), + } as any); + + service.start(taskManagerStart, analytics, esClient); + + receiver.getIndices.mockResolvedValue(mockIndexSettings); + receiver.getDataStreams.mockResolvedValue(mockDataStreams); + receiver.getIndexTemplatesStats.mockResolvedValue(mockIndexTemplates); + receiver.getIndicesStats.mockImplementation(async function* () { + yield* mockIndexStats; + }); + receiver.isIlmStatsAvailable.mockResolvedValue(true); + receiver.getIlmsStats.mockImplementation(async function* () { + yield { index_name: 'test-index-1', phase: 'hot', age: '1d', policy_name: 'policy1' }; + }); + receiver.getIlmsPolicies.mockImplementation(async function* () { + yield { policy_name: 'policy1', modified_date: '2023-01-01', phases: {} }; + }); + }); + + it('should successfully publish all metadata types', async () => { + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(receiver.getIndices).toHaveBeenCalled(); + expect(receiver.getDataStreams).toHaveBeenCalled(); + expect(receiver.getIndexTemplatesStats).toHaveBeenCalled(); + expect(receiver.getIndicesStats).toHaveBeenCalledWith(['test-index-1'], 10); + expect(receiver.isIlmStatsAvailable).toHaveBeenCalled(); + expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1']); + expect(receiver.getIlmsPolicies).toHaveBeenCalledWith(['policy1'], 30); + + expect(sender.reportEBT).toHaveBeenCalledTimes(6); + }); + + it('should skip publication when configuration is undefined', async () => { + service['configuration'] = undefined; // eslint-disable-line dot-notation + + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(receiver.getIndices).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith( + 'Index query size is 0, skipping indices metadata publish' + ); + }); + + it('should skip publication when index_query_size is 0', async () => { + service['configuration'] = { ...mockConfiguration, index_query_size: 0 }; // eslint-disable-line dot-notation + + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(receiver.getIndices).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith( + 'Index query size is 0, skipping indices metadata publish' + ); + }); + + it('should handle ILM stats unavailable', async () => { + receiver.isIlmStatsAvailable.mockResolvedValue(false); + + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(receiver.getIlmsStats).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith('ILM explain API is not available'); + }); + + it('should apply thresholds correctly', async () => { + // eslint-disable-next-line dot-notation + service['configuration'] = { + ...mockConfiguration, + datastreams_threshold: 1, + indices_settings_threshold: 1, + indices_threshold: 1, + }; + + await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation + + expect(receiver.getIndicesStats).toHaveBeenCalledWith(['test-index-1'], 10); + expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1']); + }); + + it('should throw error when not initialized', async () => { + const uninitializedService = new IndicesMetadataService(logger, configurationService); + + // eslint-disable-next-line dot-notation + await expect(uninitializedService['publishIndicesMetadata']()).rejects.toThrow( + 'Indices metadata service not initialized' + ); + }); + }); + + describe('task runner', () => { + let taskRunner: any; + let taskInstance: ConcreteTaskInstance; + + beforeEach(() => { + service.setup(taskManager); + + const taskDefinition = + taskManager.registerTaskDefinitions.mock.calls[0][0]['IndicesMetadata:IndicesMetadataTask']; + taskInstance = { state: { lastRun: '2023-01-01' } } as unknown as ConcreteTaskInstance; + taskRunner = taskDefinition.createTaskRunner({ taskInstance }); + + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockImplementation((callback) => { + callback(mockConfiguration); + return subscription; + }), + } as any); + + service.start(taskManagerStart, analytics, esClient); + }); + + it('should run publishIndicesMetadata and return state', async () => { + jest.spyOn(service as any, 'publishIndicesMetadata').mockResolvedValue(undefined); + + const result = await taskRunner.run(); + + expect(service['publishIndicesMetadata']).toHaveBeenCalled(); // eslint-disable-line dot-notation + expect(result).toEqual({ state: taskInstance.state }); + }); + + it('should handle task cancellation', async () => { + await taskRunner.cancel(); + + expect(logger.warn).toHaveBeenCalledWith('Task timed out', { + task: 'indices-metadata:indices-metadata-task:1.0.0', + }); + }); + }); + + describe('publish EBT', () => { + beforeEach(() => { + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockImplementation((callback) => { + callback(mockConfiguration); + return subscription; + }), + } as any); + + service.start(taskManagerStart, analytics, esClient); + }); + + describe('publishDatastreamsStats', () => { + it('should publish datastreams and return count', () => { + const result = service['publishDatastreamsStats'](mockDataStreams); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }), + { items: mockDataStreams } + ); + expect(result).toBe(1); + expect(logger.debug).toHaveBeenCalledWith('Data streams events sent', { count: 1 }); + }); + }); + + describe('publishIndicesSettings', () => { + it('should publish indices settings and return count', () => { + const result = service['publishIndicesSettings'](mockIndexSettings); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: INDEX_SETTINGS_EVENT.eventType }), + { items: mockIndexSettings } + ); + expect(result).toBe(1); + expect(logger.debug).toHaveBeenCalledWith('Indices settings sent', { count: 1 }); + }); + }); + + describe('publishIndicesStats', () => { + it('should publish indices stats and return count', async () => { + receiver.getIndicesStats.mockImplementation(async function* () { + yield* mockIndexStats; + }); + + const result = await service['publishIndicesStats'](['test-index-1']); // eslint-disable-line dot-notation + + expect(receiver.getIndicesStats).toHaveBeenCalledWith(['test-index-1'], 10); + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: INDEX_STATS_EVENT.eventType }), + { items: mockIndexStats } + ); + expect(result).toBe(1); + expect(logger.debug).toHaveBeenCalledWith('Indices stats sent', { count: 1 }); + }); + + it('should return 0 when configuration is undefined', async () => { + service['configuration'] = undefined; // eslint-disable-line dot-notation + + const result = await service['publishIndicesStats'](['test-index-1']); // eslint-disable-line dot-notation + + expect(result).toBe(0); + expect(receiver.getIndicesStats).not.toHaveBeenCalled(); + }); + }); + + describe('publishIlmStats', () => { + it('should publish ILM stats and return policy names', async () => { + const mockIlmStats = [ + { index_name: 'test-index-1', phase: 'hot', age: '1d', policy_name: 'policy1' }, + { index_name: 'test-index-2', phase: 'warm', age: '7d', policy_name: 'policy2' }, + ]; + + receiver.getIlmsStats.mockImplementation(async function* () { + yield* mockIlmStats; + }); + + const result = await service['publishIlmStats'](['test-index-1', 'test-index-2']); // eslint-disable-line dot-notation + + expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1', 'test-index-2']); + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: ILM_STATS_EVENT.eventType }), + { items: mockIlmStats } + ); + expect(result).toEqual(new Set(['policy1', 'policy2'])); + expect(result.size).toBe(2); + }); + + it('should skip stats without policy names', async () => { + const mockIlmStats = [ + { index_name: 'test-index-1', phase: 'hot', age: '1d', policy_name: 'policy1' }, + { index_name: 'test-index-2', phase: 'hot', age: '1d', policy_name: undefined }, + ]; + + receiver.getIlmsStats.mockImplementation(async function* () { + yield* mockIlmStats; + }); + + const result = await service['publishIlmStats'](['test-index-1', 'test-index-2']); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: ILM_STATS_EVENT.eventType }), + { items: [mockIlmStats[0]] } + ); + expect(result).toEqual(new Set(['policy1'])); + expect(result.size).toBe(1); + }); + }); + + describe('publishIlmPolicies', () => { + it('should publish ILM policies and return count', async () => { + const mockPolicies = [{ policy_name: 'policy1', modified_date: '2023-01-01', phases: {} }]; + + receiver.getIlmsPolicies.mockImplementation(async function* () { + yield* mockPolicies; + }); + + const result = await service['publishIlmPolicies'](new Set(['policy1'])); // eslint-disable-line dot-notation + + expect(receiver.getIlmsPolicies).toHaveBeenCalledWith(['policy1'], 30); + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: ILM_POLICY_EVENT.eventType }), + { items: mockPolicies } + ); + expect(result).toBe(1); + }); + + it('should return 0 when configuration is undefined', async () => { + service['configuration'] = undefined; // eslint-disable-line dot-notation + + const result = await service['publishIlmPolicies'](new Set(['policy1'])); // eslint-disable-line dot-notation + + expect(result).toBe(0); + expect(receiver.getIlmsPolicies).not.toHaveBeenCalled(); + }); + }); + + describe('publishIndexTemplatesStats', () => { + it('should publish index templates stats and return count', async () => { + const result = await service['publishIndexTemplatesStats'](mockIndexTemplates); // eslint-disable-line dot-notation + + expect(sender.reportEBT).toHaveBeenCalledWith( + expect.objectContaining({ eventType: INDEX_TEMPLATES_EVENT.eventType }), + { items: mockIndexTemplates } + ); + expect(result).toBe(1); + expect(logger.debug).toHaveBeenCalledWith('Index templates stats sent', { count: 1 }); + }); + }); + }); + + describe('error scenarios', () => { + beforeEach(() => { + configurationService.getIndicesMetadataConfiguration$.mockReturnValue({ + subscribe: jest.fn().mockImplementation((callback) => { + callback(mockConfiguration); + return subscription; + }), + } as any); + + service.start(taskManagerStart, analytics, esClient); + }); + + it('should handle receiver errors during publishIndicesMetadata', async () => { + const error = new Error('Elasticsearch error'); + receiver.getIndices.mockRejectedValue(error); + + await expect(service['publishIndicesMetadata']()).rejects.toThrow('Elasticsearch error'); // eslint-disable-line dot-notation + }); + + it('should handle sender errors during publishDatastreamsStats', () => { + const error = new Error('Analytics error'); + sender.reportEBT.mockImplementation(() => { + throw error; + }); + + expect(() => service['publishDatastreamsStats'](mockDataStreams)).toThrow('Analytics error'); // eslint-disable-line dot-notation + }); + }); +}); diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts new file mode 100644 index 0000000000000..3ec0f9f11afa4 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts @@ -0,0 +1,296 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Subscription } from 'rxjs'; +import type { + ConcreteTaskInstance, + TaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { AnalyticsServiceStart, ElasticsearchClient, LogMeta, Logger } from '@kbn/core/server'; +import type { + DataStream, + DataStreams, + IlmPolicies, + IlmsStats, + IndexSettings, + IndexTemplateInfo, + IndexTemplatesStats, + IndicesMetadataConfiguration, + IndicesSettings, + IndicesStats, +} from './indices_metadata.types'; + +import { + DATA_STREAM_EVENT, + ILM_POLICY_EVENT, + ILM_STATS_EVENT, + INDEX_STATS_EVENT, + INDEX_SETTINGS_EVENT, + INDEX_TEMPLATES_EVENT, +} from '../ebt/events'; +import { MetadataReceiver } from './receiver'; +import { MetadataSender } from './sender'; +import { ConfigurationService } from './configuration'; + +const TASK_TYPE = 'IndicesMetadata:IndicesMetadataTask'; +const TASK_ID = 'indices-metadata:indices-metadata-task:1.0.0'; +const INTERVAL = '24h'; + +export class IndicesMetadataService { + private readonly logger: Logger; + private readonly configurationService: ConfigurationService; + + private receiver!: MetadataReceiver; + private sender!: MetadataSender; + private subscription$!: Subscription; + private configuration?: IndicesMetadataConfiguration; + + constructor(logger: Logger, configurationService: ConfigurationService) { + this.logger = logger.get(IndicesMetadataService.name); + this.configurationService = configurationService; + } + + public setup(taskManager: TaskManagerSetupContract) { + this.logger.debug('Setting up indices metadata service'); + this.registerIndicesMetadataTask(taskManager); + } + + public start( + taskManager: TaskManagerStartContract, + analytics: AnalyticsServiceStart, + esClient: ElasticsearchClient + ) { + this.logger.debug('Starting indices metadata service'); + + this.receiver = new MetadataReceiver(this.logger, esClient); + this.sender = new MetadataSender(this.logger, analytics); + + this.subscription$ = this.configurationService + .getIndicesMetadataConfiguration$() + .subscribe((configuration) => { + this.logger.debug('Indices metadata configuration updated', { configuration } as LogMeta); + if (configuration) { + this.configuration = configuration; + } + }); + + this.scheduleIndicesMetadataTask(taskManager).catch((error) => { + this.logger.error('Failed to schedule Indices Metadata Task', { error }); + }); + } + + public stop() { + this.logger.debug('Stopping indices metadata service'); + this.subscription$?.unsubscribe(); + } + + private async publishIndicesMetadata() { + this.ensureInitialized(); + + if (!this.configuration || this.configuration.index_query_size === 0) { + this.logger.debug('Index query size is 0, skipping indices metadata publish'); + return; + } + + // 1. Get cluster stats and list of indices and datastreams + const [indicesSettings, dataStreams, indexTemplates] = await Promise.all([ + this.receiver.getIndices(), + this.receiver.getDataStreams(), + this.receiver.getIndexTemplatesStats(), + ]); + + const indices = indicesSettings.map((index) => index.index_name); + + // 2. Publish datastreams stats + const dsCount = this.publishDatastreamsStats( + dataStreams.slice(0, this.configuration.datastreams_threshold) + ); + + // 3. Publish indices settings + const indicesSettingsCount = this.publishIndicesSettings( + indicesSettings.slice(0, this.configuration.indices_settings_threshold) + ); + + // 4. Get and publish indices stats + const indicesCount: number = await this.publishIndicesStats( + indices.slice(0, this.configuration.indices_threshold) + ); + + // 5. Get ILM stats and publish them + let ilmNames = new Set(); + if (await this.receiver.isIlmStatsAvailable()) { + ilmNames = await this.publishIlmStats(indices.slice(0, this.configuration.indices_threshold)); + } else { + this.logger.debug('ILM explain API is not available'); + } + + // 6. Publish ILM policies + const policyCount = await this.publishIlmPolicies(ilmNames); + + // 7. Publish index templates + const indexTemplatesCount: number = await this.publishIndexTemplatesStats( + indexTemplates.slice(0, this.configuration.indices_threshold) + ); + + this.logger.debug('EBT events sent', { + datastreams: dsCount, + ilms: ilmNames.size, + indices: indicesCount, + indicesSettings: indicesSettingsCount, + policies: policyCount, + templates: indexTemplatesCount, + } as LogMeta); + } + + private ensureInitialized() { + if (!this.receiver || !this.sender) { + throw new Error('Indices metadata service not initialized'); + } + } + + private publishDatastreamsStats(stats: DataStream[]): number { + const events: DataStreams = { items: stats }; + this.sender.reportEBT(DATA_STREAM_EVENT, events); + this.logger.debug('Data streams events sent', { count: events.items.length } as LogMeta); + return events.items.length; + } + + private registerIndicesMetadataTask(taskManager: TaskManagerSetupContract) { + const service = this; + + this.logger.debug('About to register task', { task: TASK_ID } as LogMeta); + + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Metrics Data Access - Indices Metadata Task', + description: 'This task periodically pushes indices metadata to the telemetry service.', + timeout: '2m', + maxAttempts: 1, + + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + async run() { + const { state } = taskInstance; + await service.publishIndicesMetadata(); + return { state }; + }, + + async cancel() { + service.logger.warn('Task timed out', { task: TASK_ID } as LogMeta); + }, + }; + }, + }, + }); + + this.logger.debug('Task registered', { task: TASK_ID, type: TASK_TYPE } as LogMeta); + } + + private async scheduleIndicesMetadataTask( + taskManager: TaskManagerStartContract + ): Promise { + this.logger.debug('About to schedule task', { task: TASK_ID } as LogMeta); + + const taskInstance = await taskManager.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + schedule: { interval: INTERVAL }, + params: {}, + state: {}, + scope: ['uptime'], + }); + + this.logger.debug('Task scheduled', { + task: TASK_ID, + interval: taskInstance.schedule?.interval, + } as LogMeta); + + return taskInstance; + } + + private async publishIndicesStats(indices: string[]): Promise { + if (!this.configuration) { + return 0; + } + const indicesStats: IndicesStats = { + items: [], + }; + + for await (const stat of this.receiver.getIndicesStats( + indices, + this.configuration.index_query_size + )) { + indicesStats.items.push(stat); + } + this.sender.reportEBT(INDEX_STATS_EVENT, indicesStats); + this.logger.debug('Indices stats sent', { count: indicesStats.items.length } as LogMeta); + return indicesStats.items.length; + } + + private publishIndicesSettings(settings: IndexSettings[]): number { + const indicesSettings: IndicesSettings = { + items: settings, + }; + + this.sender.reportEBT(INDEX_SETTINGS_EVENT, indicesSettings); + this.logger.debug('Indices settings sent', { count: indicesSettings.items.length } as LogMeta); + return indicesSettings.items.length; + } + private async publishIlmStats(indices: string[]): Promise> { + const ilmNames = new Set(); + const ilmsStats: IlmsStats = { + items: [], + }; + + for await (const stat of this.receiver.getIlmsStats(indices)) { + if (stat.policy_name !== undefined) { + ilmNames.add(stat.policy_name); + ilmsStats.items.push(stat); + } + } + + this.sender.reportEBT(ILM_STATS_EVENT, ilmsStats); + this.logger.debug('ILM stats sent', { count: ilmNames.size } as LogMeta); + + return ilmNames; + } + + async publishIlmPolicies(ilmNames: Set): Promise { + if (!this.configuration) { + return 0; + } + + const ilmPolicies: IlmPolicies = { + items: [], + }; + + for await (const policy of this.receiver.getIlmsPolicies( + Array.from(ilmNames.values()), + this.configuration.ilm_policy_query_size + )) { + ilmPolicies.items.push(policy); + } + this.sender.reportEBT(ILM_POLICY_EVENT, ilmPolicies); + this.logger.debug('ILM policies sent', { count: ilmPolicies.items.length } as LogMeta); + return ilmPolicies.items.length; + } + + async publishIndexTemplatesStats(indexTemplates: IndexTemplateInfo[]): Promise { + const templateStats: IndexTemplatesStats = { + items: indexTemplates, + }; + + this.sender.reportEBT(INDEX_TEMPLATES_EVENT, templateStats); + this.logger.debug('Index templates stats sent', { + count: templateStats.items.length, + } as LogMeta); + + return templateStats.items.length; + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts new file mode 100644 index 0000000000000..776c8e8c7722c --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { DateTime } from '@elastic/elasticsearch/lib/api/types'; +import { schema, TypeOf } from '@kbn/config-schema'; + +export const IndicesMetadataConfigurationSchema = schema.object({ + indices_threshold: schema.number(), + datastreams_threshold: schema.number(), + indices_settings_threshold: schema.number(), + index_query_size: schema.number(), + ilm_stats_query_size: schema.number(), + ilm_policy_query_size: schema.number(), +}); + +export type IndicesMetadataConfiguration = TypeOf; + +export interface IlmPolicies { + items: IlmPolicy[]; +} + +export interface IlmPolicy { + policy_name: string; + modified_date: DateTime; + phases: IlmPhases; +} + +export interface IlmPhases { + cold?: IlmPhase; + delete?: IlmPhase; + frozen?: IlmPhase; + hot?: IlmPhase; + warm?: IlmPhase; +} + +export interface IlmPhase { + min_age: string; +} + +export interface IlmsStats { + items: IlmStats[]; +} + +export interface IlmStats { + index_name: string; + phase?: string; + age?: string; + policy_name?: string; +} + +export interface IndexTemplatesStats { + items: IndexTemplateInfo[]; +} + +export interface IndexTemplateInfo { + template_name: string; + index_mode?: string; + datastream: boolean; + package_name?: string; + managed_by?: string; + beat?: string; + is_managed?: boolean; + composed_of: string[]; + source_enabled?: boolean; + source_includes: string[]; + source_excludes: string[]; +} + +export interface IndicesStats { + items: IndexStats[]; +} + +export interface IndexStats { + index_name: string; + + query_total?: number; + query_time_in_millis?: number; + + // values for primary shards + docs_count_primaries?: number; + docs_deleted_primaries?: number; + docs_total_size_in_bytes_primaries?: number; + + // values for primary and replica shards + docs_count?: number; + docs_deleted?: number; + docs_total_size_in_bytes?: number; + + index_failed?: number; + index_failed_due_to_version_conflict?: number; +} + +export interface IndicesSettings { + items: IndexSettings[]; +} + +export interface IndexSettings { + index_name: string; + index_mode?: string; + default_pipeline?: string; + final_pipeline?: string; + source_mode?: string; +} + +export interface Index { + index_name: string; + ilm_policy?: string; +} + +export interface DataStreams { + items: DataStream[]; +} +export interface DataStream { + datastream_name: string; + ilm_policy?: string; + template?: string; + indices?: Index[]; +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts new file mode 100644 index 0000000000000..1c898025b2fb9 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts @@ -0,0 +1,310 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, LogMeta, Logger } from '@kbn/core/server'; +import type { + IlmExplainLifecycleRequest, + IlmGetLifecycleRequest, + IndicesGetDataStreamRequest, + IndicesGetIndexTemplateRequest, + IndicesGetRequest, + IndicesStatsRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { + DataStream, + IlmPhase, + IlmPhases, + IlmPolicy, + IlmStats, + Index, + IndexSettings, + IndexStats, + IndexTemplateInfo, +} from './indices_metadata.types'; +import { chunkedBy } from '../utils'; + +export class MetadataReceiver { + private readonly logger: Logger; + + constructor(logger: Logger, private readonly esClient: ElasticsearchClient) { + this.logger = logger.get(MetadataReceiver.name); + } + + public async getIndices(): Promise { + this.logger.debug('Fetching indices'); + + const request: IndicesGetRequest = { + index: '*', + expand_wildcards: ['open', 'hidden'], + filter_path: [ + '*.mappings._source.mode', + '*.settings.index.default_pipeline', + '*.settings.index.final_pipeline', + '*.settings.index.mode', + '*.settings.index.provided_name', + ], + }; + + return this.esClient.indices + .get(request) + .then((indices) => + Object.entries(indices).map(([index, value]) => { + return { + index_name: index, + default_pipeline: value.settings?.index?.default_pipeline, + final_pipeline: value.settings?.index?.final_pipeline, + index_mode: value.settings?.index?.mode, + source_mode: value.mappings?._source?.mode, + } as IndexSettings; + }) + ) + .catch((error) => { + this.logger.warn('Error fetching indices', { error_message: error } as LogMeta); + throw error; + }); + } + + public async getDataStreams(): Promise { + this.logger.debug('Fetching datstreams'); + + const request: IndicesGetDataStreamRequest = { + name: '*', + expand_wildcards: ['open', 'hidden'], + filter_path: ['data_streams.name', 'data_streams.indices'], + }; + + return this.esClient.indices + .getDataStream(request) + .then((response) => + response.data_streams.map((ds) => { + return { + datastream_name: ds.name, + indices: + ds.indices?.map((index) => { + return { + index_name: index.index_name, + ilm_policy: index.ilm_policy, + } as Index; + }) ?? [], + } as DataStream; + }) + ) + .catch((error) => { + this.logger.error('Error fetching datastreams', { error_message: error } as LogMeta); + throw error; + }); + } + + public async *getIndicesStats(indices: string[], chunkSize: number) { + const safeChunkSize = Math.min(chunkSize, 3000); + + this.logger.debug('Fetching indices stats'); + + const groupedIndices = this.chunkStringsByMaxLength(indices, safeChunkSize); + + this.logger.debug('Splitted indices into groups', { + groups: groupedIndices.length, + indices: indices.length, + } as LogMeta); + + for (const group of groupedIndices) { + const request: IndicesStatsRequest = { + index: group, + level: 'indices', + metric: ['docs', 'search', 'store'], + expand_wildcards: ['open', 'hidden'], + filter_path: [ + 'indices.*.total.search.query_total', + 'indices.*.total.search.query_time_in_millis', + 'indices.*.total.docs.count', + 'indices.*.total.docs.deleted', + 'indices.*.total.store.size_in_bytes', + ], + }; + + try { + const response = await this.esClient.indices.stats(request); + for (const [indexName, stats] of Object.entries(response.indices ?? {})) { + yield { + index_name: indexName, + query_total: stats.total?.search?.query_total, + query_time_in_millis: stats.total?.search?.query_time_in_millis, + docs_count: stats.total?.docs?.count, + docs_deleted: stats.total?.docs?.deleted, + docs_total_size_in_bytes: stats.total?.store?.size_in_bytes, + } as IndexStats; + } + } catch (error) { + this.logger.error('Error fetching indices stats', { error_message: error } as LogMeta); + throw error; + } + } + } + + public async isIlmStatsAvailable() { + const request: IlmExplainLifecycleRequest = { + index: '-invalid-index', + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const result = await this.esClient.ilm + .explainLifecycle(request) + .then(() => { + return true; + }) + .catch((error) => { + return error.meta.statusCode === 404; + }); + + return result; + } + + public async *getIlmsStats(indices: string[]) { + const groupedIndices = this.chunkStringsByMaxLength(indices); + + this.logger.debug('Splitted ilms into groups', { + groups: groupedIndices.length, + indices: indices.length, + } as LogMeta); + + for (const group of groupedIndices) { + const request: IlmExplainLifecycleRequest = { + index: group.join(','), + only_managed: false, + filter_path: ['indices.*.phase', 'indices.*.age', 'indices.*.policy'], + }; + + const data = await this.esClient.ilm.explainLifecycle(request); + + try { + for (const [indexName, stats] of Object.entries(data.indices ?? {})) { + const entry = { + index_name: indexName, + phase: ('phase' in stats && stats.phase) || undefined, + age: ('age' in stats && stats.age) || undefined, + policy_name: ('policy' in stats && stats.policy) || undefined, + } as IlmStats; + + yield entry; + } + } catch (error) { + this.logger.error('Error fetching ilm stats', { error_message: error } as LogMeta); + throw error; + } + } + } + + public async getIndexTemplatesStats(): Promise { + this.logger.debug('Fetching index templates'); + + const request: IndicesGetIndexTemplateRequest = { + name: '*', + filter_path: [ + 'index_templates.name', + 'index_templates.index_template.template.settings.index.mode', + 'index_templates.index_template.data_stream', + 'index_templates.index_template._meta.package.name', + 'index_templates.index_template._meta.managed_by', + 'index_templates.index_template._meta.beat', + 'index_templates.index_template._meta.managed', + 'index_templates.index_template.composed_of', + 'index_templates.index_template.template.mappings._source.enabled', + 'index_templates.index_template.template.mappings._source.includes', + 'index_templates.index_template.template.mappings._source.excludes', + ], + }; + + return this.esClient.indices + .getIndexTemplate(request) + .then((response) => + response.index_templates.map((props) => { + const datastream = props.index_template?.data_stream !== undefined; + return { + template_name: props.name, + index_mode: props.index_template.template?.settings?.index?.mode, + package_name: props.index_template._meta?.package?.name, + datastream, + managed_by: props.index_template._meta?.managed_by, + beat: props.index_template._meta?.beat, + is_managed: props.index_template._meta?.managed, + composed_of: props.index_template.composed_of, + source_enabled: props.index_template.template?.mappings?._source?.enabled, + source_includes: props.index_template.template?.mappings?._source?.includes ?? [], + source_excludes: props.index_template.template?.mappings?._source?.excludes ?? [], + } as IndexTemplateInfo; + }) + ) + .catch((error) => { + this.logger.warn('Error fetching index templates', { error_message: error } as LogMeta); + throw error; + }); + } + + public async *getIlmsPolicies(ilms: string[], chunkSize: number) { + const safeChunkSize = Math.min(chunkSize, 3000); + + const phase = (obj: unknown): IlmPhase | null | undefined => { + let value: IlmPhase | null | undefined; + if (obj !== null && obj !== undefined && typeof obj === 'object' && 'min_age' in obj) { + value = { + min_age: obj.min_age, + } as IlmPhase; + } + return value; + }; + + const groupedIlms = this.chunkStringsByMaxLength(ilms, safeChunkSize); + + this.logger.debug('Splitted ilms into groups', { + groups: groupedIlms.length, + ilms: ilms.length, + } as LogMeta); + + for (const group of groupedIlms) { + this.logger.debug('Fetching ilm policies'); + const request: IlmGetLifecycleRequest = { + name: group.join(','), + filter_path: [ + '*.policy.phases.cold.min_age', + '*.policy.phases.delete.min_age', + '*.policy.phases.frozen.min_age', + '*.policy.phases.hot.min_age', + '*.policy.phases.warm.min_age', + '*.modified_date', + ], + }; + + const response = await this.esClient.ilm.getLifecycle(request); + try { + for (const [policyName, stats] of Object.entries(response ?? {})) { + yield { + policy_name: policyName, + modified_date: stats.modified_date, + phases: { + cold: phase(stats.policy.phases.cold), + delete: phase(stats.policy.phases.delete), + frozen: phase(stats.policy.phases.frozen), + hot: phase(stats.policy.phases.hot), + warm: phase(stats.policy.phases.warm), + } as IlmPhases, + } as IlmPolicy; + } + } catch (error) { + this.logger.error('Error fetching ilm policies', { + error_message: error.message, + } as LogMeta); + throw error; + } + } + } + + private chunkStringsByMaxLength(strings: string[], maxLength: number = 3072): string[][] { + // plus 1 for the comma separator + return chunkedBy(strings, maxLength, (index) => index.length + 1); + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/sender.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/sender.ts new file mode 100644 index 0000000000000..27768c0e3be36 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/sender.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AnalyticsServiceStart, EventTypeOpts, LogMeta, Logger } from '@kbn/core/server'; + +export class MetadataSender { + private readonly logger: Logger; + + constructor(logger: Logger, private readonly analytics: AnalyticsServiceStart) { + this.logger = logger.get(MetadataSender.name); + } + + public reportEBT(eventTypeOpts: EventTypeOpts, eventData: T): void { + this.logger.debug('Reporting event', { eventType: eventTypeOpts.eventType } as LogMeta); + this.analytics.reportEvent(eventTypeOpts.eventType, eventData as object); + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/utils.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/utils.ts new file mode 100644 index 0000000000000..af9589e88f8f5 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/lib/utils.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Utility class to accumulate and manage weighted chunks of items. + * + * @template T The type of elements being chunked. + */ +export class Chunked { + public weight: number = 0; + + constructor(public chunks: T[][] = [], public current: T[] = []) {} + + /** + * Finalizes the current chunk (if not empty), adds it to `chunks`, and returns all non-empty chunks. + */ + public flush(): T[][] { + if (this.current.length !== 0) { + this.chunks.push(this.current); + this.current = []; + } + return this.chunks.filter((chunk) => chunk.length > 0); + } +} + +/** + * Splits a list of items into weighted chunks, where the sum of weights in each chunk does not exceed the specified size. + * + * @template T The type of elements in the list. + * @param list The array of items to be chunked. + * @param size The maximum allowed sum of weights in a chunk. + * @param weight A function returning the weight of each item. + * @returns An array of chunks, each chunk being an array of items. + * + * Iterates through the list, accumulating items into a chunk as long as the total weight does not exceed `size`. + * If adding an item would exceed the limit, the current chunk is finalized and a new chunk is started. + * Uses the Chunked class internally to manage chunk state. + */ +export function chunkedBy(list: T[], size: number, weight: (v: T) => number): T[][] { + function chunk(acc: Chunked, value: T): Chunked { + const currentWeight = weight(value); + if (acc.weight + currentWeight <= size) { + acc.current.push(value); + acc.weight += currentWeight; + } else { + acc.chunks.push(acc.current); + acc.current = [value]; + acc.weight = currentWeight; + } + return acc; + } + + return list.reduce(chunk, new Chunked()).flush(); +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/plugin.ts b/x-pack/platform/plugins/private/indices_metadata/server/plugin.ts new file mode 100644 index 0000000000000..3af6b59ac04bb --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/plugin.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Observable } from 'rxjs'; +import { + type CoreSetup, + type CoreStart, + type PluginInitializerContext, + type Plugin, +} from '@kbn/core/server'; +import type { Logger, LogMeta } from '@kbn/core/server'; +import { IndicesMetadataService } from './lib/services/indices_metadata'; +import { registerEbtEvents } from './lib/ebt/events'; +import type { + IndicesMetadataPluginSetup, + IndicesMetadataPluginStart, + IndicesMetadataPluginSetupDeps, + IndicesMetadataPluginStartDeps, +} from './plugin.types'; +import { DEFAULT_CDN_CONFIG, DEFAULT_INDICES_METADATA_CONFIGURATION } from './lib/constants'; +import { PluginConfig } from './config'; +import { CdnConfig } from './lib/services/artifact.types'; +import { ArtifactService } from './lib/services/artifact'; +import { ConfigurationService } from './lib/services/configuration'; + +export class IndicesMetadataPlugin + implements + Plugin< + IndicesMetadataPluginSetup, + IndicesMetadataPluginStart, + IndicesMetadataPluginSetupDeps, + IndicesMetadataPluginStartDeps + > +{ + private readonly logger: Logger; + private readonly config$: Observable; + + private readonly indicesMetadataService: IndicesMetadataService; + private readonly configurationService: ConfigurationService; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + this.config$ = context.config.create(); + + this.configurationService = new ConfigurationService(this.logger); + this.indicesMetadataService = new IndicesMetadataService( + this.logger, + this.configurationService + ); + } + + public setup(core: CoreSetup, plugin: IndicesMetadataPluginSetupDeps) { + const { taskManager } = plugin; + + this.indicesMetadataService.setup(taskManager); + + registerEbtEvents(core.analytics); + } + + public start(core: CoreStart, plugin: IndicesMetadataPluginStartDeps) { + this.logger.debug('Starting indices metadata plugin'); + + this.config$.subscribe(async (pluginConfig) => { + this.logger.debug('PluginConfig changed', { pluginConfig } as LogMeta); + + if (pluginConfig.enabled) { + this.logger.info('Updating indices metadata configuration'); + + const cdnConfig = this.effectiveCdnConfig(pluginConfig); + const info = await core.elasticsearch.client.asInternalUser.info(); + const artifactService = new ArtifactService(this.logger, info, cdnConfig); + + this.configurationService.start(artifactService, DEFAULT_INDICES_METADATA_CONFIGURATION); + this.indicesMetadataService.start( + plugin.taskManager, + core.analytics, + core.elasticsearch.client.asInternalUser + ); + } else { + this.logger.info('Indices metadata plugin is disabled, stopping services'); + this.configurationService.stop(); + this.indicesMetadataService.stop(); + } + }); + } + + public stop() { + this.logger.debug('Stopping indices metadata plugin'); + this.indicesMetadataService.stop(); + this.configurationService.stop(); + } + + private effectiveCdnConfig({ cdn }: PluginConfig): CdnConfig { + return { + url: cdn?.url ?? DEFAULT_CDN_CONFIG.url, + pubKey: cdn?.publicKey ?? DEFAULT_CDN_CONFIG.pubKey, + requestTimeout: cdn?.requestTimeout ?? DEFAULT_CDN_CONFIG.requestTimeout, + }; + } +} diff --git a/x-pack/platform/plugins/private/indices_metadata/server/plugin.types.ts b/x-pack/platform/plugins/private/indices_metadata/server/plugin.types.ts new file mode 100644 index 0000000000000..6b55f59053371 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/server/plugin.types.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; + +export type IndicesMetadataPluginSetup = void; + +export type IndicesMetadataPluginStart = void; + +export interface IndicesMetadataPluginSetupDeps { + taskManager: TaskManagerSetupContract; +} + +export interface IndicesMetadataPluginStartDeps { + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/platform/plugins/private/indices_metadata/tsconfig.json b/x-pack/platform/plugins/private/indices_metadata/tsconfig.json new file mode 100644 index 0000000000000..92e1de315bbb5 --- /dev/null +++ b/x-pack/platform/plugins/private/indices_metadata/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "server/**/*" + ], + "kbn_references": [ + "@kbn/core", + "@kbn/task-manager-plugin", + "@kbn/config-schema" + ], + "exclude": [ + "target/**/*" + ] +} diff --git a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index d6138ab50a35a..3fc98fe9c8925 100644 --- a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -51,6 +51,7 @@ export default function ({ getService }: FtrProviderContext) { 'Fleet-Metrics-Task', 'Fleet-Usage-Logger', 'Fleet-Usage-Sender', + 'IndicesMetadata:IndicesMetadataTask', 'ML:saved-objects-sync', 'ProductDocBase:EnsureUpToDate', 'ProductDocBase:InstallAll', diff --git a/yarn.lock b/yarn.lock index 3aa2e710116c3..bbb03d00eb4be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6036,6 +6036,10 @@ version "0.0.0" uid "" +"@kbn/indices-metadata-plugin@link:x-pack/platform/plugins/private/indices_metadata": + version "0.0.0" + uid "" + "@kbn/inference-cli@link:x-pack/platform/packages/shared/kbn-inference-cli": version "0.0.0" uid ""