diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 50dd09a6366e8..1427c8cee8bd7 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -113,15 +113,11 @@ export class APMPlugin const getCoreStart = () => core.getStartServices().then(([coreStart]) => coreStart); - const alertsIndexPattern = ruleDataService.getFullAssetName( - 'observability-apm*' + const assetName = 'observability-apm'; + const componentTemplateName = ruleDataService.getFullAssetName( + 'apm-mappings' ); - const initializeRuleDataTemplates = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName( - 'apm-mappings' - ); - if (!ruleDataService.isWriteEnabled()) { return; } @@ -154,19 +150,7 @@ export class APMPlugin }, }); - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName('apm-index-template'), - body: { - index_patterns: [alertsIndexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], - }, - }); - await ruleDataService.updateIndexMappingsMatchingPattern( - alertsIndexPattern - ); + await ruleDataService.updateIndexMappingsForAsset(assetName); }); // initialize eagerly @@ -178,8 +162,12 @@ export class APMPlugin const ruleDataClient = ruleDataService.getRuleDataClient( APM_SERVER_FEATURE_ID, - ruleDataService.getFullAssetName('observability-apm'), - () => initializeRuleDataTemplatesPromise + assetName, + () => initializeRuleDataTemplatesPromise, + [ + ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + componentTemplateName, + ] ); const resourcePlugins = mapValues(plugins, (value, key) => { diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts index f67837cff0df1..f5a6dee73a323 100644 --- a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts +++ b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts @@ -25,13 +25,10 @@ export const createRuleDataClient = ({ logger: Logger; ruleDataService: RuleRegistryPluginSetupContract['ruleDataService']; }) => { + const componentTemplateName = ruleDataService.getFullAssetName( + `${registrationContext}-mappings` + ); const initializeRuleDataTemplates = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName( - `${registrationContext}-mappings` - ); - - const indexNamePattern = ruleDataService.getFullAssetName(`${registrationContext}*`); - if (!ruleDataService.isWriteEnabled()) { return; } @@ -48,7 +45,7 @@ export const createRuleDataClient = ({ }, }); - await ruleDataService.createOrUpdateIndexTemplate({ + /*await ruleDataService.createOrUpdateIndexTemplate({ name: ruleDataService.getFullAssetName(registrationContext), body: { index_patterns: [indexNamePattern], @@ -57,9 +54,9 @@ export const createRuleDataClient = ({ componentTemplateName, ], }, - }); + });*/ - await ruleDataService.updateIndexMappingsMatchingPattern(indexNamePattern); + await ruleDataService.updateIndexMappingsForAsset(registrationContext); }); // initialize eagerly @@ -69,7 +66,11 @@ export const createRuleDataClient = ({ return ruleDataService.getRuleDataClient( ownerFeatureId, - ruleDataService.getFullAssetName(registrationContext), - () => initializeRuleDataTemplatesPromise + registrationContext, + () => initializeRuleDataTemplatesPromise, + [ + ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + componentTemplateName, + ] ); }; diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 868e234fcb2a1..8fdc63041add8 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -100,10 +100,12 @@ export class ObservabilityPlugin implements Plugin { const start = () => core.getStartServices().then(([coreStart]) => coreStart); + // ???? This appears to be a stub const ruleDataClient = plugins.ruleRegistry.ruleDataService.getRuleDataClient( 'observability', - plugins.ruleRegistry.ruleDataService.getFullAssetName(), - () => Promise.resolve() + '', + () => Promise.resolve(), + [], ); registerRoutes({ diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts index a9e559a6b1932..e5a2e1ee53cc0 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts @@ -5,8 +5,10 @@ * 2.0. */ +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/api/types'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; +import { DEFAULT_ILM_POLICY_ID } from '../../common/assets'; import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors'; import { IRuleDataClient, @@ -111,24 +113,86 @@ export class RuleDataClient implements IRuleDataClient { }; } + createNamespacedIndexTemplate({ + primaryNamespacedAlias, + secondaryNamespacedAlias, + namespace, + }: { + primaryNamespacedAlias: string; + secondaryNamespacedAlias?: string; + namespace?: string; + }): IndicesPutIndexTemplateRequest { + return { + name: primaryNamespacedAlias, + body: { + index_patterns: [`${primaryNamespacedAlias}-*`], + composed_of: [...this.options.componentTemplateNames], + template: { + aliases: + secondaryNamespacedAlias != null + ? { + [secondaryNamespacedAlias]: { + is_write_index: false, + }, + } + : undefined, + settings: { + 'index.lifecycle': { + name: DEFAULT_ILM_POLICY_ID, + // TODO: fix the types in the ES package, they don't include rollover_alias??? + // @ts-expect-error + rollover_alias: primaryNamespacedAlias, + }, + }, + }, + _meta: { + namespace, + }, + // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace + // then newly created indices will use the matching template with the *longest* namespace + priority: namespace?.length, + }, + }; + } + async createWriteTargetIfNeeded({ namespace }: { namespace?: string }) { - const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); + const primaryNamespacedAlias = getNamespacedAlias({ alias: this.options.alias, namespace }); const clusterClient = await this.getClusterClient(); const { body: aliasExists } = await clusterClient.indices.existsAlias({ - name: alias, + name: primaryNamespacedAlias, }); - const concreteIndexName = `${alias}-000001`; + const concreteIndexName = `${primaryNamespacedAlias}-000001`; if (!aliasExists) { + const secondaryNamespacedAlias = + this.options.secondaryAlias != null + ? getNamespacedAlias({ alias: this.options.secondaryAlias, namespace }) + : undefined; + const template = this.createNamespacedIndexTemplate({ + primaryNamespacedAlias, + secondaryNamespacedAlias, + namespace, + }); + // TODO: need a way to update this template if/when we decide to make changes to the + // built in index template. Probably do it as part of updateIndexMappingsForAsset? + // (Before upgrading any indices, find and upgrade all namespaced index templates - component templates + // will already have been upgraded by solutions or rule registry, in the case of technical/ECS templates) + // With the current structure, it's tricky because the index template creation + // depends on both the namespace and secondary alias, both of which are not currently available + // to updateIndexMappingsForAsset. We can make the secondary alias available since + // it's known at plugin startup time, but + // the namespace values can really only come from the existing templates that we're trying to update + // - maybe we want to store the namespace as a _meta field on the index template for easy retrieval + await clusterClient.indices.putIndexTemplate(template); try { await clusterClient.indices.create({ index: concreteIndexName, body: { aliases: { - [alias]: { + [primaryNamespacedAlias]: { is_write_index: true, }, }, diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 277121074f7f2..d2a0b24166114 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -51,6 +51,8 @@ export interface RuleDataClientConstructorOptions { getClusterClient: () => Promise; isWriteEnabled: boolean; ready: () => Promise; + componentTemplateNames: string[]; alias: string; feature: ValidFeatureId; + secondaryAlias?: string; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts index f81340889e4b5..a8fcd79eab51b 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts @@ -195,10 +195,11 @@ export class RuleDataPluginService { return this._createOrUpdateLifecyclePolicy(policy); } - async updateIndexMappingsMatchingPattern(pattern: string) { + async updateIndexMappingsForAsset(assetName: string) { await this.wait(); const clusterClient = await this.getClusterClient(); - const { body: aliasesResponse } = await clusterClient.indices.getAlias({ index: pattern }); + const pattern = `${this.getFullAssetName(assetName)}-*`; + const { body: aliasesResponse } = await clusterClient.indices.getAlias({ name: pattern }); const writeIndicesAndAliases: Array<{ index: string; alias: string }> = []; Object.entries(aliasesResponse).forEach(([index, aliases]) => { Object.entries(aliases.aliases).forEach(([aliasName, aliasProperties]) => { @@ -235,13 +236,21 @@ export class RuleDataPluginService { return [this.options.index, assetName].filter(Boolean).join('-'); } - getRuleDataClient(feature: ValidFeatureId, alias: string, initialize: () => Promise) { + getRuleDataClient( + feature: ValidFeatureId, + assetName: string, + initialize: () => Promise, + componentTemplateNames: string[], + secondaryAlias?: string + ) { return new RuleDataClient({ - alias, + alias: this.getFullAssetName(assetName), feature, getClusterClient: () => this.getClusterClient(), isWriteEnabled: this.isWriteEnabled(), ready: initialize, + componentTemplateNames, + secondaryAlias, }); } } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts index 348c0fc6b1cfc..15a7b1da86ce9 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts @@ -21,7 +21,7 @@ const createRuleDataPluginService = () => { createOrUpdateIndexTemplate: jest.fn(), createOrUpdateLifecyclePolicy: jest.fn(), getRuleDataClient: jest.fn(), - updateIndexMappingsMatchingPattern: jest.fn(), + updateIndexMappingsForAsset: jest.fn(), }; return mocked; }; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index fd3a32a2fa689..b7192ff75526e 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -203,14 +203,12 @@ export class Plugin implements IPlugin { if (!ruleDataService.isWriteEnabled()) { return; } - const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings'); await ruleDataService.createOrUpdateComponentTemplate({ name: componentTemplateName, @@ -224,7 +222,7 @@ export class Plugin implements IPlugin initializeRuleDataTemplatesPromise + assetName, + () => initializeRuleDataTemplatesPromise, + [ + ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + ruleDataService.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME), + componentTemplateName, + ], + config.signalsIndex ); // Register rule types via rule-registry diff --git a/x-pack/plugins/uptime/server/plugin.ts b/x-pack/plugins/uptime/server/plugin.ts index d4bc9ca1b4d5d..0f4313157485e 100644 --- a/x-pack/plugins/uptime/server/plugin.ts +++ b/x-pack/plugins/uptime/server/plugin.ts @@ -34,11 +34,9 @@ export class Plugin implements PluginType { public setup(core: CoreSetup, plugins: UptimeCorePlugins) { this.logger = this.initContext.logger.get(); const { ruleDataService } = plugins.ruleRegistry; - + const assetName = 'observability.synthetics'; + const componentTemplateName = ruleDataService.getFullAssetName('synthetics-mappings'); const ready = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName('synthetics-mappings'); - const alertsIndexPattern = ruleDataService.getFullAssetName('observability.synthetics*'); - if (!ruleDataService.isWriteEnabled()) { return; } @@ -55,18 +53,7 @@ export class Plugin implements PluginType { }, }); - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName('synthetics-index-template'), - body: { - index_patterns: [alertsIndexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], - }, - }); - - await ruleDataService.updateIndexMappingsMatchingPattern(alertsIndexPattern); + await ruleDataService.updateIndexMappingsForAsset(assetName); }); // initialize eagerly @@ -76,8 +63,9 @@ export class Plugin implements PluginType { const ruleDataClient = ruleDataService.getRuleDataClient( 'synthetics', - ruleDataService.getFullAssetName('observability.synthetics'), - () => initializeRuleDataTemplatesPromise + assetName, + () => initializeRuleDataTemplatesPromise, + [ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), componentTemplateName] ); initServerWithKibana(