elastic · kibanamachine · Jun 23, 2021 · Jun 23, 2021
diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { estypes } from '@elastic/elasticsearch';
 // Follow pattern from https://github.com/elastic/kibana/pull/52447
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import type { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
@@ -299,8 +300,8 @@ export interface RegistryDataStream {
 }
 
 export interface RegistryElasticsearch {
-  'index_template.settings'?: object;
-  'index_template.mappings'?: object;
+  'index_template.settings'?: estypes.IndicesIndexSettings;
+  'index_template.mappings'?: estypes.MappingTypeMapping;
 }
 
 export interface RegistryDataStreamPermissions {
@@ -425,7 +426,7 @@ export interface IndexTemplate {
   _meta: object;
 }
 
-export interface TemplateRef {
+export interface IndexTemplateEntry {
   templateName: string;
   indexTemplate: IndexTemplate;
 }
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
@@ -11,15 +11,15 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s
 import { ElasticsearchAssetType } from '../../../../types';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   RegistryElasticsearch,
   InstallablePackage,
 } from '../../../../types';
 import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
-import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install';
+import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install';
 
 import {
   generateMappings,
@@ -34,52 +34,44 @@ export const installTemplates = async (
   esClient: ElasticsearchClient,
   paths: string[],
   savedObjectsClient: SavedObjectsClientContract
-): Promise<TemplateRef[]> => {
+): Promise<IndexTemplateEntry[]> => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global index templates
   // Install component templates first, as they are used by the index templates
   await installPreBuiltComponentTemplates(paths, esClient);
   await installPreBuiltTemplates(paths, esClient);
 
   // remove package installation's references to index templates
-  await removeAssetsFromInstalledEsByType(
-    savedObjectsClient,
-    installablePackage.name,
-    ElasticsearchAssetType.indexTemplate
-  );
+  await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [
+    ElasticsearchAssetType.indexTemplate,
+    ElasticsearchAssetType.componentTemplate,
+  ]);
   // build templates per data stream from yml files
   const dataStreams = installablePackage.data_streams;
   if (!dataStreams) return [];
+
+  const installedTemplatesNested = await Promise.all(
+    dataStreams.map((dataStream) =>
+      installTemplateForDataStream({
+        pkg: installablePackage,
+        esClient,
+        dataStream,
+      })
+    )
+  );
+  const installedTemplates = installedTemplatesNested.flat();
+
   // get template refs to save
-  const installedTemplateRefs = dataStreams.map((dataStream) => ({
-    id: generateTemplateName(dataStream),
-    type: ElasticsearchAssetType.indexTemplate,
-  }));
+  const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates);
 
   // add package installation's references to index templates
-  await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs);
-
-  if (dataStreams) {
-    const installTemplatePromises = dataStreams.reduce<Array<Promise<TemplateRef>>>(
-      (acc, dataStream) => {
-        acc.push(
-          installTemplateForDataStream({
-            pkg: installablePackage,
-            esClient,
-            dataStream,
-          })
-        );
-        return acc;
-      },
-      []
-    );
-
-    const res = await Promise.all(installTemplatePromises);
-    const installedTemplates = res.flat();
+  await saveInstalledEsRefs(
+    savedObjectsClient,
+    installablePackage.name,
+    installedIndexTemplateRefs
+  );
 
-    return installedTemplates;
-  }
-  return [];
+  return installedTemplates;
 };
 
 const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => {
@@ -160,7 +152,7 @@ export async function installTemplateForDataStream({
   pkg: InstallablePackage;
   esClient: ElasticsearchClient;
   dataStream: RegistryDataStream;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const fields = await loadFieldsFromYaml(pkg, dataStream.path);
   return installTemplate({
     esClient,
@@ -171,84 +163,118 @@ export async function installTemplateForDataStream({
   });
 }
 
+interface TemplateMapEntry {
+  _meta: { package: { name: string } };
+  template:
+    | {
+        mappings: NonNullable<RegistryElasticsearch['index_template.mappings']>;
+      }
+    | {
+        settings: NonNullable<RegistryElasticsearch['index_template.settings']> | object;
+      };
+}
+type TemplateMap = Record<string, TemplateMapEntry>;
 function putComponentTemplate(
-  body: object | undefined,
-  name: string,
-  esClient: ElasticsearchClient
-): { clusterPromise: Promise<any>; name: string } | undefined {
-  if (body) {
-    const esClientParams = {
-      name,
-      body,
-    };
-
-    return {
-      // @ts-expect-error body expected to be ClusterPutComponentTemplateRequest
-      clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }),
-      name,
-    };
+  esClient: ElasticsearchClient,
+  params: {
+    body: TemplateMapEntry;
+    name: string;
+    create?: boolean;
   }
+): { clusterPromise: Promise<any>; name: string } {
+  const { name, body, create = false } = params;
+  return {
+    clusterPromise: esClient.cluster.putComponentTemplate(
+      // @ts-expect-error body is missing required key `settings`. TemplateMapEntry has settings *or* mappings
+      { name, body, create },
+      { ignore: [404] }
+    ),
+    name,
+  };
 }
 
-function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
-  let mappingsTemplate;
-  let settingsTemplate;
+const mappingsSuffix = '@mappings';
+const settingsSuffix = '@settings';
+const userSettingsSuffix = '@custom';
+type TemplateBaseName = string;
+type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`;
+
+const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
+  name.endsWith(userSettingsSuffix);
+
+function buildComponentTemplates(params: {
+  templateName: string;
+  registryElasticsearch: RegistryElasticsearch | undefined;
+  packageName: string;
+}) {
+  const { templateName, registryElasticsearch, packageName } = params;
+  const mappingsTemplateName = `${templateName}${mappingsSuffix}`;
+  const settingsTemplateName = `${templateName}${settingsSuffix}`;
+  const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`;
+
+  const templatesMap: TemplateMap = {};
+  const _meta = { package: { name: packageName } };
 
   if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
-    mappingsTemplate = {
+    templatesMap[mappingsTemplateName] = {
       template: {
-        mappings: {
-          ...registryElasticsearch['index_template.mappings'],
-        },
+        mappings: registryElasticsearch['index_template.mappings'],
       },
+      _meta,
     };
   }
 
   if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
-    settingsTemplate = {
+    templatesMap[settingsTemplateName] = {
       template: {
         settings: registryElasticsearch['index_template.settings'],
       },
+      _meta,
     };
   }
-  return { settingsTemplate, mappingsTemplate };
-}
 
-async function installDataStreamComponentTemplates(
-  templateName: string,
-  registryElasticsearch: RegistryElasticsearch | undefined,
-  esClient: ElasticsearchClient
-) {
-  const templates: string[] = [];
-  const componentPromises: Array<Promise<any>> = [];
+  // return empty/stub template
+  templatesMap[userSettingsTemplateName] = {
+    template: {
+      settings: {},
+    },
+    _meta,
+  };
 
-  const compTemplates = buildComponentTemplates(registryElasticsearch);
+  return templatesMap;
+}
 
-  const mappings = putComponentTemplate(
-    compTemplates.mappingsTemplate,
-    `${templateName}-mappings`,
-    esClient
-  );
+async function installDataStreamComponentTemplates(params: {
+  templateName: string;
+  registryElasticsearch: RegistryElasticsearch | undefined;
+  esClient: ElasticsearchClient;
+  packageName: string;
+}) {
+  const { templateName, registryElasticsearch, esClient, packageName } = params;
+  const templates = buildComponentTemplates({ templateName, registryElasticsearch, packageName });
+  const templateNames = Object.keys(templates);
+  const templateEntries = Object.entries(templates);
 
-  const settings = putComponentTemplate(
-    compTemplates.settingsTemplate,
-    `${templateName}-settings`,
-    esClient
+  // TODO: Check return values for errors
+  await Promise.all(
+    templateEntries.map(async ([name, body]) => {
+      if (isUserSettingsTemplate(name)) {
+        // look for existing user_settings template
+        const result = await esClient.cluster.getComponentTemplate({ name }, { ignore: [404] });
+        const hasUserSettingsTemplate = result.body.component_templates?.length === 1;
+        if (!hasUserSettingsTemplate) {
+          // only add if one isn't already present
+          const { clusterPromise } = putComponentTemplate(esClient, { body, name, create: true });
+          return clusterPromise;
+        }
+      } else {
+        const { clusterPromise } = putComponentTemplate(esClient, { body, name });
+        return clusterPromise;
+      }
+    })
   );
 
-  if (mappings) {
-    templates.push(mappings.name);
-    componentPromises.push(mappings.clusterPromise);
-  }
-
-  if (settings) {
-    templates.push(settings.name);
-    componentPromises.push(settings.clusterPromise);
-  }
-
-  // TODO: Check return values for errors
-  await Promise.all(componentPromises);
-  return templates;
+  return templateNames;
 }
 
 export async function installTemplate({
@@ -263,7 +289,7 @@ export async function installTemplate({
   dataStream: RegistryDataStream;
   packageVersion: string;
   packageName: string;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const validFields = processFields(fields);
   const mappings = generateMappings(validFields);
   const templateName = generateTemplateName(dataStream);
@@ -310,11 +336,12 @@ export async function installTemplate({
     await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] });
   }
 
-  const composedOfTemplates = await installDataStreamComponentTemplates(
+  const composedOfTemplates = await installDataStreamComponentTemplates({
     templateName,
-    dataStream.elasticsearch,
-    esClient
-  );
+    registryElasticsearch: dataStream.elasticsearch,
+    esClient,
+    packageName,
+  });
 
   const template = getTemplate({
     type: dataStream.type,
@@ -342,3 +369,21 @@ export async function installTemplate({
     indexTemplate: template,
   };
 }
+
+export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
+  return installedTemplates.flatMap((installedTemplate) => {
+    const indexTemplates = [
+      {
+        id: installedTemplate.templateName,
+        type: ElasticsearchAssetType.indexTemplate,
+      },
+    ];
+    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
+      (componentTemplateId) => ({
+        id: componentTemplateId,
+        type: ElasticsearchAssetType.componentTemplate,
+      })
+    );
+    return indexTemplates.concat(componentTemplates);
+  });
+}
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -10,7 +10,7 @@ import type { ElasticsearchClient } from 'kibana/server';
 import type { Field, Fields } from '../../fields/field';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   IndexTemplate,
   IndexTemplateMappings,
 } from '../../../../types';
@@ -456,7 +456,7 @@ function getBaseTemplate(
 
 export const updateCurrentWriteIndices = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<void> => {
   if (!templates.length) return;
 
@@ -471,7 +471,7 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur
 
 const queryDataStreamsFromTemplates = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<CurrentDataStream[]> => {
   const dataStreamPromises = templates.map((template) => {
     return getDataStreams(esClient, template);
@@ -482,7 +482,7 @@ const queryDataStreamsFromTemplates = async (
 
 const getDataStreams = async (
   esClient: ElasticsearchClient,
-  template: TemplateRef
+  template: IndexTemplateEntry
 ): Promise<CurrentDataStream[] | undefined> => {
   const { templateName, indexTemplate } = template;
   const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` });