[Fleet] Install only OTel component template mappings for otelcol inputs (#237165)

Use OTel mappings more strictly in OTel integrations. Main index
template is still managed by Fleet, but only OTel component templates
are used for mappings. Other templates are still used for Fleet-managed
settings.
Packages are not expected to define fields with this approach, but they
still can.

Fields, and `subobjects: false` are removed from test package. This
needs to be adjusted in a follow up in package spec as packages are
required to contain fields now.

Author: jsoriano

x-pack/platform/plugins/shared/fleet/cypress/e2e/input_packages_real.cy.ts

@@ -131,7 +131,7 @@ describe('OTel input package with custom data stream type', () => {
   const agentPolicyId = 'test-otel-input-package-policy';
   const agentPolicyName = 'Test input package policy';
   const packagePolicyName = 'input-package-policy';
-  const datasetName = 'check'; // Default from the package.
+  const datasetName = 'httpcheckreceiver'; // Default from the package.
   const dataStreamType = 'logs';
 
   before(() => {

x-pack/platform/plugins/shared/fleet/cypress/packages/otel_input_package-1.0.0.zip

@@ -153,8 +153,6 @@ export const OTEL_LOGS_COMPONENT_TEMPLATES = [
   OTEL_COMPONENT_SEMCONV_RESOURCE_TO_ECS_MAPPINGS,
 ];
 export const OTEL_TRACES_COMPONENT_TEMPLATES = [
-  STACK_COMPONENT_TEMPLATE_TRACES_MAPPINGS,
-  STACK_COMPONENT_TEMPLATE_TRACES_SETTINGS,
   OTEL_COMPONENT_TEMPLATE_MAPPINGS,
   OTEL_COMPONENT_TEMPLATE_SETTINGS,
   OTEL_COMPONENT_SEMCONV_RESOURCE_TO_ECS_MAPPINGS,

x-pack/platform/plugins/shared/fleet/server/constants/fleet_es_assets.ts

@@ -647,7 +647,6 @@ describe('EPM index template install', () => {
           },
           template: {
             mappings: {
-              dynamic: true,
               dynamic_templates: undefined,
               properties: {
                 test_dimension: {
@@ -798,7 +797,6 @@ describe('EPM index template install', () => {
           },
           template: {
             mappings: {
-              dynamic: true,
               dynamic_templates: undefined,
               properties: {
                 test_dimension: {

x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts

@@ -428,17 +428,6 @@ export function buildComponentTemplates(params: {
       isSyntheticSourceEnabledByDefault ||
       isTimeSeriesEnabledByDefault);
 
-  const isPkgConfiguringDynamicSettings =
-    Object.keys(mappingsRuntimeFields).length > 0 || indexTemplateMappings?.dynamic !== undefined;
-
-  // Setting overrides for otel input packages, but only if the packages don't explicitly disable dynamic mappings or use `dynamic: runtime`
-  // Override the `dynamic: false` set in otel@mappings to avoid conflicts in case
-  const shouldOverrideSettingsForOtelInputs = isOtelInputType && !isPkgConfiguringDynamicSettings;
-
-  // Override `subobjects: false` to avoid conflicts with traces-otel@mappings
-  const shouldOverrideSettingsForOtelInputsTraces =
-    shouldOverrideSettingsForOtelInputs && type === 'traces' && !indexTemplateMappings.runtime;
-
   templatesMap[packageTemplateName] = {
     template: {
       settings: {
@@ -472,8 +461,6 @@ export function buildComponentTemplates(params: {
           : {}),
         dynamic_templates: mappingsDynamicTemplates.length ? mappingsDynamicTemplates : undefined,
         ...omit(indexTemplateMappings, 'properties', 'dynamic_templates', 'runtime'),
-        ...(shouldOverrideSettingsForOtelInputs ? { dynamic: true } : {}),
-        ...(shouldOverrideSettingsForOtelInputsTraces ? { subobjects: undefined } : {}),
       },
       ...(lifecycle ? { lifecycle } : {}),
     },

x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -27,9 +27,6 @@ import {
   OTEL_COMPONENT_TEMPLATE_TRACES_CUSTOM_MAPPINGS,
   OTEL_COMPONENT_TEMPLATE_TRACES_MAPPINGS,
   STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
-  STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS,
-  STACK_COMPONENT_TEMPLATE_TRACES_MAPPINGS,
-  STACK_COMPONENT_TEMPLATE_TRACES_SETTINGS,
 } from '../../../../constants/fleet_es_assets';
 
 import { createAppContextStartContractMock } from '../../../../mocks';
@@ -342,14 +339,11 @@ describe('EPM template', () => {
       });
 
       expect(template.composed_of).toStrictEqual([
-        STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
-        STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
         OTEL_COMPONENT_TEMPLATE_MAPPINGS,
         OTEL_COMPONENT_TEMPLATE_SETTINGS,
         OTEL_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
         OTEL_COMPONENT_SEMCONV_RESOURCE_TO_ECS_MAPPINGS,
         ...composedOfTemplates,
-        STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
         FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
         FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
       ]);
@@ -373,14 +367,11 @@ describe('EPM template', () => {
         isOtelInputType: true,
       });
       expect(template.composed_of).toStrictEqual([
-        STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS,
         OTEL_COMPONENT_TEMPLATE_MAPPINGS,
         OTEL_COMPONENT_TEMPLATE_SETTINGS,
         OTEL_COMPONENT_TEMPLATE_METRICS_MAPPINGS,
         OTEL_COMPONENT_SEMCONV_RESOURCE_TO_ECS_MAPPINGS,
         ...composedOfTemplates,
-
-        STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
         FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
         FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
       ]);
@@ -404,15 +395,11 @@ describe('EPM template', () => {
         isOtelInputType: true,
       });
       expect(template.composed_of).toStrictEqual([
-        STACK_COMPONENT_TEMPLATE_TRACES_MAPPINGS,
-        STACK_COMPONENT_TEMPLATE_TRACES_SETTINGS,
         OTEL_COMPONENT_TEMPLATE_MAPPINGS,
         OTEL_COMPONENT_TEMPLATE_SETTINGS,
         OTEL_COMPONENT_SEMCONV_RESOURCE_TO_ECS_MAPPINGS,
         OTEL_COMPONENT_TEMPLATE_TRACES_MAPPINGS,
         ...composedOfTemplates,
-
-        STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
         FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
         FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
       ]);

x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -121,16 +121,15 @@ export function getTemplate({
     );
   }
 
-  const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries);
+  const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries, isOtelInputType);
 
   const isEventIngestedEnabled = (config?: FleetConfigType): boolean =>
     Boolean(!config?.agentIdVerificationEnabled && config?.eventIngestedEnabled);
 
   template.composed_of = [
     ...esBaseComponents,
-    ...(isOtelInputType ? getOtelBaseComponents(type) : []),
     ...(template.composed_of || []),
-    STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
+    ...(isOtelInputType ? [] : [STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS]),
     FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
     ...(appContextService.getConfig()?.agentIdVerificationEnabled
       ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME]
@@ -144,7 +143,15 @@ export function getTemplate({
   return template;
 }
 
-const getBaseEsComponents = (type: string, isIndexModeTimeSeries: boolean): string[] => {
+const getBaseEsComponents = (
+  type: string,
+  isIndexModeTimeSeries: boolean,
+  isOTelInputType?: boolean
+): string[] => {
+  if (isOTelInputType) {
+    return getOtelBaseComponents(type);
+  }
+
   if (type === 'metrics') {
     if (isIndexModeTimeSeries) {
       return [STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS];