From db96f181a414a147b5856a04b286756be699da0d Mon Sep 17 00:00:00 2001 From: Tere Date: Mon, 20 Apr 2026 12:27:03 +0200 Subject: [PATCH 1/2] [Fleet] Resolve streams and secrets by effective input name Follow up to named/qualified inputs (name ?? type): registry data stream streams reference streams[].input by effective name while package policy inputs still carry the real agent type. Align stream template compilation, stream var validation, stream secret var lookup, and EPM template indexing with getInputEffectiveName(input). Adds a Jest case for manifests whose stream.input uses the input name. Made-with: Cursor --- .../services/validate_package_policy.ts | 2 +- .../epm/packages/get_template_inputs.ts | 8 +-- .../server/services/package_policy.test.ts | 49 +++++++++++++++++++ .../fleet/server/services/package_policy.ts | 5 +- .../services/secrets/package_policies.ts | 5 +- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts index 408c677d75a3a..678cb71a1d038 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts @@ -481,7 +481,7 @@ export const validatePackagePolicy = ( if (input.streams.length) { input.streams.forEach((stream) => { const streamValidationResults: PackagePolicyConfigValidationResults = {}; - const streamKey = `${stream.data_stream.dataset}-${input.type}`; + const streamKey = `${stream.data_stream.dataset}-${getInputEffectiveName(input)}`; const streamVarDefs = streamVarDefsByDatasetAndInput[streamKey]; const streamVarGroups = streamVarGroupsByDatasetAndInput[streamKey]; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts index fab04416127af..f283321211430 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts @@ -17,6 +17,7 @@ import { import { getStreamsForInputType, + getInputEffectiveName, packageToPackagePolicy, } from '../../../../common/services/package_to_package_policy'; import { _compilePackagePolicyInputs } from '../../package_policy'; @@ -292,10 +293,11 @@ function buildIndexedPackage(packageInfo: PackageInfo): PackageWithInputAndStrea const inputs = getNormalizedInputs(policyTemplate); inputs.forEach((packageInput) => { - const inputId = `${policyTemplate.name}-${packageInput.type}`; + const inputEffectiveName = getInputEffectiveName(packageInput); + const inputId = `${policyTemplate.name}-${inputEffectiveName}`; const streams = getStreamsForInputType( - packageInput.type, + inputEffectiveName, packageInfo, isIntegrationPolicyTemplate(policyTemplate) && policyTemplate.data_streams ? policyTemplate.data_streams @@ -308,7 +310,7 @@ function buildIndexedPackage(packageInfo: PackageInfo): PackageWithInputAndStrea } > >((acc, stream) => { - const streamId = `${packageInput.type}-${stream.data_stream.dataset}`; + const streamId = `${inputEffectiveName}-${stream.data_stream.dataset}`; acc[streamId] = { ...stream, }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts index 10a1ad8f46262..3f69a2da79b12 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts @@ -2143,6 +2143,55 @@ describe('Package policy service', () => { ]); }); + it('should compile stream templates when stream.input references a named input (name ?? type)', async () => { + const inputs = await _compilePackagePolicyInputs( + { + name: 'test', + version: '1.0.0', + data_streams: [ + { + type: 'logs', + dataset: 'package.dataset1', + streams: [{ input: 'named_logfile', template_path: 'some_template_path.yml' }], + path: 'dataset1', + }, + ], + policy_templates: [ + { + inputs: [{ type: 'log', name: 'named_logfile' }], + }, + ], + } as unknown as PackageInfo, + {}, + [ + { + type: 'log', + name: 'named_logfile', + enabled: true, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + vars: { + paths: { + value: ['/var/log/set.log'], + }, + }, + }, + ], + }, + ], + ASSETS_MAP_FIXTURES + ); + + expect(inputs[0].streams[0].compiled_stream).toEqual({ + metricset: ['dataset1'], + paths: ['/var/log/set.log'], + type: 'log', + }); + }); + it('should compile integration stream data_stream.dataset from package stream var default value', async () => { const pkgInfo = { name: 'dsvarpkg', diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts index e6662d97a4a52..2c09ecb0c64a0 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts @@ -3808,12 +3808,13 @@ function _compilePackageStream( stream = _applyIndexPrivileges(packageDataStream, streamIn); + const effectiveInputName = getInputEffectiveName(input); const streamFromPkg = (packageDataStream.streams || []).find( - (pkgStream) => pkgStream.input === input.type + (pkgStream) => pkgStream.input === effectiveInputName ); if (!streamFromPkg) { throw new StreamNotFoundError( - `Stream template not found, unable to find stream for input ${input.type}` + `Stream template not found, unable to find stream for input ${effectiveInputName} (type: ${input.type})` ); } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/secrets/package_policies.ts b/x-pack/platform/plugins/shared/fleet/server/services/secrets/package_policies.ts index 03e59698af77b..073fe9ee73ee7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/secrets/package_policies.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/secrets/package_policies.ts @@ -13,6 +13,7 @@ import type { NewPackagePolicy, RegistryStream, UpdatePackagePolicy } from '../. import { SO_SEARCH_LIMIT } from '../../../common'; import { doesPackageHaveIntegrations, + getInputEffectiveName, getNormalizedDataStreams, getNormalizedInputs, } from '../../../common/services'; @@ -391,7 +392,9 @@ function _getInputSecretPaths( if (input.streams.length) { input.streams.forEach((stream, streamIndex) => { const streamVarDefs = - streamSecretVarDefsByDatasetAndInput[`${stream.data_stream.dataset}-${input.type}`]; + streamSecretVarDefsByDatasetAndInput[ + `${stream.data_stream.dataset}-${getInputEffectiveName(input)}` + ]; if (streamVarDefs && Object.keys(streamVarDefs).length) { Object.entries(stream.vars || {}).forEach(([name, configEntry]) => { if (streamVarDefs[name]) { From a4a5e8e2298bba84442b1628503310091f7b5396 Mon Sep 17 00:00:00 2001 From: Tere Date: Mon, 20 Apr 2026 13:05:35 +0200 Subject: [PATCH 2/2] Fix Fleet template input keys to use getInputEffectiveName Align getTemplateInputs placeholder lookup and templatePackagePolicyToFullInputStreams input ids with buildIndexedPackage (policy_template + name ?? type) so inputs with explicit names still resolve registry vars and YAML comments. Made-with: Cursor --- .../epm/packages/get_template_inputs.ts | 10 ++++--- .../epm/packages/get_templates_inputs.test.ts | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts index f283321211430..7425f63e403d9 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts @@ -88,9 +88,10 @@ export const templatePackagePolicyToFullInputStreams = ( packagePolicyInputs.forEach((input) => { const streamsIdsMap = new Map(); + const inputEffectiveName = getInputEffectiveName(input); const inputId = input.policy_template - ? `${input.policy_template}-${input.type}` - : `${input.type}`; + ? `${input.policy_template}-${inputEffectiveName}` + : inputEffectiveName; const fullInputStream = { // @ts-ignore-next-line the following id is actually one level above the one in fullInputStream, but the linter thinks it gets overwritten id: inputId, @@ -166,9 +167,10 @@ export async function getTemplateInputs( if (format === 'yml') { // Add a placeholder to all variables without default value for (const inputWithStreamIds of inputsWithStreamIds) { + const inputEffectiveName = getInputEffectiveName(inputWithStreamIds); const inputId = inputWithStreamIds.policy_template - ? `${inputWithStreamIds.policy_template}-${inputWithStreamIds.type}` - : inputWithStreamIds.type; + ? `${inputWithStreamIds.policy_template}-${inputEffectiveName}` + : inputEffectiveName; const packageInput = indexedInputsAndStreams[inputId]; if (!packageInput) { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_templates_inputs.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_templates_inputs.test.ts index 1180fa8d302b1..4b0a448fe2270 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_templates_inputs.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_templates_inputs.test.ts @@ -221,6 +221,34 @@ describe('Fleet - templatePackagePolicyToFullInputStreams', () => { ]); }); + it('uses getInputEffectiveName for template input id when name differs from type', async () => { + expect( + await templatePackagePolicyToFullInputStreams([ + { + ...mockInput2, + name: 'custom-metrics', + type: 'test-metrics', + }, + ]) + ).toEqual([ + { + id: 'some-template-custom-metrics', + type: 'test-metrics', + streams: [ + { + data_stream: { + dataset: 'foo', + type: 'metrics', + }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + id: 'test-metrics-foo', + }, + ], + }, + ]); + }); + it('returns agent inputs without disabled streams', async () => { expect( await templatePackagePolicyToFullInputStreams([