diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts index 75f14c1c3b890..4ec9490f026d4 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts @@ -6,11 +6,15 @@ */ jest.mock('../epm/packages'); +jest.mock('../app_context'); import { DATASET_VAR_NAME } from '../../../common/constants'; import type { PackagePolicy } from '../../types'; import { PackagePolicyValidationError } from '../../errors'; +import { appContextService } from '../app_context'; +import { createAppContextStartContractMock } from '../../mocks'; + import type { DataStreamMeta } from './package_policies_to_agent_permissions'; import { ELASTIC_CONNECTORS_INDEX_PERMISSIONS, @@ -410,6 +414,13 @@ packageInfoCache.set('non_dynamic_pkg-1.0.0', { }); describe('storedPackagePoliciesToAgentPermissions()', () => { + beforeEach(() => { + appContextService.start(createAppContextStartContractMock()); + jest.spyOn(appContextService, 'getExperimentalFeatures').mockReturnValue({ + enableOtelIntegrations: true, + } as any); + }); + it('Returns `undefined` if there are no package policies', async () => { const permissions = await storedPackagePoliciesToAgentPermissions(packageInfoCache, 'test', []); expect(permissions).toBeUndefined(); @@ -830,11 +841,11 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { 'package-policy-uuid-test-456': { indices: [ { - names: ['traces-otel-traces-test'], + names: ['traces-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'], }, { - names: ['logs-otel-traces-test'], + names: ['logs-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'], }, ], @@ -887,8 +898,11 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { expect(permissions).toMatchObject({ 'package-policy-otel-var-over-compiled': { indices: [ - { names: ['traces-zipkinreceiver-ep'], privileges: ['auto_configure', 'create_doc'] }, - { names: ['logs-zipkinreceiver-ep'], privileges: ['auto_configure', 'create_doc'] }, + { + names: ['traces-zipkinreceiver.otel-ep'], + privileges: ['auto_configure', 'create_doc'], + }, + { names: ['logs-zipkinreceiver.otel-ep'], privileges: ['auto_configure', 'create_doc'] }, ], }, }); @@ -936,7 +950,7 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { expect(permissions).toMatchObject({ 'package-policy-otel-dataset-empty-object': { indices: expect.arrayContaining([ - { names: ['logs-otel-traces-test'], privileges: ['auto_configure', 'create_doc'] }, + { names: ['logs-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'] }, ]), }, }); @@ -984,7 +998,7 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { expect(permissions).toMatchObject({ 'package-policy-otel-dataset-array': { indices: expect.arrayContaining([ - { names: ['logs-otel-traces-test'], privileges: ['auto_configure', 'create_doc'] }, + { names: ['logs-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'] }, ]), }, }); @@ -1032,7 +1046,7 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { expect(permissions).toMatchObject({ 'package-policy-otel-dataset-object': { indices: expect.arrayContaining([ - { names: ['logs-my.custom-test'], privileges: ['auto_configure', 'create_doc'] }, + { names: ['logs-my.custom.otel-test'], privileges: ['auto_configure', 'create_doc'] }, ]), }, }); @@ -1113,14 +1127,14 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { expect(wsPermissions).toMatchObject({ 'package-policy-otel-dataset-whitespace': { indices: expect.arrayContaining([ - { names: ['logs-otel-traces-test'], privileges: ['auto_configure', 'create_doc'] }, + { names: ['logs-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'] }, ]), }, }); expect(nestedPermissions).toMatchObject({ 'package-policy-otel-dataset-empty-nested': { indices: expect.arrayContaining([ - { names: ['logs-otel-traces-test'], privileges: ['auto_configure', 'create_doc'] }, + { names: ['logs-otel-traces.otel-test'], privileges: ['auto_configure', 'create_doc'] }, ]), }, }); @@ -1228,11 +1242,11 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { 'package-policy-otel-span-ns-only': { indices: [ { - names: ['traces-otel-traces-*'], + names: ['traces-otel-traces.otel-*'], privileges: ['auto_configure', 'create_doc'], }, { - names: ['logs-otel-traces-*'], + names: ['logs-otel-traces.otel-*'], privileges: ['auto_configure', 'create_doc'], }, ], @@ -1711,6 +1725,292 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { }); }); + describe('OTel .otel suffix on agent permissions', () => { + it('appends .otel suffix for non-dynamic OTel integration package logs stream', async () => { + const packagePolicies: PackagePolicy[] = [ + { + id: 'policy-otel-integration-logs', + name: 'otel-integration-logs', + namespace: 'default', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + inputs: [ + { + type: 'otelcol', + enabled: true, + streams: [ + { + id: 'stream-1', + enabled: true, + data_stream: { type: 'logs', dataset: 'myintegration' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + 'default', + packagePolicies + ); + expect(permissions).toMatchObject({ + 'policy-otel-integration-logs': { + indices: [ + { + names: ['logs-myintegration.otel-default'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }); + }); + + it('does not double-append .otel when dataset already ends in .otel', async () => { + const packagePolicies: PackagePolicy[] = [ + { + id: 'policy-otel-already-suffixed', + name: 'otel-already-suffixed', + namespace: 'default', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + inputs: [ + { + type: 'otelcol', + enabled: true, + streams: [ + { + id: 'stream-1', + enabled: true, + data_stream: { type: 'logs', dataset: 'generic.otel' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + 'default', + packagePolicies + ); + expect(permissions).toMatchObject({ + 'policy-otel-already-suffixed': { + indices: [ + { + names: ['logs-generic.otel-default'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }); + }); + + it('does not append .otel when enableOtelIntegrations is false', async () => { + jest.spyOn(appContextService, 'getExperimentalFeatures').mockReturnValue({ + enableOtelIntegrations: false, + } as any); + + const packagePolicies: PackagePolicy[] = [ + { + id: 'policy-otel-flag-off', + name: 'otel-flag-off', + namespace: 'default', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + inputs: [ + { + type: 'otelcol', + enabled: true, + streams: [ + { + id: 'stream-1', + enabled: true, + data_stream: { type: 'logs', dataset: 'myintegration' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + 'default', + packagePolicies + ); + expect(permissions).toMatchObject({ + 'policy-otel-flag-off': { + indices: [ + { + names: ['logs-myintegration-default'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }); + }); + + it('does not append .otel when dynamic_dataset is true (wildcard already covers .otel)', async () => { + const packagePolicies: PackagePolicy[] = [ + { + id: 'policy-otel-dynamic-dataset', + name: 'otel-dynamic-dataset', + namespace: 'default', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + inputs: [ + { + type: 'otelcol', + enabled: true, + streams: [ + { + id: 'stream-1', + enabled: true, + data_stream: { + type: 'logs', + dataset: 'myintegration', + elasticsearch: { dynamic_dataset: true, dynamic_namespace: true }, + }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + 'default', + packagePolicies + ); + expect(permissions).toMatchObject({ + 'policy-otel-dynamic-dataset': { + indices: [ + { + names: ['logs-*-*'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }); + }); + + it('does not append .otel when dataset_is_prefix is true (wildcard already covers .otel)', async () => { + // dataset_is_prefix lives on the registry RegistryDataStream, not on the policy stream's + // data_stream object. The implementation looks it up from getNormalizedDataStreams(pkg), + // so the package fixture must declare it there. + packageInfoCache.set('otel_prefix_pkg-1.0.0', { + name: 'otel_prefix_pkg', + version: '1.0.0', + latestVersion: '1.0.0', + release: 'ga', + format_version: '2.7.0', + title: 'OTel Prefix Pkg', + description: '', + type: 'integration', + status: 'not_installed', + assets: { kibana: {}, elasticsearch: {} }, + policy_templates: [ + { + name: 'otel', + title: 'OTel', + description: 'OTel input', + inputs: [{ type: 'otelcol', title: 'OTel', description: 'OTel' }], + }, + ], + data_streams: [ + { + type: 'logs', + dataset: 'myintegration', + title: 'My Integration Logs', + release: 'ga', + package: 'otel_prefix_pkg', + path: 'logs', + dataset_is_prefix: true, + streams: [{ input: 'otelcol', title: 'OTel Logs', template_path: '' }], + }, + ], + } as any); + + const packagePolicies: PackagePolicy[] = [ + { + id: 'policy-otel-dataset-prefix', + name: 'otel-dataset-prefix', + namespace: 'default', + enabled: true, + package: { name: 'otel_prefix_pkg', version: '1.0.0', title: 'OTel Prefix Pkg' }, + inputs: [ + { + type: 'otelcol', + enabled: true, + streams: [ + { + id: 'stream-1', + enabled: true, + data_stream: { type: 'logs', dataset: 'myintegration' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + 'default', + packagePolicies + ); + expect(permissions).toMatchObject({ + 'policy-otel-dataset-prefix': { + indices: [ + { + names: ['logs-myintegration.*-default'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }); + }); + }); + describe('data_stream.type undefined handling', () => { it('throws for non-dynamic package stream with undefined data_stream.type', () => { const packagePolicies: PackagePolicy[] = [ diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts index 717e39cffedc8..c4b633969e8a1 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts @@ -16,9 +16,12 @@ import { FLEET_UNIVERSAL_PROFILING_COLLECTOR_PACKAGE, FLEET_UNIVERSAL_PROFILING_SYMBOLIZER_PACKAGE, OTEL_COLLECTOR_INPUT_TYPE, + OTEL_TEMPLATE_SUFFIX, USE_APM_VAR_NAME, } from '../../../common/constants'; +import { appContextService } from '../app_context'; + import { getNormalizedDataStreams } from '../../../common/services'; import type { @@ -68,13 +71,38 @@ export const AGENTLESS_INDEX_PERMISSIONS = [ 'view_index_metadata', ]; +/** + * Appends the `.otel` suffix to a dataset string for OTel input streams when building agent + * index privileges, mirroring the EPM side (`getRegistryDataStreamAssetBaseName` with + * `isOtelInputType`). Only applies when: + * - the source input is `otelcol` + * - `enableOtelIntegrations` is on (same gate as EPM) + * - the stream is not dynamic_dataset (wildcard already covers .otel) + * - the stream is not dataset_is_prefix (wildcard `.*` already covers .otel) + * - the dataset doesn't already end in `.otel` (defensive against double-append) + */ +function applyOtelDatasetSuffixIfNeeded( + dataset: string, + opts: { + isOtelInput: boolean; + dynamicDataset?: boolean; + datasetIsPrefix?: boolean; + } +): string { + if (!opts.isOtelInput) return dataset; + if (!appContextService.getExperimentalFeatures().enableOtelIntegrations) return dataset; + if (opts.dynamicDataset) return dataset; + if (opts.datasetIsPrefix) return dataset; + if (dataset.endsWith(`.${OTEL_TEMPLATE_SUFFIX}`)) return dataset; + return `${dataset}.${OTEL_TEMPLATE_SUFFIX}`; +} + export function storedPackagePoliciesToAgentPermissions( packageInfoCache: Map, agentPolicyNamespace: string, packagePolicies?: PackagePolicy[], agentInputs?: FullAgentPolicyInput[] | TemplateAgentPolicyInput[] ): FullAgentPolicyOutputPermissions | undefined { - // I'm not sure what permissions to return for this case, so let's return the defaults if (!packagePolicies) { throw new PackagePolicyRequestError( 'storedPackagePoliciesToAgentPermissions should be called with a PackagePolicy' @@ -163,14 +191,9 @@ export function storedPackagePoliciesToAgentPermissions( const otelcolPipelines = agentInputs?.find((i) => i.type === OTEL_COLLECTOR_INPUT_TYPE) ?.streams?.[0]?.service?.pipelines; - let signalTypes: string[]; - if (otelcolPipelines) { - // Use pipelines if available - signalTypes = extractSignalTypesFromPipelines(otelcolPipelines); - } else { - // If no pipelines found, return empty array - signalTypes = []; - } + const signalTypes = otelcolPipelines + ? extractSignalTypesFromPipelines(otelcolPipelines) + : []; const baseMeta: DataStreamMeta = { type: 'logs', @@ -220,17 +243,36 @@ export function storedPackagePoliciesToAgentPermissions( ); } + const rawDataset = isOtelInput + ? getEffectiveOtelStreamDataset(stream) + : stream.compiled_stream?.data_stream?.dataset ?? stream.data_stream.dataset; + + // Look up dataset_is_prefix from the registry data stream definition — + // it is not stored on the policy stream's data_stream object. + const registryDs = dataStreams.find( + (rds) => + rds.type === stream.data_stream.type && + rds.dataset === stream.data_stream.dataset + ); + const datasetIsPrefix = registryDs?.dataset_is_prefix; + const ds: DataStreamMeta = { type: stream.data_stream.type, - dataset: isOtelInput - ? getEffectiveOtelStreamDataset(stream) - : stream.compiled_stream?.data_stream?.dataset ?? stream.data_stream.dataset, + dataset: applyOtelDatasetSuffixIfNeeded(rawDataset, { + isOtelInput, + dynamicDataset: stream.data_stream.elasticsearch?.dynamic_dataset, + datasetIsPrefix, + }), }; if (stream.data_stream.elasticsearch) { ds.elasticsearch = stream.data_stream.elasticsearch; } + if (datasetIsPrefix) { + ds.dataset_is_prefix = true; + } + dataStreams_.push(ds); if (isOtelInput && stream.data_stream.type === 'traces') { @@ -243,7 +285,11 @@ export function storedPackagePoliciesToAgentPermissions( ); dataStreams_.push({ type: 'logs', - dataset: getEffectiveOtelStreamDataset(stream), + dataset: applyOtelDatasetSuffixIfNeeded(rawDataset, { + isOtelInput: true, + dynamicDataset: spanEventElasticsearch?.dynamic_dataset, + datasetIsPrefix, + }), ...(spanEventElasticsearch ? { elasticsearch: spanEventElasticsearch } : {}), });