diff --git a/x-pack/platform/plugins/shared/fleet/common/constants/epm.ts b/x-pack/platform/plugins/shared/fleet/common/constants/epm.ts index f55a7294c0273..e6a5b5f5b26de 100644 --- a/x-pack/platform/plugins/shared/fleet/common/constants/epm.ts +++ b/x-pack/platform/plugins/shared/fleet/common/constants/epm.ts @@ -49,6 +49,7 @@ export const PACKAGE_TEMPLATE_SUFFIX = '@package'; export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom'; export const DATASET_VAR_NAME = 'data_stream.dataset'; +export const DATA_STREAM_TYPE_VAR_NAME = 'data_stream.type'; export const CUSTOM_INTEGRATION_PACKAGE_SPEC_VERSION = '2.9.0'; diff --git a/x-pack/platform/plugins/shared/fleet/common/services/policy_template.test.ts b/x-pack/platform/plugins/shared/fleet/common/services/policy_template.test.ts index 4065d0c863c9e..a01acf369aa59 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/policy_template.test.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/policy_template.test.ts @@ -11,6 +11,7 @@ import type { PackageInfo, RegistryVarType, PackageListItem, + RegistryDataStream, } from '../types'; import { @@ -281,6 +282,75 @@ describe('getNormalizedDataStreams', () => { expect(result[0].streams).toHaveLength(1); expect(result?.[0].streams?.[0]?.vars).toEqual([datasetVar]); }); + + const inputPkg: PackageInfo = { + name: 'log', + type: 'input', + title: 'Custom logs', + version: '2.4.0', + description: 'Collect custom logs with Elastic Agent.', + format_version: '3.1.5', + owner: { github: '' }, + assets: {} as any, + data_streams: [], + policy_templates: [ + { + name: 'logs', + type: 'logs', + title: 'Custom log file', + description: 'Collect logs from custom files.', + input: 'logfile', + template_path: 'input.yml.hbs', + vars: [ + { + name: 'paths', + type: 'text', + title: 'Paths', + multi: true, + required: true, + show_user: true, + default: ['/var/log/nginx/access.log*'], + }, + ], + }, + ], + latestVersion: '1.3.0', + keepPoliciesUpToDate: false, + status: 'not_installed', + }; + const expectedInputPackageDataStream: RegistryDataStream = { + type: 'logs', + dataset: 'log.logs', + elasticsearch: { + dynamic_dataset: true, + dynamic_namespace: true, + }, + title: expect.any(String), + release: 'ga', + package: 'log', + path: 'log.logs', + streams: [ + { + input: 'logfile', + vars: expect.any(Array), + template_path: 'input.yml.hbs', + title: 'Custom log file', + description: 'Custom log file', + enabled: true, + }, + ], + }; + it('should build data streams for input package', () => { + expect(getNormalizedDataStreams(inputPkg)).toEqual([expectedInputPackageDataStream]); + }); + it('should use user-defined data stream type in input package', () => { + expect(getNormalizedDataStreams(inputPkg, undefined, 'metrics')).toEqual([ + { + ...expectedInputPackageDataStream, + type: 'metrics', + }, + ]); + }); }); describe('filterPolicyTemplatesTiles', () => { diff --git a/x-pack/platform/plugins/shared/fleet/common/services/policy_template.ts b/x-pack/platform/plugins/shared/fleet/common/services/policy_template.ts index efa65a880576a..07b3b85671472 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/policy_template.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/policy_template.ts @@ -68,7 +68,8 @@ export const getNormalizedInputs = (policyTemplate: RegistryPolicyTemplate): Reg export const getNormalizedDataStreams = ( packageInfo: PackageInfo | InstallablePackage, - datasetName?: string + datasetName?: string, + dataStreamType?: string ): RegistryDataStream[] => { if (packageInfo.type !== 'input') { return packageInfo.data_streams || []; @@ -84,7 +85,7 @@ export const getNormalizedDataStreams = ( const dataset = datasetName || createDefaultDatasetName(packageInfo, policyTemplate); const dataStream: RegistryDataStream = { - type: policyTemplate.type, + type: dataStreamType || policyTemplate.type, dataset, title: policyTemplate.title + ' Dataset', release: packageInfo.release || 'ga', diff --git a/x-pack/platform/plugins/shared/fleet/cypress/e2e/input_packages_real.cy.ts b/x-pack/platform/plugins/shared/fleet/cypress/e2e/input_packages_real.cy.ts new file mode 100644 index 0000000000000..c5c5c685cc896 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/cypress/e2e/input_packages_real.cy.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ADD_INTEGRATION_POLICY_BTN, + CREATE_PACKAGE_POLICY_SAVE_BTN, + INTEGRATION_NAME_LINK, + POLICY_EDITOR, +} from '../screens/integrations'; +import { EXISTING_HOSTS_TAB } from '../screens/fleet'; +import { CONFIRM_MODAL } from '../screens/navigation'; + +import { API_VERSIONS } from '../../common'; +import { cleanupAgentPolicies } from '../tasks/cleanup'; +import { login } from '../tasks/login'; +import { request } from '../tasks/common'; + +const INPUT_TEST_PACKAGE = 'input_package-1.0.0'; + +describe('Input package with custom data stream type', () => { + beforeEach(() => { + login(); + }); + + const agentPolicyId = 'test-input-package-policy'; + const agentPolicyName = 'Test input package policy'; + const packagePolicyName = 'input-package-policy'; + const datasetName = 'logs'; // Default from the package. + const dataStreamType = 'metrics'; + + before(() => { + cy.task('installTestPackage', INPUT_TEST_PACKAGE); + + request({ + method: 'POST', + url: `/api/fleet/agent_policies`, + body: { + id: agentPolicyId, + name: agentPolicyName, + description: 'desc', + namespace: 'default', + monitoring_enabled: [], + }, + headers: { 'kbn-xsrf': 'cypress', 'Elastic-Api-Version': `${API_VERSIONS.public.v1}` }, + }); + }); + + after(() => { + cleanupAgentPolicies(); + cy.task('uninstallTestPackage', INPUT_TEST_PACKAGE); + }); + + it('should successfully create a package policy', () => { + cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/overview`); + cy.getBySel(ADD_INTEGRATION_POLICY_BTN).click(); + + cy.getBySel(POLICY_EDITOR.POLICY_NAME_INPUT).click().clear().type(packagePolicyName); + cy.getBySel('multiTextInput-paths') + .find('[data-test-subj="multiTextInputRow-0"]') + .click() + .type('/var/log/test.log'); + + cy.getBySel('multiTextInput-tags') + .find('[data-test-subj="multiTextInputRow-0"]') + .click() + .type('tag1'); + + // Select metrics data stream type. + cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click(); + cy.get('[data-test-subj="packagePolicyDataStreamType"') + .find(`label[for="${dataStreamType}"]`) + .click(); + + cy.getBySel(EXISTING_HOSTS_TAB).click(); + + cy.getBySel(POLICY_EDITOR.AGENT_POLICY_SELECT).click(); + cy.getBySel('agentPolicyMultiItem').each(($el) => { + if ($el.text() === agentPolicyName) { + $el.trigger('click'); + } + }); + cy.wait(1000); // wait for policy id to be set + cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click(); + + cy.getBySel(CONFIRM_MODAL.CANCEL_BUTTON).click(); + }); + + it(`${dataStreamType} checkbox should be checked`, () => { + cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/policies`); + + cy.getBySel(INTEGRATION_NAME_LINK).contains(packagePolicyName).click(); + + cy.get('button').contains('Change defaults').click(); + cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click(); + cy.get('[data-test-subj="packagePolicyDataStreamType"') + .find(`input#${dataStreamType}`) + .should('be.checked'); + }); + + it('should not allow to edit data stream type', () => { + cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/policies`); + + cy.getBySel(INTEGRATION_NAME_LINK).contains(packagePolicyName).click(); + + cy.get('button').contains('Change defaults').click(); + cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click(); + cy.get('[data-test-subj="packagePolicyDataStreamType"') + .find('input') + .should('have.length', 3) + .each(($el) => cy.wrap($el).should('be.disabled')); + }); + + it('has an index template', () => { + cy.visit(`app/management/data/index_management/templates/${dataStreamType}-${datasetName}`); + + // Check that the index pattern appears in the view. + cy.get('[data-test-subj="templateDetails"').contains(`${dataStreamType}-${datasetName}-*`); + }); +}); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx index 8a10726b08df8..e89705f177721 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx @@ -10,10 +10,14 @@ import ReactMarkdown from 'react-markdown'; import styled from 'styled-components'; import { uniq } from 'lodash'; import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; import { EuiFlexGrid, EuiFlexGroup, EuiFlexItem, + EuiFormRow, + EuiLink, + EuiRadioGroup, EuiSwitch, EuiText, EuiSpacer, @@ -24,9 +28,12 @@ import { useRouteMatch } from 'react-router-dom'; import { useQuery } from '@tanstack/react-query'; -import { DATASET_VAR_NAME } from '../../../../../../../../../common/constants'; +import { + DATASET_VAR_NAME, + DATA_STREAM_TYPE_VAR_NAME, +} from '../../../../../../../../../common/constants'; -import { useConfig, sendGetDataStreams } from '../../../../../../../../hooks'; +import { useConfig, sendGetDataStreams, useStartServices } from '../../../../../../../../hooks'; import { getRegistryDataStreamAssetBaseName, @@ -76,6 +83,8 @@ export const PackagePolicyInputStreamConfig = memo( forceShowErrors, isEditPage, }) => { + const { docLinks } = useStartServices(); + const config = useConfig(); const isExperimentalDataStreamSettingsEnabled = config.enableExperimental?.includes('experimentalDataStreamSettings') ?? false; @@ -95,6 +104,10 @@ export const PackagePolicyInputStreamConfig = memo( const customDatasetVar = packagePolicyInputStream.vars?.[DATASET_VAR_NAME]; const customDatasetVarValue = customDatasetVar?.value?.dataset || customDatasetVar?.value; + const customDataStreamTypeVar = packagePolicyInputStream.vars?.[DATA_STREAM_TYPE_VAR_NAME]; + const customDataStreamTypeVarValue = + customDataStreamTypeVar?.value || packagePolicyInputStream.data_stream.type || 'logs'; + const { exists: indexTemplateExists, isLoading: isLoadingIndexTemplate } = useIndexTemplateExists( getRegistryDataStreamAssetBaseName({ @@ -255,7 +268,7 @@ export const PackagePolicyInputStreamConfig = memo( })} {/* Advanced section */} - {hasAdvancedOptions && ( + {(hasAdvancedOptions || packageInfo.type === 'input') && ( @@ -288,6 +301,75 @@ export const PackagePolicyInputStreamConfig = memo( {isShowingAdvanced ? ( <> + {packageInfo.type === 'input' && ( + + + } + helpText={ + isEditPage ? ( + + ) : ( + + {i18n.translate( + 'xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyNamespaceHelpLearnMoreLabel', + { defaultMessage: 'Learn more' } + )} + + ), + }} + /> + ) + } + > + { + updatePackagePolicyInputStream({ + vars: { + ...packagePolicyInputStream.vars, + [DATA_STREAM_TYPE_VAR_NAME]: { + type: 'string', + value: type, + }, + }, + }); + }} + /> + + + )} {advancedVars.map((varDef) => { if (!packagePolicyInputStream.vars) return null; const { name: varName, type: varType } = varDef; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts index 9004d544f399d..13e52feb93008 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts @@ -72,6 +72,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { compiled_stream: { fooKey: 'fooValue1', fooKey2: ['fooValue2'], + data_stream: { dataset: 'foo' }, // data_stream.dataset can be set in the compiled stream, ensure that rest of data_stream object is properly merged. }, }, { @@ -124,6 +125,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { compiled_stream: { fooKey: 'fooValue1', fooKey2: ['fooValue2'], + data_stream: { dataset: 'foo' }, // data_stream.dataset can be set in the compiled stream, ensure that rest of data_stream object is properly merged. }, }, ], diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index f2a39c6f8800b..1ec210c033297 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -123,10 +123,15 @@ export const getFullInputStreams = ( .filter((stream) => stream.enabled || allStreamEnabled) .map((stream) => { const streamId = stream.id; + const { data_stream: compiledDataStream, ...compiledStream } = + stream.compiled_stream ?? {}; const fullStream: FullAgentPolicyInputStream = { id: streamId, - data_stream: stream.data_stream, - ...stream.compiled_stream, + data_stream: { + ...stream.data_stream, + ...compiledDataStream, + }, + ...compiledStream, ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { acc[key] = value; return acc; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap index 56e4f7ee09319..3e93e0ccd8b6a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap @@ -8,10 +8,14 @@ exports[`Fleet - getTemplateInputs should work for input package 1`] = ` streams: # Custom log file: Custom log file - id: logfile-log.logs + data_stream: + type: logs + elasticsearch: + dynamic_dataset: true + dynamic_namespace: true + # dataset: + # # Dataset name: Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use \`-\` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). ignore_older: 72h - # data_stream: - # dataset: - # # Dataset name: Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use \`-\` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). # paths: # - # Log file path: Path to log files to be collected # exclude_files: diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.test.ts index 6b3a31eda649e..d7f878e97008d 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.test.ts @@ -578,7 +578,12 @@ describe('installAssetsForInputPackagePolicy', () => { force: false, logger: mockedLogger, packagePolicy: { - inputs: [{ type: 'log', streams: [{ type: 'log', vars: { dataset: 'test.tata' } }] }], + inputs: [ + { + type: 'log', + streams: [{ data_stream: { type: 'log' }, vars: { dataset: 'test.tata' } }], + }, + ], } as any, }) ).rejects.toThrowError(PackageNotFoundError); @@ -610,7 +615,12 @@ describe('installAssetsForInputPackagePolicy', () => { { name: 'log', type: 'log', - streams: [{ type: 'log', vars: { 'data_stream.dataset': { value: 'test.tata' } } }], + streams: [ + { + data_stream: { type: 'log' }, + vars: { 'data_stream.dataset': { value: 'test.tata' } }, + }, + ], }, ], } as any, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts index 8c6f78845805d..c11fa1343a1bb 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts @@ -50,6 +50,7 @@ import { AUTO_UPGRADE_POLICIES_PACKAGES, CUSTOM_INTEGRATION_PACKAGE_SPEC_VERSION, DATASET_VAR_NAME, + DATA_STREAM_TYPE_VAR_NAME, GENERIC_DATASET_NAME, } from '../../../../common/constants'; import { @@ -1288,7 +1289,11 @@ export async function installAssetsForInputPackagePolicy(opts: { if (pkgInfo.type !== 'input') return; const datasetName = packagePolicy.inputs[0].streams[0].vars?.[DATASET_VAR_NAME]?.value; - const [dataStream] = getNormalizedDataStreams(pkgInfo, datasetName); + const dataStreamType = + packagePolicy.inputs[0].streams[0].vars?.[DATA_STREAM_TYPE_VAR_NAME]?.value || + packagePolicy.inputs[0].streams[0].data_stream?.type || + 'logs'; + const [dataStream] = getNormalizedDataStreams(pkgInfo, datasetName, dataStreamType); const existingDataStreams = await dataStreamService.getMatchingDataStreams(esClient, { type: dataStream.type, dataset: datasetName, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts index 54bbada018bf4..61e6b708add04 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts @@ -52,6 +52,7 @@ import { DATASET_VAR_NAME, LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, + DATA_STREAM_TYPE_VAR_NAME, } from '../../common/constants'; import type { PostDeletePackagePoliciesResponse, @@ -3085,6 +3086,29 @@ export function _validateRestrictedFieldsNotModifiedOrThrow(opts: { }) ); } + + if ( + oldStream && + oldStream?.vars?.[DATA_STREAM_TYPE_VAR_NAME] && + oldStream?.vars[DATA_STREAM_TYPE_VAR_NAME]?.value !== + stream?.vars?.[DATA_STREAM_TYPE_VAR_NAME]?.value + ) { + // seeing this error in dev? Package policy must be called with prepareInputPackagePolicyDataset function first in UI code + appContextService + .getLogger() + .debug( + () => + `Rejecting package policy update due to data stream type change, old val '${ + oldStream.vars![DATA_STREAM_TYPE_VAR_NAME].value + }, new val '${JSON.stringify(stream?.vars?.[DATA_STREAM_TYPE_VAR_NAME]?.value)}'` + ); + throw new PackagePolicyValidationError( + i18n.translate('xpack.fleet.updatePackagePolicy.datasetCannotBeModified', { + defaultMessage: + 'Package policy data stream type cannot be modified for input only packages, please create a new package policy.', + }) + ); + } } } }