Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
fa8a050
Add UI components to select ds type
jsoriano Mar 11, 2025
0f31982
Move UI element to stream editor
jsoriano Mar 12, 2025
7a90ba3
Fix update of policy input stream
jsoriano Mar 12, 2025
f6376f8
Remove unused variable
jsoriano Mar 12, 2025
0b921b9
Fix comparison
jsoriano Mar 12, 2025
866630a
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine Mar 12, 2025
1489f2f
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano Apr 2, 2025
10123cc
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano Apr 2, 2025
a279a34
Use data stream type instead of input type for stream ids
jsoriano Apr 2, 2025
c580901
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano Apr 2, 2025
2e89a91
Take data stream user config into account when creating it
jsoriano Apr 16, 2025
88772ae
Set the data stream type where the dataset is also configured
jsoriano Apr 16, 2025
ff1b760
Revert possibly unneeded change
jsoriano Apr 16, 2025
ebf07cc
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano Apr 22, 2025
698afc9
Fix data stream override in streams when generating agent inputs from…
jsoriano Apr 24, 2025
d07739a
Add unit tests
jsoriano Apr 24, 2025
80a3139
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano Apr 24, 2025
2661c95
Persist data stream type in variable
jsoriano Apr 24, 2025
c040ff0
Linting
jsoriano Apr 24, 2025
fde3239
Ensure data stream type cannot be edited
jsoriano May 7, 2025
1e7d998
Add cypress test case
jsoriano May 7, 2025
97c84dc
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano May 7, 2025
ca960b9
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 7, 2025
68e1a24
Add required field
jsoriano May 8, 2025
643d2af
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano May 8, 2025
1c04b43
Try looking for disabled inputs
jsoriano May 8, 2025
f0b728a
Add more tests
jsoriano May 8, 2025
2434d12
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 8, 2025
310d593
Fix tests
jsoriano May 8, 2025
6323ba4
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano May 8, 2025
f561a81
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine May 8, 2025
e0a1526
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 8, 2025
adb28d1
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine May 8, 2025
c1c9ef5
Try to fix more tests
jsoriano May 8, 2025
05a5b4e
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano May 9, 2025
788c432
Try to fix more errors
jsoriano May 9, 2025
44680ad
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 9, 2025
7213359
Merge branch 'main' into input-package-ds-type
elasticmachine May 13, 2025
be8d738
Merge branch 'main' into input-package-ds-type
elasticmachine May 19, 2025
65ae4d3
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano May 22, 2025
7eb2694
Avoid changing the order of stream subfields
jsoriano May 22, 2025
124fbc5
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 22, 2025
d520cbf
Linting
jsoriano May 23, 2025
390fe14
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano May 23, 2025
fe56b3b
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine May 23, 2025
7695e9a
Fix test file
jsoriano May 24, 2025
916acd2
Merge remote-tracking branch 'jsoriano/input-package-ds-type' into in…
jsoriano May 24, 2025
fb3493a
Try again
jsoriano May 26, 2025
30d39f6
Merge remote-tracking branch 'origin/main' into input-package-ds-type
jsoriano May 26, 2025
b4f0a5b
One more space
jsoriano May 26, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ export const PACKAGE_TEMPLATE_SUFFIX = '@package';
export const USER_SETTINGS_TEMPLATE_SUFFIX = '@custom';

export const DATASET_VAR_NAME = 'data_stream.dataset';
export const DATA_STREAM_TYPE_VAR_NAME = 'data_stream.type';

export const CUSTOM_INTEGRATION_PACKAGE_SPEC_VERSION = '2.9.0';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import type {
PackageInfo,
RegistryVarType,
PackageListItem,
RegistryDataStream,
} from '../types';

import {
Expand Down Expand Up @@ -281,6 +282,75 @@ describe('getNormalizedDataStreams', () => {
expect(result[0].streams).toHaveLength(1);
expect(result?.[0].streams?.[0]?.vars).toEqual([datasetVar]);
});

const inputPkg: PackageInfo = {
name: 'log',
type: 'input',
title: 'Custom logs',
version: '2.4.0',
description: 'Collect custom logs with Elastic Agent.',
format_version: '3.1.5',
owner: { github: '' },
assets: {} as any,
data_streams: [],
policy_templates: [
{
name: 'logs',
type: 'logs',
title: 'Custom log file',
description: 'Collect logs from custom files.',
input: 'logfile',
template_path: 'input.yml.hbs',
vars: [
{
name: 'paths',
type: 'text',
title: 'Paths',
multi: true,
required: true,
show_user: true,
default: ['/var/log/nginx/access.log*'],
},
],
},
],
latestVersion: '1.3.0',
keepPoliciesUpToDate: false,
status: 'not_installed',
};
const expectedInputPackageDataStream: RegistryDataStream = {
type: 'logs',
dataset: 'log.logs',
elasticsearch: {
dynamic_dataset: true,
dynamic_namespace: true,
},
title: expect.any(String),
release: 'ga',
package: 'log',
path: 'log.logs',
streams: [
{
input: 'logfile',
vars: expect.any(Array),
template_path: 'input.yml.hbs',
title: 'Custom log file',
description: 'Custom log file',
enabled: true,
},
],
};
it('should build data streams for input package', () => {
expect(getNormalizedDataStreams(inputPkg)).toEqual([expectedInputPackageDataStream]);
});
it('should use user-defined data stream type in input package', () => {
expect(getNormalizedDataStreams(inputPkg, undefined, 'metrics')).toEqual([
{
...expectedInputPackageDataStream,
type: 'metrics',
},
]);
});
});

describe('filterPolicyTemplatesTiles', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ export const getNormalizedInputs = (policyTemplate: RegistryPolicyTemplate): Reg

export const getNormalizedDataStreams = (
packageInfo: PackageInfo | InstallablePackage,
datasetName?: string
datasetName?: string,
dataStreamType?: string
): RegistryDataStream[] => {
if (packageInfo.type !== 'input') {
return packageInfo.data_streams || [];
Expand All @@ -84,7 +85,7 @@ export const getNormalizedDataStreams = (
const dataset = datasetName || createDefaultDatasetName(packageInfo, policyTemplate);

const dataStream: RegistryDataStream = {
type: policyTemplate.type,
type: dataStreamType || policyTemplate.type,
dataset,
title: policyTemplate.title + ' Dataset',
release: packageInfo.release || 'ga',
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import {
ADD_INTEGRATION_POLICY_BTN,
CREATE_PACKAGE_POLICY_SAVE_BTN,
INTEGRATION_NAME_LINK,
POLICY_EDITOR,
} from '../screens/integrations';
import { EXISTING_HOSTS_TAB } from '../screens/fleet';
import { CONFIRM_MODAL } from '../screens/navigation';

import { API_VERSIONS } from '../../common';
import { cleanupAgentPolicies } from '../tasks/cleanup';
import { login } from '../tasks/login';
import { request } from '../tasks/common';

const INPUT_TEST_PACKAGE = 'input_package-1.0.0';

describe('Input package with custom data stream type', () => {
beforeEach(() => {
login();
});

const agentPolicyId = 'test-input-package-policy';
const agentPolicyName = 'Test input package policy';
const packagePolicyName = 'input-package-policy';
const datasetName = 'logs'; // Default from the package.
const dataStreamType = 'metrics';

before(() => {
cy.task('installTestPackage', INPUT_TEST_PACKAGE);

request({
method: 'POST',
url: `/api/fleet/agent_policies`,
body: {
id: agentPolicyId,
name: agentPolicyName,
description: 'desc',
namespace: 'default',
monitoring_enabled: [],
},
headers: { 'kbn-xsrf': 'cypress', 'Elastic-Api-Version': `${API_VERSIONS.public.v1}` },
});
});

after(() => {
cleanupAgentPolicies();
cy.task('uninstallTestPackage', INPUT_TEST_PACKAGE);
});

it('should successfully create a package policy', () => {
cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/overview`);
cy.getBySel(ADD_INTEGRATION_POLICY_BTN).click();

cy.getBySel(POLICY_EDITOR.POLICY_NAME_INPUT).click().clear().type(packagePolicyName);
cy.getBySel('multiTextInput-paths')
.find('[data-test-subj="multiTextInputRow-0"]')
.click()
.type('/var/log/test.log');

cy.getBySel('multiTextInput-tags')
.find('[data-test-subj="multiTextInputRow-0"]')
.click()
.type('tag1');

// Select metrics data stream type.
cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click();
cy.get('[data-test-subj="packagePolicyDataStreamType"')
.find(`label[for="${dataStreamType}"]`)
.click();

cy.getBySel(EXISTING_HOSTS_TAB).click();

cy.getBySel(POLICY_EDITOR.AGENT_POLICY_SELECT).click();
cy.getBySel('agentPolicyMultiItem').each(($el) => {
if ($el.text() === agentPolicyName) {
$el.trigger('click');
}
});
cy.wait(1000); // wait for policy id to be set
cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click();

cy.getBySel(CONFIRM_MODAL.CANCEL_BUTTON).click();
});

it(`${dataStreamType} checkbox should be checked`, () => {
cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/policies`);

cy.getBySel(INTEGRATION_NAME_LINK).contains(packagePolicyName).click();

cy.get('button').contains('Change defaults').click();
cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click();
cy.get('[data-test-subj="packagePolicyDataStreamType"')
.find(`input#${dataStreamType}`)
.should('be.checked');
});

it('should not allow to edit data stream type', () => {
cy.visit(`/app/integrations/detail/${INPUT_TEST_PACKAGE}/policies`);

cy.getBySel(INTEGRATION_NAME_LINK).contains(packagePolicyName).click();

cy.get('button').contains('Change defaults').click();
cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click();
cy.get('[data-test-subj="packagePolicyDataStreamType"')
.find('input')
.should('have.length', 3)
.each(($el) => cy.wrap($el).should('be.disabled'));
});

it('has an index template', () => {
cy.visit(`app/management/data/index_management/templates/${dataStreamType}-${datasetName}`);

// Check that the index pattern appears in the view.
cy.get('[data-test-subj="templateDetails"').contains(`${dataStreamType}-${datasetName}-*`);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,14 @@ import ReactMarkdown from 'react-markdown';
import styled from 'styled-components';
import { uniq } from 'lodash';
import { FormattedMessage } from '@kbn/i18n-react';
import { i18n } from '@kbn/i18n';
import {
EuiFlexGrid,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiLink,
EuiRadioGroup,
EuiSwitch,
EuiText,
EuiSpacer,
Expand All @@ -25,9 +29,12 @@ import { useRouteMatch } from 'react-router-dom';

import { useQuery } from '@tanstack/react-query';

import { DATASET_VAR_NAME } from '../../../../../../../../../common/constants';
import {
DATASET_VAR_NAME,
DATA_STREAM_TYPE_VAR_NAME,
} from '../../../../../../../../../common/constants';

import { useConfig, sendGetDataStreams } from '../../../../../../../../hooks';
import { useConfig, sendGetDataStreams, useStartServices } from '../../../../../../../../hooks';

import {
getRegistryDataStreamAssetBaseName,
Expand Down Expand Up @@ -79,6 +86,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
isEditPage,
totalStreams,
}) => {
const { docLinks } = useStartServices();

const config = useConfig();
const isExperimentalDataStreamSettingsEnabled =
config.enableExperimental?.includes('experimentalDataStreamSettings') ?? false;
Expand All @@ -98,6 +107,10 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
const customDatasetVar = packagePolicyInputStream.vars?.[DATASET_VAR_NAME];
const customDatasetVarValue = customDatasetVar?.value?.dataset || customDatasetVar?.value;

const customDataStreamTypeVar = packagePolicyInputStream.vars?.[DATA_STREAM_TYPE_VAR_NAME];
const customDataStreamTypeVarValue =
customDataStreamTypeVar?.value || packagePolicyInputStream.data_stream.type || 'logs';

const { exists: indexTemplateExists, isLoading: isLoadingIndexTemplate } =
useIndexTemplateExists(
getRegistryDataStreamAssetBaseName({
Expand Down Expand Up @@ -295,7 +308,7 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
})}

{/* Advanced section */}
{hasAdvancedOptions && (
{(hasAdvancedOptions || packageInfo.type === 'input') && (
<Fragment>
<EuiFlexItem>
<EuiFlexGroup justifyContent="spaceBetween" alignItems="center">
Expand Down Expand Up @@ -328,6 +341,75 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
</EuiFlexItem>
{isShowingAdvanced ? (
<>
{packageInfo.type === 'input' && (
<EuiFlexItem>
<EuiFormRow
label={
<FormattedMessage
id="xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyDataStreamTypeInputLabel"
defaultMessage="Data Stream Type"
/>
}
helpText={
isEditPage ? (
<FormattedMessage
id="xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyInputOnlyEditDataStreamTypeHelpLabel"
defaultMessage="The data stream type cannot be changed for this integration. Create a new integration policy to use a different input type."
/>
) : (
<FormattedMessage
id="xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyDataStreamTypeHelpLabel"
defaultMessage="Select a data stream type for this policy. This setting changes the name of the integration's data stream. {learnMore}."
values={{
learnMore: (
<EuiLink
href={docLinks.links.fleet.datastreamsNamingScheme}
target="_blank"
>
{i18n.translate(
'xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyNamespaceHelpLearnMoreLabel',
{ defaultMessage: 'Learn more' }
)}
</EuiLink>
),
}}
/>
)
}
>
<EuiRadioGroup
Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be probably nice to add some kind of E2E or integration test to check that this radio button works and that the expected index template is created after installing the package and adding a policy.

@juliaElastic @nchaulet wdyt? do you have recommendations about where to start adding such a test?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

data-test-subj="packagePolicyDataStreamType"
disabled={isEditPage}
idSelected={customDataStreamTypeVarValue}
options={[
{
id: 'logs',
label: 'Logs',
},
{
id: 'metrics',
label: 'Metrics',
},
{
id: 'traces',
label: 'Traces',
},
Copy link
Copy Markdown
Member Author

@jsoriano jsoriano Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it ok to allow all types for any input package?

In principle I would say so, it is up to the user to know what kind of data they are collecting.

If we think that some packages should be limited to only certain types, we would need to add something in the spec.

]}
onChange={(type: string) => {
updatePackagePolicyInputStream({
vars: {
...packagePolicyInputStream.vars,
[DATA_STREAM_TYPE_VAR_NAME]: {
type: 'string',
value: type,
},
},
});
}}
/>
</EuiFormRow>
</EuiFlexItem>
)}
{advancedVars.map((varDef) => {
if (!packagePolicyInputStream.vars) return null;
const { name: varName, type: varType } = varDef;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
compiled_stream: {
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
data_stream: { dataset: 'foo' }, // data_stream.dataset can be set in the compiled stream, ensure that rest of data_stream object is properly merged.
},
},
{
Expand Down Expand Up @@ -124,6 +125,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
compiled_stream: {
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
data_stream: { dataset: 'foo' }, // data_stream.dataset can be set in the compiled stream, ensure that rest of data_stream object is properly merged.
},
},
],
Expand Down
Loading
Loading