diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml index 96725197f9b39..10f8270c4eceb 100644 --- a/.buildkite/ftr_oblt_serverless_configs.yml +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -42,7 +42,7 @@ enabled: - x-pack/solutions/observability/test/serverless/functional/configs/config.telemetry.ts # serverless config files that run deployment-agnostic tests - x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.serverless.config.ts - - x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.serverless.config.ts + - x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.serverless.config.ts - x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts - x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.synthetics.serverless.config.ts - x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.serverless.config.ts diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json index fbba673b180ad..85be282c151d1 100644 --- a/oas_docs/bundle.json +++ b/oas_docs/bundle.json @@ -49428,6 +49428,9 @@ }, "type": "object" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "output_secret_storage_requirements_met": { "type": "boolean" }, @@ -49578,6 +49581,9 @@ "deprecated": true, "type": "boolean" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "kibana_ca_sha256": { "deprecated": true, "type": "string" @@ -49661,6 +49667,9 @@ }, "type": "object" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "output_secret_storage_requirements_met": { "type": "boolean" }, diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json index ffb664220c4d0..1848982250657 100644 --- a/oas_docs/bundle.serverless.json +++ b/oas_docs/bundle.serverless.json @@ -48916,6 +48916,9 @@ }, "type": "object" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "output_secret_storage_requirements_met": { "type": "boolean" }, @@ -49066,6 +49069,9 @@ "deprecated": true, "type": "boolean" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "kibana_ca_sha256": { "deprecated": true, "type": "string" @@ -49149,6 +49155,9 @@ }, "type": "object" }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "output_secret_storage_requirements_met": { "type": "boolean" }, diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index d9fea087fd3a2..ef680e2fe1979 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -47978,6 +47978,8 @@ paths: - success nullable: true type: string + integration_knowledge_enabled: + type: boolean output_secret_storage_requirements_met: type: boolean preconfigured_fields: @@ -48087,6 +48089,8 @@ paths: has_seen_add_data_notice: deprecated: true type: boolean + integration_knowledge_enabled: + type: boolean kibana_ca_sha256: deprecated: true type: string @@ -48146,6 +48150,8 @@ paths: - success nullable: true type: string + integration_knowledge_enabled: + type: boolean output_secret_storage_requirements_met: type: boolean preconfigured_fields: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 2d0f24de77b48..8d0d4cfa15c95 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -50915,6 +50915,8 @@ paths: - success nullable: true type: string + integration_knowledge_enabled: + type: boolean output_secret_storage_requirements_met: type: boolean preconfigured_fields: @@ -51024,6 +51026,8 @@ paths: has_seen_add_data_notice: deprecated: true type: boolean + integration_knowledge_enabled: + type: boolean kibana_ca_sha256: deprecated: true type: string @@ -51083,6 +51087,8 @@ paths: - success nullable: true type: string + integration_knowledge_enabled: + type: boolean output_secret_storage_requirements_met: type: boolean preconfigured_fields: diff --git a/packages/kbn-check-saved-objects-cli/current_fields.json b/packages/kbn-check-saved-objects-cli/current_fields.json index ba197685063c5..a38cd53340522 100644 --- a/packages/kbn-check-saved-objects-cli/current_fields.json +++ b/packages/kbn-check-saved-objects-cli/current_fields.json @@ -806,6 +806,7 @@ "fleet_server_hosts", "has_seen_add_data_notice", "ilm_migration_status", + "integration_knowledge_enabled", "output_secret_storage_requirements_met", "prerelease_integrations_enabled", "secret_storage_requirements_met", diff --git a/packages/kbn-check-saved-objects-cli/current_mappings.json b/packages/kbn-check-saved-objects-cli/current_mappings.json index ea48166617bcc..80c8775c78da9 100644 --- a/packages/kbn-check-saved-objects-cli/current_mappings.json +++ b/packages/kbn-check-saved-objects-cli/current_mappings.json @@ -2663,6 +2663,9 @@ "dynamic": false, "properties": {} }, + "integration_knowledge_enabled": { + "type": "boolean" + }, "output_secret_storage_requirements_met": { "type": "boolean" }, diff --git a/packages/kbn-check-saved-objects-cli/src/migrations/__fixtures__/ingest_manager_settings/10.7.0.json b/packages/kbn-check-saved-objects-cli/src/migrations/__fixtures__/ingest_manager_settings/10.7.0.json new file mode 100644 index 0000000000000..86ab87ac5333f --- /dev/null +++ b/packages/kbn-check-saved-objects-cli/src/migrations/__fixtures__/ingest_manager_settings/10.7.0.json @@ -0,0 +1,51 @@ +{ + "10.6.0": [ + { + "has_seen_add_data_notice": true, + "prerelease_integrations_enabled": false, + "id": "id", + "version": "1", + "preconfigured_fields": ["fleet_server_hosts"], + "secret_storage_requirements_met": true, + "output_secret_storage_requirements_met": true, + "action_secret_storage_requirements_met": true, + "use_space_awareness_migration_status": "success", + "use_space_awareness_migration_started_at": "2022-10-11T13:45:20.123Z", + "delete_unenrolled_agents": { + "enabled": true, + "is_preconfigured": false + }, + "ilm_migration_status": { + "logs": "success", + "metrics": "success", + "synthetics": null + }, + "ssl_secret_storage_requirements_met": true + } + ], + "10.7.0": [ + { + "has_seen_add_data_notice": true, + "prerelease_integrations_enabled": false, + "id": "id", + "version": "1", + "preconfigured_fields": ["fleet_server_hosts"], + "secret_storage_requirements_met": true, + "output_secret_storage_requirements_met": true, + "action_secret_storage_requirements_met": true, + "use_space_awareness_migration_status": "success", + "use_space_awareness_migration_started_at": "2022-10-11T13:45:20.123Z", + "delete_unenrolled_agents": { + "enabled": true, + "is_preconfigured": false + }, + "ilm_migration_status": { + "logs": "success", + "metrics": "success", + "synthetics": null + }, + "ssl_secret_storage_requirements_met": true, + "integration_knowledge_enabled": true + } + ] +} diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 4cdd01063964d..4475cf8918559 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -133,7 +133,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-download-sources": "c87e062ef293585e85fccec0c865d7cef48e0ff9a919d7781d5f7627d275484b", "ingest-outputs": "4f3451469b080548fd0f2ca414a81d91bd0d5690c34378376433ab1ae960ce5c", "ingest-package-policies": "9b9a83b94a99e0574a69999310b00917ddd5bfd44a6079badf54026029fed597", - "ingest_manager_settings": "ff92838e01da020f9a1fe0f9154a9769adb74489422adb518faec5459c9834e5", + "ingest_manager_settings": "535e93b324c94e9012cb185666a2a915e3c96db65ee1994a13beb066d1be7684", "intercept_interaction_record": "c0220185d080f25abd14c4c29f72b2e5270f3ce55a775c8c7ac9177e0f2c810d", "intercept_trigger_record": "e10a20733bbaaae951478e45c9f2cc70677299060222edd5a6f5db29591c5f96", "inventory-view": "f66434444d3576f801f112fc3584634054af081bbf6e78eb84a41462c62e53ba", @@ -830,8 +830,9 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-package-policies|warning: The SO type owner should ensure these transform functions DO NOT mutate after they are defined.", "================================================================================================================================", "ingest_manager_settings|global: aa0735c39adb396365f28cf118204854b9d5d71e", - "ingest_manager_settings|mappings: 73b7ee2e92003e1f56e13769012f3d8951b76605", + "ingest_manager_settings|mappings: 8c1c22339d8aaa9999dd904bcb03612ef17b51dc", "ingest_manager_settings|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709", + "ingest_manager_settings|10.7.0: c0e7e5407ddadc18ab645a2972f4f1831f91192ac50462a1e72ab9470f18269d", "ingest_manager_settings|10.6.0: 807307a9a72ab12f568dfa03131fce9b9a7414c1649418d40db06cff6c62ab5e", "ingest_manager_settings|10.5.0: cf8c901d4394ae835fb669c4512a89a633b3d3267648b0b1afb3bc0deb7613bd", "ingest_manager_settings|10.4.0: 764f19710e944dab032c05a10694aaf8b202d61786f6910898edf5c1e21b92f9", @@ -1332,7 +1333,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-download-sources": "10.1.0", "ingest-outputs": "10.8.0", "ingest-package-policies": "10.21.0", - "ingest_manager_settings": "10.6.0", + "ingest_manager_settings": "10.7.0", "intercept_interaction_record": "10.1.0", "intercept_trigger_record": "10.1.0", "inventory-view": "10.2.0", @@ -1480,7 +1481,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-download-sources": "10.1.0", "ingest-outputs": "10.8.0", "ingest-package-policies": "10.21.0", - "ingest_manager_settings": "10.6.0", + "ingest_manager_settings": "10.7.0", "intercept_interaction_record": "10.1.0", "intercept_trigger_record": "10.1.0", "inventory-view": "10.2.0", diff --git a/x-pack/platform/plugins/shared/fleet/common/experimental_features.ts b/x-pack/platform/plugins/shared/fleet/common/experimental_features.ts index da55584702bc0..bb8fa24f701d0 100644 --- a/x-pack/platform/plugins/shared/fleet/common/experimental_features.ts +++ b/x-pack/platform/plugins/shared/fleet/common/experimental_features.ts @@ -18,7 +18,7 @@ const _allowedExperimentalValues = { enableOtelIntegrations: true, enableAgentStatusAlerting: true, enableAgentPrivilegeLevelChange: false, - installIntegrationsKnowledge: false, + installIntegrationsKnowledge: true, enableFleetPolicyRevisionsCleanupTask: true, agentlessPoliciesAPI: true, // When enabled, agentless policies API will be enabled. useAgentlessAPIInUI: true, // When enabled, Fleet UI will use agentless policies API to create agentless policies. diff --git a/x-pack/platform/plugins/shared/fleet/common/types/models/settings.ts b/x-pack/platform/plugins/shared/fleet/common/types/models/settings.ts index c12bf84758215..0ca6e2178ac73 100644 --- a/x-pack/platform/plugins/shared/fleet/common/types/models/settings.ts +++ b/x-pack/platform/plugins/shared/fleet/common/types/models/settings.ts @@ -24,6 +24,7 @@ export interface BaseSettings { metrics?: 'success' | null; synthetics?: 'success' | null; }; + integration_knowledge_enabled?: boolean; } export interface Settings extends BaseSettings { diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/eis_cost_tour.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/eis_cost_tour.tsx new file mode 100644 index 0000000000000..70937d18295bd --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/eis_cost_tour.tsx @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import type { EuiTourStepProps } from '@elastic/eui'; +import { EuiButton, EuiButtonEmpty, EuiText, EuiTourStep, useEuiTheme } from '@elastic/eui'; + +import { useDismissableTour } from '../../../../../../../hooks'; + +export interface EisCostTourProps { + anchorPosition?: EuiTourStepProps['anchorPosition']; + ctaLink?: string; + isCloudEnabled: boolean; + children: React.ReactElement; +} + +export const EisCostTour = ({ + anchorPosition = 'downCenter', + ctaLink, + isCloudEnabled, + children, +}: EisCostTourProps) => { + const { euiTheme } = useEuiTheme(); + + const { isOpen, dismiss } = useDismissableTour('EIS_COSTS'); + + if (!isOpen || !isCloudEnabled) { + return children; + } + + return ( + +

+ {i18n.translate('xpack.fleet.eisCosts.tour.description', { + defaultMessage: + 'Integration documentation and metadata are automatically indexed using Elastic Inference Service (EIS) to help agents understand your environment. Manage settings here.', + })} +

+ + } + isStepOpen={isOpen} + anchorPosition={anchorPosition} + step={1} + stepsTotal={1} + onFinish={dismiss} + footerAction={[ + + {i18n.translate('xpack.fleet.eisCosts.tour.dismiss', { + defaultMessage: 'Dismiss', + })} + , + ...(ctaLink + ? [ + + {i18n.translate('xpack.fleet.eisCosts.tour.cta', { + defaultMessage: 'Learn more', + })} + , + ] + : []), + ]} + > + {children} + + ); +}; diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/installed_integration_action_menu.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/installed_integration_action_menu.tsx index 0d94230e55bf5..1d5bd29a0865b 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/installed_integration_action_menu.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/installed_integration_action_menu.tsx @@ -18,26 +18,31 @@ import { FormattedMessage } from '@kbn/i18n-react'; import type { InstalledPackageUIPackageListItem } from '../types'; import { useInstalledIntegrationsActions } from '../hooks/use_installed_integrations_actions'; import { ExperimentalFeaturesService } from '../../../../../services'; -import { useLicense } from '../../../../../hooks'; +import { useLicense, useStartServices } from '../../../../../hooks'; + +import { IntegrationKnowledgeFlyout } from './integration_knowledge_flyout'; +import { EisCostTour } from './eis_cost_tour'; export const InstalledIntegrationsActionMenu: React.FunctionComponent<{ selectedItems: InstalledPackageUIPackageListItem[]; }> = ({ selectedItems }) => { - const [isOpen, setIsOpen] = useState(false); + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const [showIntegrationKnowledgeFlyout, setShowIntegrationKnowledgeFlyout] = useState(false); const { enablePackageRollback } = ExperimentalFeaturesService.get(); const licenseService = useLicense(); + const { cloud, docLinks } = useStartServices(); const button = ( - setIsOpen((s) => !s)} + - - + setIsPopoverOpen((s) => !s)}> + + + ); const { @@ -49,20 +54,25 @@ export const InstalledIntegrationsActionMenu: React.FunctionComponent<{ } = useInstalledIntegrationsActions(); const openUpgradeModal = useCallback(() => { - setIsOpen(false); + setIsPopoverOpen(false); return bulkUpgradeIntegrationsWithConfirmModal(selectedItems); }, [selectedItems, bulkUpgradeIntegrationsWithConfirmModal]); const openUninstallModal = useCallback(async () => { - setIsOpen(false); + setIsPopoverOpen(false); return bulkUninstallIntegrationsWithConfirmModal(selectedItems); }, [selectedItems, bulkUninstallIntegrationsWithConfirmModal]); const openRollbackModal = useCallback(async () => { - setIsOpen(false); + setIsPopoverOpen(false); return bulkRollbackIntegrationsWithConfirmModal(selectedItems); }, [selectedItems, bulkRollbackIntegrationsWithConfirmModal]); + const openManageIntegrationKnowledgeFlyout = useCallback(() => { + setIsPopoverOpen(false); + setShowIntegrationKnowledgeFlyout(true); + }, []); + const items = useMemo(() => { const hasUpgreadableIntegrations = selectedItems.some( (item) => @@ -80,78 +90,100 @@ export const InstalledIntegrationsActionMenu: React.FunctionComponent<{ !!item.installationInfo?.previous_version && !item.installationInfo?.is_rollback_ttl_expired ); - return [ + const menuItems = [ , - - {hasUninstallableIntegrations ? ( - - } + ]; + + if (selectedItems.length > 0) { + menuItems.push( + ...[ + - - ) : ( - - )} - , - ...(enablePackageRollback - ? [ - + , + + {hasUninstallableIntegrations ? ( + + } + > + + + ) : ( - , - ] - : []), - ]; + )} + , + ...(enablePackageRollback + ? [ + + + , + ] + : []), + ] + ); + } + + return menuItems; }, [ selectedItems, openUninstallModal, @@ -159,18 +191,24 @@ export const InstalledIntegrationsActionMenu: React.FunctionComponent<{ openRollbackModal, enablePackageRollback, licenseService, + openManageIntegrationKnowledgeFlyout, ]); return ( - setIsOpen(false)} - panelPaddingSize="none" - anchorPosition="downLeft" - > - - + <> + setIsPopoverOpen(false)} + panelPaddingSize="none" + anchorPosition="downLeft" + > + + + {showIntegrationKnowledgeFlyout && ( + setShowIntegrationKnowledgeFlyout(false)} /> + )} + ); }; diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/integration_knowledge_flyout.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/integration_knowledge_flyout.tsx new file mode 100644 index 0000000000000..15ce1aa272217 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/installed_integrations/components/integration_knowledge_flyout.tsx @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiFlyoutHeader, + EuiLink, + EuiSwitch, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { i18n } from '@kbn/i18n'; + +import { + useAuthz, + useGetSettingsQuery, + usePutSettingsMutation, + useStartServices, +} from '../../../../../hooks'; +import { Loading } from '../../../../../components'; + +export const IntegrationKnowledgeFlyout: React.FunctionComponent<{ + onClose: () => void; +}> = ({ onClose }) => { + const [integrationKnowledgeSetting, setIntegrationKnowledgeSetting] = + React.useState(false); + + const authz = useAuthz(); + const { cloud, notifications, docLinks } = useStartServices(); + + const { data: settings, isInitialLoading: isSettingsInitialLoading } = useGetSettingsQuery({ + enabled: authz.fleet.readSettings, + }); + + useEffect(() => { + const isEnabled = Boolean(settings?.item.integration_knowledge_enabled); + setIntegrationKnowledgeSetting(isEnabled); + }, [settings?.item.integration_knowledge_enabled]); + + const { mutateAsync: mutateSettingsAsync } = usePutSettingsMutation(); + + const updateSettings = useCallback( + async (integrationKnowledgeEnabled: boolean) => { + try { + setIntegrationKnowledgeSetting(integrationKnowledgeEnabled); + const res = await mutateSettingsAsync({ + integration_knowledge_enabled: integrationKnowledgeEnabled, + }); + + if (res.error) { + throw res.error; + } + notifications.toasts.addSuccess({ + title: i18n.translate('xpack.fleet.integrationKnowledgeFlyout.successUpdatingSettings', { + defaultMessage: 'Integration knowledge setting updated successfully', + }), + }); + } catch (error) { + setIntegrationKnowledgeSetting(!integrationKnowledgeEnabled); + notifications.toasts.addError(error, { + title: i18n.translate('xpack.fleet.integrationKnowledgeFlyout.errorUpdatingSettings', { + defaultMessage: 'Error updating integration knowledge setting', + }), + }); + } + }, + [mutateSettingsAsync, notifications.toasts] + ); + + const saveIntegrationKnowledgeSetting = (setting: boolean) => { + updateSettings(setting); + onClose(); + }; + const costNodes = + cloud?.isCloudEnabled || cloud?.isServerlessEnabled ? ( + <> +

+ +

+

+ {' '} + + + +

+ + ) : null; + return ( + + + +

+ +

+ + + + {isSettingsInitialLoading ? ( + + ) : ( + + + +

+ +

+

+ + + + ), + aiAssistant: ( + + + + ), + }} + /> +

+

+ +

+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+ {costNodes} + + + + setIntegrationKnowledgeSetting(e.target.checked)} + label={ + + } + /> + + + )} + + + + + + + + + + saveIntegrationKnowledgeSetting(integrationKnowledgeSetting)} + aria-label="Save" + disabled={ + integrationKnowledgeSetting === + Boolean(settings?.item.integration_knowledge_enabled) + } + > + + + + + + + ); +}; diff --git a/x-pack/platform/plugins/shared/fleet/public/constants/index.ts b/x-pack/platform/plugins/shared/fleet/public/constants/index.ts index b923c29019c4e..097f2f4585f63 100644 --- a/x-pack/platform/plugins/shared/fleet/public/constants/index.ts +++ b/x-pack/platform/plugins/shared/fleet/public/constants/index.ts @@ -57,6 +57,7 @@ export const TOUR_STORAGE_KEYS = { GRANULAR_PRIVILEGES: 'fleet.granularPrivileges', AGENT_EXPORT_CSV: 'fleet.agentExportCSVTour', AUTO_UPGRADE_AGENTS: 'fleet.autoUpgradeAgentsTour', + EIS_COSTS: 'fleet.eisCostsTour', }; export interface TourConfig { diff --git a/x-pack/platform/plugins/shared/fleet/server/plugin.ts b/x-pack/platform/plugins/shared/fleet/server/plugin.ts index c49837171a275..6cf409cb81dfd 100644 --- a/x-pack/platform/plugins/shared/fleet/server/plugin.ts +++ b/x-pack/platform/plugins/shared/fleet/server/plugin.ts @@ -160,6 +160,7 @@ import { registerAgentlessDeploymentSyncTask, scheduleAgentlessDeploymentSyncTask, } from './tasks/agentless/deployment_sync_task'; +import { registerReindexIntegrationKnowledgeTask } from './tasks/reindex_integration_knowledge_task'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -664,6 +665,7 @@ export class FleetPlugin registerPackagesBulkOperationTask(deps.taskManager); registerSetupTasks(deps.taskManager); registerAgentlessDeploymentSyncTask(deps.taskManager, this.configInitialValue); + registerReindexIntegrationKnowledgeTask(deps.taskManager); this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core); this.checkDeletedFilesTask = new CheckDeletedFilesTask({ diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.ts b/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.ts index 13c1a1317ebf8..eeda186a4bc4a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.ts @@ -12,8 +12,9 @@ import type { PutSettingsRequestSchema, PutSpaceSettingsRequestSchema, } from '../../types'; -import { settingsService } from '../../services'; +import { appContextService, settingsService } from '../../services'; import { getSpaceSettings, saveSpaceSettings } from '../../services/spaces/space_settings'; +import { scheduleReindexIntegrationKnowledgeTask } from '../../tasks/reindex_integration_knowledge_task'; export const getSpaceSettingsHandler: FleetRequestHandler = async (context, request, response) => { const soClient = (await context.fleet).internalSoClient; @@ -73,6 +74,10 @@ export const putSettingsHandler: FleetRequestHandler< try { const settings = await settingsService.saveSettings(soClient, request.body); + if (request.body.integration_knowledge_enabled) { + await scheduleReindexIntegrationKnowledgeTask(appContextService.getTaskManagerStart()!); + } + const body = { item: settings, }; diff --git a/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts b/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts index 756b102d3b956..506c6a09b8276 100644 --- a/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts @@ -34,7 +34,7 @@ import { SPACE_SETTINGS_SAVED_OBJECT_TYPE, } from '../constants'; -import { SettingsSchemaV5, SettingsSchemaV6 } from '../types'; +import { SettingsSchemaV5, SettingsSchemaV6, SettingsSchemaV7 } from '../types'; import { migrateSyntheticsPackagePolicyToV8120 } from './migrations/synthetics/to_v8_12_0'; @@ -180,6 +180,7 @@ export const getSavedObjectTypes = ( dynamic: false, properties: {}, }, + integration_knowledge_enabled: { type: 'boolean' }, ssl_secret_storage_requirements_met: { type: 'boolean' }, }, }, @@ -257,6 +258,20 @@ export const getSavedObjectTypes = ( create: SettingsSchemaV6, }, }, + 7: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + integration_knowledge_enabled: { type: 'boolean' }, + }, + }, + ], + schemas: { + forwardCompatibility: SettingsSchemaV7.extends({}, { unknowns: 'ignore' }), + create: SettingsSchemaV7, + }, + }, }, }, [LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.test.ts index 31efe6b259690..479536f969f9a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.test.ts @@ -1710,7 +1710,8 @@ owner: elastic`, expect(mockKnowledgeBaseIndex.getPackageKnowledgeBaseFromIndex).toHaveBeenCalledWith( esClient, - 'nginx' + 'nginx', + undefined ); expect(result).toEqual({ @@ -1747,7 +1748,8 @@ owner: elastic`, expect(mockKnowledgeBaseIndex.getPackageKnowledgeBaseFromIndex).toHaveBeenCalledWith( esClient, - 'nginx' + 'nginx', + undefined ); expect(result).toEqual({ @@ -1769,7 +1771,8 @@ owner: elastic`, expect(mockKnowledgeBaseIndex.getPackageKnowledgeBaseFromIndex).toHaveBeenCalledWith( esClient, - 'nginx' + 'nginx', + undefined ); expect(result).toBeUndefined(); @@ -1791,7 +1794,8 @@ owner: elastic`, expect(mockKnowledgeBaseIndex.getPackageKnowledgeBaseFromIndex).toHaveBeenCalledWith( esClient, - 'nginx' + 'nginx', + undefined ); expect(mockLogger.warn).toHaveBeenCalledWith( @@ -1814,7 +1818,8 @@ owner: elastic`, expect(mockKnowledgeBaseIndex.getPackageKnowledgeBaseFromIndex).toHaveBeenCalledWith( esClient, - '' + '', + undefined ); expect(result).toBeUndefined(); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts index 50799bb97bbb2..e712475ce0137 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts @@ -978,11 +978,16 @@ export async function getAgentTemplateAssetsMap({ export async function getPackageKnowledgeBase(options: { esClient: ElasticsearchClient; pkgName: string; + abortController?: AbortController; }): Promise { - const { esClient, pkgName } = options; + const { esClient, pkgName, abortController } = options; try { - const knowledgeBaseItems = await getPackageKnowledgeBaseFromIndex(esClient, pkgName); + const knowledgeBaseItems = await getPackageKnowledgeBaseFromIndex( + esClient, + pkgName, + abortController + ); if (knowledgeBaseItems.length === 0) { return undefined; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.test.ts new file mode 100644 index 0000000000000..fbcf37b3ff5b9 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; + +import { getSettings } from '../../settings'; +import { appContextService } from '../../app_context'; + +import { getIntegrationKnowledgeSetting } from './get_integration_knowledge_setting'; + +jest.mock('../../settings'); + +jest.mock('../../app_context'); + +describe('getIntegrationKnowledgeSetting', () => { + const mockSoClient = {} as jest.Mocked; + + it('should return true if feature flag is enabled and no user setting', async () => { + (appContextService.getExperimentalFeatures as jest.Mock).mockReturnValue({ + installIntegrationsKnowledge: true, + }); + (getSettings as jest.Mock).mockResolvedValue({}); + + const result = await getIntegrationKnowledgeSetting(mockSoClient); + expect(result).toBe(true); + }); + + it('should return false if config is disabled and no user setting', async () => { + (appContextService.getConfig as jest.Mock).mockReturnValue({ + experimentalFeatures: { + integrationKnowledge: false, + }, + }); + (appContextService.getExperimentalFeatures as jest.Mock).mockReturnValue({ + installIntegrationsKnowledge: true, + }); + (getSettings as jest.Mock).mockResolvedValue({}); + + const result = await getIntegrationKnowledgeSetting(mockSoClient); + expect(result).toBe(false); + }); + + it('should return false if feature flag is enabled and user setting is disabled', async () => { + (appContextService.getExperimentalFeatures as jest.Mock).mockReturnValue({ + installIntegrationsKnowledge: true, + }); + (getSettings as jest.Mock).mockResolvedValue({ + integration_knowledge_enabled: false, + }); + + const result = await getIntegrationKnowledgeSetting(mockSoClient); + expect(result).toBe(false); + }); + + it('should return true if config is disabled and user setting is enabled', async () => { + (appContextService.getConfig as jest.Mock).mockReturnValue({ + experimentalFeatures: { + integrationKnowledge: false, + }, + }); + (appContextService.getExperimentalFeatures as jest.Mock).mockReturnValue({ + installIntegrationsKnowledge: true, + }); + (getSettings as jest.Mock).mockResolvedValue({ + integration_knowledge_enabled: true, + }); + + const result = await getIntegrationKnowledgeSetting(mockSoClient); + expect(result).toBe(true); + }); +}); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.ts new file mode 100644 index 0000000000000..b24c3faf9f196 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_integration_knowledge_setting.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; + +import { appContextService } from '../../app_context'; +import { getSettings } from '../../settings'; + +export async function getIntegrationKnowledgeSetting( + savedObjectsClient: SavedObjectsClientContract +): Promise { + const config = appContextService.getConfig(); + + const integrationKnowledgeConfig: boolean = + config?.experimentalFeatures?.integrationKnowledge ?? + appContextService.getExperimentalFeatures().installIntegrationsKnowledge; + try { + const { integration_knowledge_enabled: integrationKnowledgeEnabled } = await getSettings( + savedObjectsClient + ); + return integrationKnowledgeEnabled ?? integrationKnowledgeConfig; + } catch (err) { + appContextService + .getLogger() + .warn( + 'Error while trying to load integration knowledge flag from settings, defaulting to feature flag', + err + ); + } + return integrationKnowledgeConfig; +} diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts index 756e79deb2d51..13d4a577836e6 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts @@ -107,7 +107,7 @@ const MAX_ENSURE_INSTALL_TIME = 60 * 1000; const MAX_INSTALL_RETRIES = 5; const BASE_RETRY_DELAY_MS = 1000; // 1s -const PACKAGES_TO_INSTALL_WITH_STREAMING = [ +export const PACKAGES_TO_INSTALL_WITH_STREAMING = [ // The security_detection_engine package contains a large number of assets and // is not suitable for regular installation as it might cause OOM errors. 'security_detection_engine', diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.test.ts index a9ad14f594a32..7b28fe8305c93 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.test.ts @@ -13,6 +13,7 @@ import { saveKnowledgeBaseContentToIndex } from '../../knowledge_base_index'; import type { InstallContext } from '../_state_machine_package_install'; import { stepSaveKnowledgeBase, cleanupKnowledgeBaseStep } from './step_save_knowledge_base'; +import { getIntegrationKnowledgeSetting } from '../../get_integration_knowledge_setting'; // Mock the app context service jest.mock('../../../../app_context', () => ({ @@ -23,9 +24,6 @@ jest.mock('../../../../app_context', () => ({ info: jest.fn(), debug: jest.fn(), }), - getExperimentalFeatures: jest.fn().mockReturnValue({ - installIntegrationsKnowledge: true, - }), }, })); @@ -49,6 +47,10 @@ jest.mock('../../../../license', () => ({ }, })); +jest.mock('../../get_integration_knowledge_setting', () => ({ + getIntegrationKnowledgeSetting: jest.fn().mockResolvedValue(true), +})); + let esClient: jest.Mocked; let savedObjectsClient: jest.Mocked; @@ -698,11 +700,7 @@ describe('stepSaveKnowledgeBase', () => { }); it('should skip knowledge base processing when installIntegrationsKnowledge feature flag is disabled', async () => { - // Mock app context service to return experimental features with flag disabled - const { appContextService } = jest.requireMock('../../../../app_context'); - appContextService.getExperimentalFeatures.mockReturnValue({ - installIntegrationsKnowledge: false, - }); + (getIntegrationKnowledgeSetting as jest.Mock).mockResolvedValueOnce(false); const entries: ArchiveEntry[] = [ { @@ -722,20 +720,9 @@ describe('stepSaveKnowledgeBase', () => { // Verify that updateEsAssetReferences was NOT called const { updateEsAssetReferences } = jest.requireMock('../../es_assets_reference'); expect(updateEsAssetReferences).not.toHaveBeenCalled(); - - // Reset the mock back to enabled for other tests - appContextService.getExperimentalFeatures.mockReturnValue({ - installIntegrationsKnowledge: true, - }); }); it('should process knowledge base when installIntegrationsKnowledge feature flag is enabled', async () => { - // Ensure app context service returns experimental features with flag enabled - const { appContextService } = jest.requireMock('../../../../app_context'); - appContextService.getExperimentalFeatures.mockReturnValue({ - installIntegrationsKnowledge: true, - }); - const entries: ArchiveEntry[] = [ { path: 'test-package-1.0.0/docs/knowledge_base/guide.md', diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.ts index 5860c266e846f..29cd3d4850234 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_knowledge_base.ts @@ -7,6 +7,10 @@ import path from 'path'; +import type { Logger } from '@kbn/core/server'; + +import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; + import { FleetError } from '../../../../../errors'; import { saveKnowledgeBaseContentToIndex, @@ -24,7 +28,7 @@ import type { EsAssetReference } from '../../../../../types'; import { updateEsAssetReferences } from '../../es_assets_reference'; import type { KnowledgeBaseItem } from '../../../../../../common/types/models/epm'; import { licenseService } from '../../../../license'; -import { appContextService } from '../../../../app_context'; +import { getIntegrationKnowledgeSetting } from '../../get_integration_knowledge_setting'; export const KNOWLEDGE_BASE_PATH = 'docs/knowledge_base/'; export const DOCS_PATH_PATTERN = '/docs/'; export const KNOWLEDGE_BASE_FOLDER = 'knowledge_base/'; @@ -82,17 +86,18 @@ export async function stepSaveKnowledgeBase( const { packageInstallContext, esClient, savedObjectsClient, logger } = context; const { packageInfo, archiveIterator } = packageInstallContext; - let esReferences = context.esReferences ?? []; + const esReferences = context.esReferences ?? []; logger.debug( `Knowledge base step: Starting for package ${packageInfo.name}@${packageInfo.version}` ); - // Check if knowledge base installation is enabled via experimental feature flag - const experimentalFeatures = appContextService.getExperimentalFeatures(); - if (!experimentalFeatures.installIntegrationsKnowledge) { + const integrationKnowledgeEnabled = await getIntegrationKnowledgeSetting(savedObjectsClient); + + // Check if knowledge base installation is enabled via user setting + if (!integrationKnowledgeEnabled) { logger.debug( - `Knowledge base step: Skipping knowledge base save - installIntegrationsKnowledge experimental feature is disabled` + `Knowledge base step: Skipping knowledge base save - integration knowledge enabled setting is disabled` ); return { esReferences }; } @@ -104,6 +109,25 @@ export async function stepSaveKnowledgeBase( return { esReferences }; } + return await indexKnowledgeBase( + esReferences, + savedObjectsClient, + esClient, + logger, + packageInfo, + archiveIterator + ); +} + +export async function indexKnowledgeBase( + esReferences: EsAssetReference[], + savedObjectsClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + logger: Logger, + packageInfo: { name: string; version: string }, + archiveIterator: ArchiveIterator, + abortController?: AbortController +): Promise<{ esReferences: EsAssetReference[] }> { // Extract knowledge base content directly from the archive const knowledgeBaseItems = await extractKnowledgeBaseFromArchive( archiveIterator, @@ -111,7 +135,9 @@ export async function stepSaveKnowledgeBase( packageInfo.version ); - logger.debug(`Knowledge base step: Found ${knowledgeBaseItems.length} items to process`); + logger.debug( + `Knowledge base step: Found ${knowledgeBaseItems.length} items to process for package ${packageInfo.name}@${packageInfo.version}` + ); // Save knowledge base content if present if (knowledgeBaseItems && knowledgeBaseItems.length > 0) { @@ -123,9 +149,12 @@ export async function stepSaveKnowledgeBase( pkgName: packageInfo.name, pkgVersion: packageInfo.version, knowledgeBaseContent: knowledgeBaseItems, + abortController, }); - logger.debug(`Knowledge base step: Saved ${documentIds.length} documents to index`); + logger.debug( + `Knowledge base step: Saved ${documentIds.length} documents to index for package ${packageInfo.name}@${packageInfo.version}` + ); // Add knowledge base asset references using the ES-generated document IDs const knowledgeBaseAssetRefs = documentIds.map((docId) => ({ diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.test.ts index 653a1e46b40bf..f84311c23be84 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.test.ts @@ -74,34 +74,40 @@ describe('knowledge_base_index', () => { const afterCall = new Date().toISOString(); - expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query: { + match: { package_name: 'test-package' }, + }, }, - }); + undefined + ); - expect(mockEsClient.bulk).toHaveBeenCalledWith({ - operations: [ - { index: { _index: INTEGRATION_KNOWLEDGE_INDEX, _id: 'test-package-test1.md' } }, - { - package_name: 'test-package', - filename: 'test1.md', - content: 'Test content 1', - version: '1.0.0', - installed_at: expect.any(String), - }, - { index: { _index: INTEGRATION_KNOWLEDGE_INDEX, _id: 'test-package-test2.md' } }, - { - package_name: 'test-package', - filename: 'test2.md', - content: 'Test content 2', - version: '1.0.0', - installed_at: expect.any(String), - }, - ], - refresh: 'wait_for', - }); + expect(mockEsClient.bulk).toHaveBeenCalledWith( + { + operations: [ + { index: { _index: INTEGRATION_KNOWLEDGE_INDEX, _id: 'test-package-test1.md' } }, + { + package_name: 'test-package', + filename: 'test1.md', + content: 'Test content 1', + version: '1.0.0', + installed_at: expect.any(String), + }, + { index: { _index: INTEGRATION_KNOWLEDGE_INDEX, _id: 'test-package-test2.md' } }, + { + package_name: 'test-package', + filename: 'test2.md', + content: 'Test content 2', + version: '1.0.0', + installed_at: expect.any(String), + }, + ], + refresh: 'wait_for', + }, + undefined + ); // Verify the function returns the expected document IDs expect(result).toEqual(['test-package-test1.md', 'test-package-test2.md']); @@ -134,12 +140,15 @@ describe('knowledge_base_index', () => { knowledgeBaseContent: [], }); - expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query: { + match: { package_name: 'test-package' }, + }, }, - }); + undefined + ); expect(mockEsClient.bulk).not.toHaveBeenCalled(); expect(result).toEqual([]); }); @@ -152,12 +161,15 @@ describe('knowledge_base_index', () => { knowledgeBaseContent: undefined as any, }); - expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query: { + match: { package_name: 'test-package' }, + }, }, - }); + undefined + ); expect(mockEsClient.bulk).not.toHaveBeenCalled(); expect(result).toEqual([]); }); @@ -192,13 +204,16 @@ describe('knowledge_base_index', () => { const result = await getPackageKnowledgeBaseFromIndex(mockEsClient, 'test-package'); - expect(mockEsClient.search).toHaveBeenCalledWith({ - index: INTEGRATION_KNOWLEDGE_INDEX, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.search).toHaveBeenCalledWith( + { + index: INTEGRATION_KNOWLEDGE_INDEX, + query: { + match: { package_name: 'test-package' }, + }, + size: 1000, }, - size: 1000, - }); + undefined + ); expect(result).toEqual([ { @@ -243,23 +258,29 @@ describe('knowledge_base_index', () => { it('should delete by package name only', async () => { await deletePackageKnowledgeBase(mockEsClient, 'test-package'); - expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query: { + match: { package_name: 'test-package' }, + }, }, - }); + undefined + ); }); it('should delete by package name', async () => { await deletePackageKnowledgeBase(mockEsClient, 'test-package'); - expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query: { - match: { package_name: 'test-package' }, + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query: { + match: { package_name: 'test-package' }, + }, }, - }); + undefined + ); }); }); }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.ts index cc902653f819d..f46e0cab19289 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/knowledge_base_index.ts @@ -22,11 +22,13 @@ export async function saveKnowledgeBaseContentToIndex({ pkgName, pkgVersion, knowledgeBaseContent, + abortController, }: { esClient: ElasticsearchClient; pkgName: string; pkgVersion: string; knowledgeBaseContent: KnowledgeBaseItem[]; + abortController?: AbortController; }): Promise { // Always delete existing documents for this package (regardless of version) // This ensures we only have one set of docs per package @@ -62,10 +64,17 @@ export async function saveKnowledgeBaseContentToIndex({ if (operations.length > 0) { const bulkResponse = await retryTransientEsErrors( async () => - esClient.bulk({ - operations, - refresh: 'wait_for', - }), + esClient.bulk( + { + operations, + refresh: 'wait_for', + }, + abortController + ? { + signal: abortController.signal, + } + : undefined + ), { logger: appContextService.getLogger() } ).catch((error) => { const logger = appContextService.getLogger(); @@ -111,18 +120,26 @@ export async function saveKnowledgeBaseContentToIndex({ export async function getPackageKnowledgeBaseFromIndex( esClient: ElasticsearchClient, - pkgName: string + pkgName: string, + abortController?: AbortController ): Promise { try { const query = { match: { package_name: pkgName }, }; - const response = await esClient.search({ - index: INTEGRATION_KNOWLEDGE_INDEX, - query, - size: DEFAULT_SIZE, - }); + const response = await esClient.search( + { + index: INTEGRATION_KNOWLEDGE_INDEX, + query, + size: DEFAULT_SIZE, + }, + abortController + ? { + signal: abortController.signal, + } + : undefined + ); return response.hits.hits.map((hit: any) => ({ fileName: hit._source.filename, @@ -139,16 +156,27 @@ export async function getPackageKnowledgeBaseFromIndex( } } -export async function deletePackageKnowledgeBase(esClient: ElasticsearchClient, pkgName: string) { +export async function deletePackageKnowledgeBase( + esClient: ElasticsearchClient, + pkgName: string, + abortController?: AbortController +) { const query = { match: { package_name: pkgName }, }; await esClient - .deleteByQuery({ - index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, - query, - }) + .deleteByQuery( + { + index: `${INTEGRATION_KNOWLEDGE_INDEX}*`, + query, + }, + abortController + ? { + signal: abortController.signal, + } + : undefined + ) .catch((error) => { const logger = appContextService.getLogger(); logger.error('Delete operation failed', error); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/settings.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/settings.test.ts index 72cb2e2ade4d7..f72960034515b 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/settings.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/settings.test.ts @@ -190,6 +190,98 @@ describe('settingsSetup', () => { expect(soClientMock.update).not.toHaveBeenCalled(); }); + + it('should update integration_knowledge_enabled if feature flag is enabled', async () => { + const soClientMock = savedObjectsClientMock.create(); + mockedAppContextService.getExperimentalFeatures.mockReturnValue({ + installIntegrationsKnowledge: true, + } as any); + + mockedAppContextService.getConfig.mockReturnValue({ + enabled: false, + agents: { + enabled: false, + elasticsearch: { + hosts: undefined, + ca_sha256: undefined, + ca_trusted_fingerprint: undefined, + }, + fleet_server: undefined, + }, + }); + + soClientMock.find.mockResolvedValue({ + total: 1, + page: 1, + per_page: 10, + saved_objects: [ + { + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + }); + + soClientMock.update.mockResolvedValueOnce({ + id: GLOBAL_SETTINGS_ID, + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + attributes: { integration_knowledge_enabled: true }, + references: [], + }); + + await settingsSetup(soClientMock); + + expect(soClientMock.update).toHaveBeenCalled(); + }); + + it('should update integration_knowledge_enabled if config is enabled', async () => { + const soClientMock = savedObjectsClientMock.create(); + + mockedAppContextService.getConfig.mockReturnValue({ + enabled: false, + agents: { + enabled: false, + elasticsearch: { + hosts: undefined, + ca_sha256: undefined, + ca_trusted_fingerprint: undefined, + }, + fleet_server: undefined, + }, + experimentalFeatures: { + integrationKnowledge: true, + }, + }); + + soClientMock.find.mockResolvedValue({ + total: 1, + page: 1, + per_page: 10, + saved_objects: [ + { + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + }); + + soClientMock.update.mockResolvedValueOnce({ + id: GLOBAL_SETTINGS_ID, + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + attributes: { integration_knowledge_enabled: true }, + references: [], + }); + + await settingsSetup(soClientMock); + + expect(soClientMock.update).toHaveBeenCalled(); + }); }); describe('getSettings', () => { @@ -552,4 +644,55 @@ describe('createDefaultSettings', () => { use_space_awareness_migration_status: 'success', }); }); + + it('should return default settings with integration_knowledge_enabled:true if feature flag is enabled', () => { + mockedAppContextService.getConfig.mockReturnValue({ + prereleaseEnabledByDefault: true, + enabled: true, + agents: { + enabled: false, + elasticsearch: { + hosts: undefined, + ca_sha256: undefined, + ca_trusted_fingerprint: undefined, + }, + }, + }); + + mockedAppContextService.getExperimentalFeatures.mockReturnValue({ + installIntegrationsKnowledge: true, + } as any); + + const result = createDefaultSettings(); + + expect(result).toEqual({ + integration_knowledge_enabled: true, + prerelease_integrations_enabled: true, + }); + }); + + it('should return default settings with integration_knowledge_enabled:true if config is enabled', () => { + mockedAppContextService.getConfig.mockReturnValue({ + prereleaseEnabledByDefault: true, + enabled: true, + agents: { + enabled: false, + elasticsearch: { + hosts: undefined, + ca_sha256: undefined, + ca_trusted_fingerprint: undefined, + }, + }, + experimentalFeatures: { + integrationKnowledge: true, + }, + }); + + const result = createDefaultSettings(); + + expect(result).toEqual({ + integration_knowledge_enabled: true, + prerelease_integrations_enabled: true, + }); + }); }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/settings.ts b/x-pack/platform/plugins/shared/fleet/server/services/settings.ts index 3f79cc90f5467..c93cd860323fb 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/settings.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/settings.ts @@ -41,6 +41,7 @@ function mapSettingsSO(settingsSo: SavedObject): Settings preconfigured_fields: getConfigFleetServerHosts() ? ['fleet_server_hosts'] : [], delete_unenrolled_agents: settingsSo.attributes.delete_unenrolled_agents, ilm_migration_status: settingsSo.attributes.ilm_migration_status, + integration_knowledge_enabled: settingsSo.attributes.integration_knowledge_enabled, }; } @@ -78,10 +79,20 @@ export async function settingsSetup(soClient: SavedObjectsClientContract) { try { const config = appContextService.getConfig(); const settings = await getSettings(soClient); + + const updatedSettings = {} as Partial; if (config?.prereleaseEnabledByDefault && !settings.prerelease_integrations_enabled) { - await saveSettings(soClient, { - prerelease_integrations_enabled: config?.prereleaseEnabledByDefault, - }); + updatedSettings.prerelease_integrations_enabled = config?.prereleaseEnabledByDefault; + } + if ( + (config?.experimentalFeatures?.integrationKnowledge ?? + appContextService.getExperimentalFeatures().installIntegrationsKnowledge) && + settings.integration_knowledge_enabled === undefined + ) { + updatedSettings.integration_knowledge_enabled = true; + } + if (Object.keys(updatedSettings).length > 0) { + await saveSettings(soClient, updatedSettings); } } catch (e) { if (e.isBoom && e.output.statusCode === 404) { @@ -185,5 +196,12 @@ export function createDefaultSettings(): BaseSettings { settings.use_space_awareness_migration_status = 'success'; } + if ( + config?.experimentalFeatures?.integrationKnowledge ?? + appContextService.getExperimentalFeatures().installIntegrationsKnowledge + ) { + settings.integration_knowledge_enabled = true; + } + return settings; } diff --git a/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.test.ts b/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.test.ts new file mode 100644 index 0000000000000..e682b9d66f91b --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.test.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { appContextService } from '../services'; +import { getInstallations, getPackageKnowledgeBase } from '../services/epm/packages'; +import { indexKnowledgeBase } from '../services/epm/packages/install_state_machine/steps'; + +import { reindexIntegrationKnowledgeForInstalledPackages } from './reindex_integration_knowledge_task'; + +jest.mock('../services'); +jest.mock('../services/epm/packages'); +jest.mock('../services/epm/registry', () => ({ + getPackage: jest.fn().mockResolvedValue({ + archiveIterator: {}, + }), +})); +jest.mock('../services/epm/packages/install_state_machine/steps', () => ({ + indexKnowledgeBase: jest.fn().mockResolvedValue(undefined), +})); +jest.mock('../services/epm/packages/bundled_packages', () => ({ + getBundledPackageForInstallation: jest.fn().mockResolvedValue({ + getBuffer: jest.fn().mockResolvedValue(Buffer.from('')), + }), +})); +jest.mock('../services/epm/archive', () => ({ + unpackBufferToAssetsMap: jest.fn().mockResolvedValue({ + archiveIterator: {}, + }), +})); + +describe('ReindexIntegrationKnowledgeTask', () => { + const abortController = new AbortController(); + beforeEach(() => { + (appContextService.getLogger as jest.Mock).mockReturnValue({ + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should not reindex knowledge base if already indexed for package version', async () => { + (getInstallations as jest.Mock).mockResolvedValue({ + saved_objects: [ + { + attributes: { + name: 'test-package', + version: '1.0.0', + install_source: 'registry', + }, + }, + ], + }); + (getPackageKnowledgeBase as jest.Mock).mockResolvedValue({ + items: [{ version: '1.0.0' }], + }); + + await reindexIntegrationKnowledgeForInstalledPackages(abortController); + + expect(indexKnowledgeBase).not.toHaveBeenCalled(); + }); + + it('should reindex knowledge base if not indexed for package version', async () => { + (getInstallations as jest.Mock).mockResolvedValue({ + saved_objects: [ + { + attributes: { + name: 'test-package', + version: '1.0.0', + install_source: 'registry', + }, + }, + { + attributes: { + name: 'test-bundled', + version: '2.0.0', + install_source: 'bundled', + }, + }, + { + attributes: { + name: 'test-upload', + version: '1.0.0', + install_source: 'upload', + }, + }, + { + attributes: { + name: 'test-custom', + version: '1.0.0', + install_source: 'custom', + }, + }, + ], + }); + (getPackageKnowledgeBase as jest.Mock).mockResolvedValue({ + items: [{ version: '0.0.1' }], + }); + + await reindexIntegrationKnowledgeForInstalledPackages(abortController); + + expect(indexKnowledgeBase).toHaveBeenCalledTimes(2); + expect(indexKnowledgeBase).toHaveBeenCalledWith( + undefined, + undefined, + undefined, + expect.anything(), + { name: 'test-package', version: '1.0.0' }, + expect.anything(), + expect.anything() + ); + expect(indexKnowledgeBase).toHaveBeenCalledWith( + undefined, + undefined, + undefined, + expect.anything(), + { name: 'test-bundled', version: '2.0.0' }, + expect.anything(), + expect.anything() + ); + }); +}); diff --git a/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.ts b/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.ts new file mode 100644 index 0000000000000..a4ccd9fc4569e --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/server/tasks/reindex_integration_knowledge_task.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { v4 as uuidv4 } from 'uuid'; + +import pMap from 'p-map'; + +import * as Registry from '../services/epm/registry'; +import { getInstallations, getPackageKnowledgeBase } from '../services/epm/packages'; +import { appContextService, licenseService } from '../services'; +import { MAX_CONCURRENT_EPM_PACKAGES_INSTALLATIONS } from '../constants'; +import { indexKnowledgeBase } from '../services/epm/packages/install_state_machine/steps'; +import { unpackBufferToAssetsMap } from '../services/epm/archive'; +import { getBundledPackageForInstallation } from '../services/epm/packages/bundled_packages'; +import { PACKAGES_TO_INSTALL_WITH_STREAMING } from '../services/epm/packages/install'; + +import { throwIfAborted } from './utils'; + +const TASK_TYPE = 'fleet:reindex_integration_knowledge'; + +export function registerReindexIntegrationKnowledgeTask( + taskManagerSetup: TaskManagerSetupContract +) { + taskManagerSetup.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Fleet Reindex integration knowledge', + timeout: '5m', + maxAttempts: 3, + createTaskRunner: ({ abortController }: { abortController: AbortController }) => { + return { + async run() { + // Check if user has appropriate license for knowledge base functionality + if (!licenseService.isEnterprise()) { + appContextService + .getLogger() + .debug(`Skipping knowledge base reindexing - requires Enterprise license`); + return; + } + + await reindexIntegrationKnowledgeForInstalledPackages(abortController); + }, + }; + }, + }, + }); +} + +export async function scheduleReindexIntegrationKnowledgeTask( + taskManagerStart: TaskManagerStartContract +) { + appContextService + .getLogger() + .info('Scheduling task to reindex integration knowledge for installed packages'); + await taskManagerStart.ensureScheduled({ + id: `${TASK_TYPE}:${uuidv4()}`, + scope: ['fleet'], + params: {}, + taskType: TASK_TYPE, + runAt: new Date(), + state: {}, + }); +} + +export async function reindexIntegrationKnowledgeForInstalledPackages( + abortController: AbortController +) { + const soClient = appContextService.getInternalUserSOClientWithoutSpaceExtension(); + const esClient = appContextService.getInternalUserESClient(); + const logger = appContextService.getLogger(); + const installedPackages = await getInstallations(soClient); + await pMap( + installedPackages.saved_objects, + async ({ attributes: installation }) => { + throwIfAborted(abortController); + const knowledgeBase = await getPackageKnowledgeBase({ + esClient, + pkgName: installation.name, + abortController, + }); + const everyKnowledgeBaseOnCurrentPackageVersion = knowledgeBase?.items.every( + (item) => item.version === installation.version + ); + if (everyKnowledgeBaseOnCurrentPackageVersion) { + logger.debug( + `Skipping reindexing knowledge base for package ${installation.name}@${installation.version} - already indexed` + ); + return; + } + + let archiveIterator; + if (installation.install_source === 'bundled') { + const matchingBundledPackage = await getBundledPackageForInstallation(installation); + if (!matchingBundledPackage) { + logger.debug( + `Skipping reindexing knowledge base for package ${installation.name}@${installation.version} - bundled package not found` + ); + return; + } + const useStreaming = PACKAGES_TO_INSTALL_WITH_STREAMING.includes(installation.name); + const archiveBuffer = await matchingBundledPackage.getBuffer(); + ({ archiveIterator } = await unpackBufferToAssetsMap({ + archiveBuffer, + contentType: 'application/zip', + useStreaming, + })); + } else if (installation.install_source !== 'registry') { + logger.debug( + `Skipping reindexing knowledge base for package ${installation.name}@${installation.version} - install source ${installation.install_source}` + ); + return; + } + if (installation.install_source === 'registry') { + ({ archiveIterator } = await Registry.getPackage(installation.name, installation.version, { + useStreaming: true, + })); + } + await indexKnowledgeBase( + installation.installed_es, + soClient, + esClient, + logger, + { name: installation.name, version: installation.version }, + archiveIterator!, + abortController + ).catch((error) => { + logger.error( + `Failed reindexing knowledge base for package ${installation.name}@${installation.version}: ${error}` + ); + }); + }, + { concurrency: MAX_CONCURRENT_EPM_PACKAGES_INSTALLATIONS } + ); +} diff --git a/x-pack/platform/plugins/shared/fleet/server/types/rest_spec/settings.ts b/x-pack/platform/plugins/shared/fleet/server/types/rest_spec/settings.ts index 57dfe80a7ec26..bdee77673c69a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/types/rest_spec/settings.ts +++ b/x-pack/platform/plugins/shared/fleet/server/types/rest_spec/settings.ts @@ -59,6 +59,7 @@ export const PutSettingsRequestSchema = { is_preconfigured: schema.boolean(), }) ), + integration_knowledge_enabled: schema.maybe(schema.boolean()), }), }; @@ -105,8 +106,12 @@ export const SettingsSchemaV6 = SettingsSchemaV5.extends({ ssl_secret_storage_requirements_met: schema.maybe(schema.boolean()), }); +export const SettingsSchemaV7 = SettingsSchemaV6.extends({ + integration_knowledge_enabled: schema.maybe(schema.boolean()), +}); + export const SettingsResponseSchema = schema.object({ - item: SettingsSchemaV6, + item: SettingsSchemaV7, }); export const PutSpaceSettingsRequestSchema = { diff --git a/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts b/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts index c8659526e820b..e6a15840aa499 100644 --- a/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts +++ b/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts @@ -279,6 +279,7 @@ export interface SettingsSOAttributes { metrics?: 'success' | null; synthetics?: 'success' | null; }; + integration_knowledge_enabled?: boolean; } export interface SpaceSettingsSOAttributes { diff --git a/x-pack/platform/test/api_integration/apis/ml/config.ts b/x-pack/platform/test/api_integration/apis/ml/config.ts index 1667bc4a4cf3c..485c82bd390c0 100644 --- a/x-pack/platform/test/api_integration/apis/ml/config.ts +++ b/x-pack/platform/test/api_integration/apis/ml/config.ts @@ -17,5 +17,14 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { reportName: 'X-Pack API Integration Tests - ml', }, indexRefreshInterval: '1s', + kbnTestServer: { + ...baseIntegrationTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...baseIntegrationTestsConfig.get('kbnTestServer.serverArgs'), + `--xpack.fleet.experimentalFeatures=${JSON.stringify({ + integrationKnowledge: false, + })}`, + ], + }, }; } diff --git a/x-pack/platform/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/platform/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 35296f8928889..8bd660595be00 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -29,6 +29,7 @@ export default function (providerContext: FtrProviderContext) { const supertest = getService('supertest'); const es: Client = getService('es'); const fleetAndAgents = getService('fleetAndAgents'); + const retry = getService('retry'); const pkgName = 'all_assets'; const pkgVersion = '0.1.0'; const logsTemplateName = `logs-${pkgName}.test_logs`; @@ -64,6 +65,7 @@ export default function (providerContext: FtrProviderContext) { pkgName, es, kibanaServer, + retry, }); }); @@ -378,6 +380,7 @@ export default function (providerContext: FtrProviderContext) { pkgName, es, kibanaServer, + retry, }); }); }); @@ -390,6 +393,7 @@ const expectAssetsInstalled = ({ pkgName, es, kibanaServer, + retry, }: { logsTemplateName: string; metricsTemplateName: string; @@ -397,6 +401,7 @@ const expectAssetsInstalled = ({ pkgName: string; es: Client; kibanaServer: any; + retry: any; }) => { it('should have installed the ILM policy', async function () { const resPolicy = await es.transport.request( @@ -596,326 +601,338 @@ const expectAssetsInstalled = ({ expect(metricsAttributes.fields).to.be(undefined); }); it('should have created the correct saved object', async function () { - const res = await kibanaServer.savedObjects.get({ - type: 'epm-packages', - id: 'all_assets', - }); - // during a reinstall the items can change - const sortedRes = { - ...res.attributes, - // verification_key_id can be null or undefined for install or reinstall cases, - // kbn/expect only does strict equality so undefined is normalised to null - verification_key_id: - res.attributes.verification_key_id === undefined - ? null - : res.attributes.verification_key_id, - installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type), - installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type), - package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type), - }; + async function verifySO() { + const res = await kibanaServer.savedObjects.get({ + type: 'epm-packages', + id: 'all_assets', + }); + // during a reinstall the items can change + const sortedRes = { + ...res.attributes, + // verification_key_id can be null or undefined for install or reinstall cases, + // kbn/expect only does strict equality so undefined is normalised to null + verification_key_id: + res.attributes.verification_key_id === undefined + ? null + : res.attributes.verification_key_id, + installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type), + installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type), + package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type), + }; - expect(sortedRes).eql({ - installed_kibana: [ - { - id: 'sample_alerting_rule_template', - type: 'alerting_rule_template', - }, - { - id: 'sample_csp_rule_template', - type: 'csp-rule-template', - }, - { - id: 'sample_dashboard', - type: 'dashboard', - }, - { - id: 'sample_dashboard2', - type: 'dashboard', - }, - { - id: 'test-*', - type: 'index-pattern', - }, - { - id: 'sample_lens', - type: 'lens', - }, - { - id: 'sample_ml_module', - type: 'ml-module', - }, - { - id: 'sample_osquery_pack_asset', - type: 'osquery-pack-asset', - }, - { - id: 'sample_osquery_saved_query', - type: 'osquery-saved-query', - }, - { - id: 'sample_search', - type: 'search', - }, - { - id: 'sample_security_ai_prompt', - type: 'security-ai-prompt', - }, - { - id: 'sample_security_rule', - type: 'security-rule', - }, - { - id: 'sample_tag', - type: 'tag', - }, - { - id: 'sample_visualization', - type: 'visualization', - }, - ], - installed_kibana_space_id: 'default', - installed_es: [ - { - id: 'logs-all_assets.test_logs@package', - type: 'component_template', - }, - { - id: 'logs@custom', - type: 'component_template', - }, - { - id: 'all_assets@custom', - type: 'component_template', - }, - { - id: 'logs-all_assets.test_logs@custom', - type: 'component_template', - }, - { - id: 'metrics-all_assets.test_metrics@package', - type: 'component_template', - }, - { - id: 'metrics@custom', - type: 'component_template', - }, - { - id: 'metrics-all_assets.test_metrics@custom', - type: 'component_template', - }, - { - id: 'logs-all_assets.test_logs-all_assets', - type: 'data_stream_ilm_policy', - }, - { - id: 'metrics-all_assets.test_metrics-all_assets', - type: 'data_stream_ilm_policy', - }, - { - id: 'all_assets', - type: 'ilm_policy', - }, - { - id: 'logs-all_assets.test_logs', - type: 'index_template', - }, - { - id: 'metrics-all_assets.test_metrics', - type: 'index_template', - }, - { - id: 'logs-all_assets.test_logs-0.1.0', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs-0.1.0-pipeline1', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs-0.1.0-pipeline2', - type: 'ingest_pipeline', - }, - { - id: 'metrics-all_assets.test_metrics-0.1.0', - type: 'ingest_pipeline', - }, - { - id: 'default', - type: 'ml_model', - }, - ], - package_assets: [ - { - id: '333a22a1-e639-5af5-ae62-907ffc83d603', - path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ilm_policy/all_assets.json', - type: 'epm-packages-assets', - }, - { - id: '256f3dad-6870-56c3-80a1-8dfa11e2d568', - path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/default.yml', - type: 'epm-packages-assets', - }, - { - id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8', - path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline1.yml', - type: 'epm-packages-assets', - }, - { - id: 'ea334ad8-80c2-5acd-934b-2a377290bf97', - path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline2.yml', - type: 'epm-packages-assets', - }, - { - id: '96c6eb85-fe2e-56c6-84be-5fda976796db', - path: 'all_assets-0.1.0/data_stream/test_logs/fields/ecs.yml', - type: 'epm-packages-assets', - }, - { - id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb', - path: 'all_assets-0.1.0/data_stream/test_logs/fields/fields.yml', - type: 'epm-packages-assets', - }, - { - id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362', - path: 'all_assets-0.1.0/data_stream/test_logs/manifest.yml', - type: 'epm-packages-assets', - }, - { - id: '691f0505-18c5-57a6-9f40-06e8affbdf7a', - path: 'all_assets-0.1.0/data_stream/test_metrics/elasticsearch/ilm_policy/all_assets.json', - type: 'epm-packages-assets', - }, - { - id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274', - path: 'all_assets-0.1.0/data_stream/test_metrics/fields/ecs.yml', - type: 'epm-packages-assets', - }, - { - id: 'f839c76e-d194-555a-90a1-3265a45789e4', - path: 'all_assets-0.1.0/data_stream/test_metrics/fields/fields.yml', - type: 'epm-packages-assets', - }, - { - id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2', - path: 'all_assets-0.1.0/data_stream/test_metrics/manifest.yml', - type: 'epm-packages-assets', - }, - { - id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71', - path: 'all_assets-0.1.0/docs/README.md', - type: 'epm-packages-assets', - }, - { - id: '4510252e-f145-5dd8-ba78-85cc8746c7f7', - path: 'all_assets-0.1.0/elasticsearch/esql_view/test_query.yml', - type: 'epm-packages-assets', - }, - { - id: 'ed5d54d5-2516-5d49-9e61-9508b0152d2b', - path: 'all_assets-0.1.0/elasticsearch/ml_model/test/default.json', - type: 'epm-packages-assets', - }, - { - id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad', - path: 'all_assets-0.1.0/img/logo_overrides_64_color.svg', - type: 'epm-packages-assets', - }, - { - id: '6440faa4-fe21-5620-9989-f0f2b9f6944f', - path: 'all_assets-0.1.0/kibana/alerting_rule_template/sample_alerting_rule_template.json', - type: 'epm-packages-assets', - }, - { - id: '943d5767-41f5-57c3-ba02-48e0f6a837db', - path: 'all_assets-0.1.0/kibana/csp_rule_template/sample_csp_rule_template.json', - type: 'epm-packages-assets', - }, - { - id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487', - path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard.json', - type: 'epm-packages-assets', - }, - { - id: '60d6d054-57e4-590f-a580-52bf3f5e7cca', - path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard2.json', - type: 'epm-packages-assets', - }, - { - id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', - path: 'all_assets-0.1.0/kibana/index_pattern/invalid.json', - type: 'epm-packages-assets', - }, - { - id: '318959c9-997b-5a14-b328-9fc7355b4b74', - path: 'all_assets-0.1.0/kibana/index_pattern/test_index_pattern.json', - type: 'epm-packages-assets', - }, - { - id: 'e21b59b5-eb76-5ab0-bef2-1c8e379e6197', - path: 'all_assets-0.1.0/kibana/lens/sample_lens.json', - type: 'epm-packages-assets', - }, - { - id: '4c758d70-ecf1-56b3-b704-6d8374841b34', - path: 'all_assets-0.1.0/kibana/ml_module/sample_ml_module.json', - type: 'epm-packages-assets', - }, - { - id: '313ddb31-e70a-59e8-8287-310d4652a9b7', - path: 'all_assets-0.1.0/kibana/osquery_pack_asset/sample_osquery_pack_asset.json', - type: 'epm-packages-assets', - }, - { - id: '24a74223-5fdb-52ca-9cb5-b2cdd2a42b07', - path: 'all_assets-0.1.0/kibana/osquery_saved_query/sample_osquery_saved_query.json', - type: 'epm-packages-assets', - }, - { - id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', - path: 'all_assets-0.1.0/kibana/search/sample_search.json', - type: 'epm-packages-assets', - }, - { - id: '5d12ad91-0624-5dce-800d-b1f9a7732f7c', - path: 'all_assets-0.1.0/kibana/security_ai_prompt/sample_security_ai_prompts.json', - type: 'epm-packages-assets', - }, - { - id: 'd8b175c3-0d42-5ec7-90c1-d1e4b307a4c2', - path: 'all_assets-0.1.0/kibana/security_rule/sample_security_rule.json', - type: 'epm-packages-assets', - }, - { - id: 'b265a5e0-c00b-5eda-ac44-2ddbd36d9ad0', - path: 'all_assets-0.1.0/kibana/tag/sample_tag.json', - type: 'epm-packages-assets', - }, - { - id: '53c94591-aa33-591d-8200-cd524c2a0561', - path: 'all_assets-0.1.0/kibana/visualization/sample_visualization.json', - type: 'epm-packages-assets', - }, - { - id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', - path: 'all_assets-0.1.0/manifest.yml', - type: 'epm-packages-assets', - }, - ], - es_index_patterns: { - test_logs: 'logs-all_assets.test_logs-*', - test_metrics: 'metrics-all_assets.test_metrics-*', - }, - name: 'all_assets', - version: '0.1.0', - install_version: '0.1.0', - install_status: 'installed', - install_started_at: res.attributes.install_started_at, - install_source: 'registry', - latest_install_failed_attempts: [], - rolled_back: false, - install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, - verification_status: 'unknown', - verification_key_id: null, + const expectedSavedObject = { + installed_kibana: [ + { + id: 'sample_alerting_rule_template', + type: 'alerting_rule_template', + }, + { + id: 'sample_csp_rule_template', + type: 'csp-rule-template', + }, + { + id: 'sample_dashboard', + type: 'dashboard', + }, + { + id: 'sample_dashboard2', + type: 'dashboard', + }, + { + id: 'test-*', + type: 'index-pattern', + }, + { + id: 'sample_lens', + type: 'lens', + }, + { + id: 'sample_ml_module', + type: 'ml-module', + }, + { + id: 'sample_osquery_pack_asset', + type: 'osquery-pack-asset', + }, + { + id: 'sample_osquery_saved_query', + type: 'osquery-saved-query', + }, + { + id: 'sample_search', + type: 'search', + }, + { + id: 'sample_security_ai_prompt', + type: 'security-ai-prompt', + }, + { + id: 'sample_security_rule', + type: 'security-rule', + }, + { + id: 'sample_tag', + type: 'tag', + }, + { + id: 'sample_visualization', + type: 'visualization', + }, + ], + installed_kibana_space_id: 'default', + installed_es: [ + { + id: 'logs-all_assets.test_logs@package', + type: 'component_template', + }, + { + id: 'logs@custom', + type: 'component_template', + }, + { + id: 'all_assets@custom', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@custom', + type: 'component_template', + }, + { + id: 'metrics-all_assets.test_metrics@package', + type: 'component_template', + }, + { + id: 'metrics@custom', + type: 'component_template', + }, + { + id: 'metrics-all_assets.test_metrics@custom', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs-all_assets', + type: 'data_stream_ilm_policy', + }, + { + id: 'metrics-all_assets.test_metrics-all_assets', + type: 'data_stream_ilm_policy', + }, + { + id: 'all_assets', + type: 'ilm_policy', + }, + { + id: 'logs-all_assets.test_logs', + type: 'index_template', + }, + { + id: 'metrics-all_assets.test_metrics', + type: 'index_template', + }, + { + id: 'logs-all_assets.test_logs-0.1.0', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs-0.1.0-pipeline1', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs-0.1.0-pipeline2', + type: 'ingest_pipeline', + }, + { + id: 'metrics-all_assets.test_metrics-0.1.0', + type: 'ingest_pipeline', + }, + { + id: 'all_assets-README.md', + type: 'knowledge_base', + }, + { + id: 'default', + type: 'ml_model', + }, + ], + package_assets: [ + { + id: '333a22a1-e639-5af5-ae62-907ffc83d603', + path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ilm_policy/all_assets.json', + type: 'epm-packages-assets', + }, + { + id: '256f3dad-6870-56c3-80a1-8dfa11e2d568', + path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/default.yml', + type: 'epm-packages-assets', + }, + { + id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8', + path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline1.yml', + type: 'epm-packages-assets', + }, + { + id: 'ea334ad8-80c2-5acd-934b-2a377290bf97', + path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline2.yml', + type: 'epm-packages-assets', + }, + { + id: '96c6eb85-fe2e-56c6-84be-5fda976796db', + path: 'all_assets-0.1.0/data_stream/test_logs/fields/ecs.yml', + type: 'epm-packages-assets', + }, + { + id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb', + path: 'all_assets-0.1.0/data_stream/test_logs/fields/fields.yml', + type: 'epm-packages-assets', + }, + { + id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362', + path: 'all_assets-0.1.0/data_stream/test_logs/manifest.yml', + type: 'epm-packages-assets', + }, + { + id: '691f0505-18c5-57a6-9f40-06e8affbdf7a', + path: 'all_assets-0.1.0/data_stream/test_metrics/elasticsearch/ilm_policy/all_assets.json', + type: 'epm-packages-assets', + }, + { + id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274', + path: 'all_assets-0.1.0/data_stream/test_metrics/fields/ecs.yml', + type: 'epm-packages-assets', + }, + { + id: 'f839c76e-d194-555a-90a1-3265a45789e4', + path: 'all_assets-0.1.0/data_stream/test_metrics/fields/fields.yml', + type: 'epm-packages-assets', + }, + { + id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2', + path: 'all_assets-0.1.0/data_stream/test_metrics/manifest.yml', + type: 'epm-packages-assets', + }, + { + id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71', + path: 'all_assets-0.1.0/docs/README.md', + type: 'epm-packages-assets', + }, + { + id: '4510252e-f145-5dd8-ba78-85cc8746c7f7', + path: 'all_assets-0.1.0/elasticsearch/esql_view/test_query.yml', + type: 'epm-packages-assets', + }, + { + id: 'ed5d54d5-2516-5d49-9e61-9508b0152d2b', + path: 'all_assets-0.1.0/elasticsearch/ml_model/test/default.json', + type: 'epm-packages-assets', + }, + { + id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad', + path: 'all_assets-0.1.0/img/logo_overrides_64_color.svg', + type: 'epm-packages-assets', + }, + { + id: '6440faa4-fe21-5620-9989-f0f2b9f6944f', + path: 'all_assets-0.1.0/kibana/alerting_rule_template/sample_alerting_rule_template.json', + type: 'epm-packages-assets', + }, + { + id: '943d5767-41f5-57c3-ba02-48e0f6a837db', + path: 'all_assets-0.1.0/kibana/csp_rule_template/sample_csp_rule_template.json', + type: 'epm-packages-assets', + }, + { + id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487', + path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard.json', + type: 'epm-packages-assets', + }, + { + id: '60d6d054-57e4-590f-a580-52bf3f5e7cca', + path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard2.json', + type: 'epm-packages-assets', + }, + { + id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', + path: 'all_assets-0.1.0/kibana/index_pattern/invalid.json', + type: 'epm-packages-assets', + }, + { + id: '318959c9-997b-5a14-b328-9fc7355b4b74', + path: 'all_assets-0.1.0/kibana/index_pattern/test_index_pattern.json', + type: 'epm-packages-assets', + }, + { + id: 'e21b59b5-eb76-5ab0-bef2-1c8e379e6197', + path: 'all_assets-0.1.0/kibana/lens/sample_lens.json', + type: 'epm-packages-assets', + }, + { + id: '4c758d70-ecf1-56b3-b704-6d8374841b34', + path: 'all_assets-0.1.0/kibana/ml_module/sample_ml_module.json', + type: 'epm-packages-assets', + }, + { + id: '313ddb31-e70a-59e8-8287-310d4652a9b7', + path: 'all_assets-0.1.0/kibana/osquery_pack_asset/sample_osquery_pack_asset.json', + type: 'epm-packages-assets', + }, + { + id: '24a74223-5fdb-52ca-9cb5-b2cdd2a42b07', + path: 'all_assets-0.1.0/kibana/osquery_saved_query/sample_osquery_saved_query.json', + type: 'epm-packages-assets', + }, + { + id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', + path: 'all_assets-0.1.0/kibana/search/sample_search.json', + type: 'epm-packages-assets', + }, + { + id: '5d12ad91-0624-5dce-800d-b1f9a7732f7c', + path: 'all_assets-0.1.0/kibana/security_ai_prompt/sample_security_ai_prompts.json', + type: 'epm-packages-assets', + }, + { + id: 'd8b175c3-0d42-5ec7-90c1-d1e4b307a4c2', + path: 'all_assets-0.1.0/kibana/security_rule/sample_security_rule.json', + type: 'epm-packages-assets', + }, + { + id: 'b265a5e0-c00b-5eda-ac44-2ddbd36d9ad0', + path: 'all_assets-0.1.0/kibana/tag/sample_tag.json', + type: 'epm-packages-assets', + }, + { + id: '53c94591-aa33-591d-8200-cd524c2a0561', + path: 'all_assets-0.1.0/kibana/visualization/sample_visualization.json', + type: 'epm-packages-assets', + }, + { + id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', + path: 'all_assets-0.1.0/manifest.yml', + type: 'epm-packages-assets', + }, + ], + es_index_patterns: { + test_logs: 'logs-all_assets.test_logs-*', + test_metrics: 'metrics-all_assets.test_metrics-*', + }, + name: 'all_assets', + version: '0.1.0', + install_version: '0.1.0', + install_status: 'installed', + install_started_at: res.attributes.install_started_at, + install_source: 'registry', + latest_install_failed_attempts: [], + rolled_back: false, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + verification_status: 'unknown', + verification_key_id: null, + }; + + expect(sortedRes).eql(expectedSavedObject); + } + + await retry.tryForTime(10000, async () => { + await verifySO(); }); }); diff --git a/x-pack/platform/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/platform/test/fleet_api_integration/apis/epm/update_assets.ts index 8aca368e73f7e..c9781beb11ccb 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/epm/update_assets.ts @@ -341,6 +341,7 @@ export default function (providerContext: FtrProviderContext) { ...res.attributes, installed_kibana: sortBy(res.attributes.installed_kibana, ['id']), package_assets: sortBy(res.attributes.package_assets, ['id']), + installed_es: sortBy(res.attributes.installed_es, ['id']), }).eql({ installed_kibana_space_id: 'default', installed_kibana: sortBy( @@ -396,85 +397,91 @@ export default function (providerContext: FtrProviderContext) { ], 'id' ), - installed_es: [ - { - id: 'all_assets', - type: 'ilm_policy', - }, - { - id: 'default', - type: 'ml_model', - }, - { - id: 'logs-all_assets.test_logs-all_assets', - type: 'data_stream_ilm_policy', - }, - { - id: 'logs-all_assets.test_logs-0.2.0', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs-0.2.0-pipeline1', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs2-0.2.0', - type: 'ingest_pipeline', - }, - { - id: 'metrics-all_assets.test_metrics-0.2.0', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs', - type: 'index_template', - }, - { - id: 'logs-all_assets.test_logs@package', - type: 'component_template', - }, - { - id: 'logs@custom', - type: 'component_template', - }, - { - id: 'all_assets@custom', - type: 'component_template', - }, - { - id: 'logs-all_assets.test_logs@custom', - type: 'component_template', - }, - { - id: 'logs-all_assets.test_logs2', - type: 'index_template', - }, - { - id: 'logs-all_assets.test_logs2@package', - type: 'component_template', - }, - { - id: 'logs-all_assets.test_logs2@custom', - type: 'component_template', - }, - { - id: 'metrics-all_assets.test_metrics', - type: 'index_template', - }, - { - id: 'metrics-all_assets.test_metrics@package', - type: 'component_template', - }, - - { - id: 'metrics@custom', - type: 'component_template', - }, - { - id: 'metrics-all_assets.test_metrics@custom', - type: 'component_template', - }, - ], + installed_es: sortBy( + [ + { + id: 'all_assets', + type: 'ilm_policy', + }, + { + id: 'default', + type: 'ml_model', + }, + { + id: 'logs-all_assets.test_logs-all_assets', + type: 'data_stream_ilm_policy', + }, + { + id: 'logs-all_assets.test_logs-0.2.0', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs-0.2.0-pipeline1', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs2-0.2.0', + type: 'ingest_pipeline', + }, + { + id: 'metrics-all_assets.test_metrics-0.2.0', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs', + type: 'index_template', + }, + { + id: 'logs-all_assets.test_logs@package', + type: 'component_template', + }, + { + id: 'logs@custom', + type: 'component_template', + }, + { + id: 'all_assets@custom', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@custom', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs2', + type: 'index_template', + }, + { + id: 'logs-all_assets.test_logs2@package', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs2@custom', + type: 'component_template', + }, + { + id: 'metrics-all_assets.test_metrics', + type: 'index_template', + }, + { + id: 'metrics-all_assets.test_metrics@package', + type: 'component_template', + }, + { + id: 'metrics@custom', + type: 'component_template', + }, + { + id: 'metrics-all_assets.test_metrics@custom', + type: 'component_template', + }, + { + id: 'all_assets-README.md', + type: 'knowledge_base', + }, + ], + 'id' + ), es_index_patterns: { test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*', diff --git a/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts b/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts index 17ff4acab1995..18ce8a412d6c6 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts @@ -192,7 +192,12 @@ export default function (providerContext: FtrProviderContext) { it('should not have created any ES assets on install', async () => { const installation = await getInstallationInfo(supertest, PACKAGE_NAME, START_VERSION); - expect(installation.installed_es).to.eql([]); + expect(installation.installed_es).to.eql([ + { + id: 'input_package_upgrade-README.md', + type: 'knowledge_base', + }, + ]); }); it('should create index templates and update installed_es on package policy creation', async () => { @@ -204,6 +209,7 @@ export default function (providerContext: FtrProviderContext) { { id: 'logs-dataset1@package', type: 'component_template' }, { id: 'logs-dataset1@custom', type: 'component_template' }, { id: 'logs@custom', type: 'component_template' }, + { id: 'input_package_upgrade-README.md', type: 'knowledge_base' }, { id: 'input_package_upgrade@custom', type: 'component_template' }, ]); diff --git a/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_rollback.ts b/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_rollback.ts index 6195cd6f3d1e2..03f9192cc4d97 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_rollback.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/package_policy/input_package_rollback.ts @@ -127,8 +127,15 @@ export default function (providerContext: FtrProviderContext) { await installPackage(PACKAGE_NAME, START_VERSION); await createPackagePolicyWithDataset(agentPolicyId, 'test*', 400); + await new Promise((resolve) => setTimeout(resolve, 1000)); + const installation = await getInstallationInfo(supertest, PACKAGE_NAME, START_VERSION); - expectIdArraysEqual(installation.installed_es, []); + expectIdArraysEqual(installation.installed_es, [ + { + id: 'input_package_upgrade-README.md', + type: 'knowledge_base', + }, + ]); await uninstallPackage(PACKAGE_NAME, START_VERSION); }); diff --git a/x-pack/platform/test/fleet_api_integration/apis/package_policy/upgrade.ts b/x-pack/platform/test/fleet_api_integration/apis/package_policy/upgrade.ts index cdfe4f6beab22..d512dfa39686d 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/package_policy/upgrade.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/package_policy/upgrade.ts @@ -1249,6 +1249,7 @@ export default function (providerContext: FtrProviderContext) { } expectedAssets.push({ id: 'logs@custom', type: 'component_template' }); + expectedAssets.push({ id: 'integration_to_input-README.md', type: 'knowledge_base' }); expectedAssets.push({ id: 'integration_to_input@custom', type: 'component_template' }); }); diff --git a/x-pack/platform/test/fleet_api_integration/apis/settings/global_settings.ts b/x-pack/platform/test/fleet_api_integration/apis/settings/global_settings.ts index 56b26595f5060..421e8fe59373a 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/settings/global_settings.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/settings/global_settings.ts @@ -50,5 +50,28 @@ export default function (providerContext: FtrProviderContext) { const updatedAgentPolicy = await apiClient.getAgentPolicy(agentPolicy.item.id); expect(updatedAgentPolicy.item.revision).to.be(agentPolicy.item.revision); }); + + it('should reindex knowledge base when setting is enabled', async function () { + await apiClient.installPackage({ pkgName: 'knowledge_base_test', pkgVersion: '1.0.0' }); + + await apiClient.putSettings({ + integration_knowledge_enabled: true, + }); + + const updatedSettings = await apiClient.getSettings(); + expect(updatedSettings.item.integration_knowledge_enabled).to.be(true); + + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const response = await apiClient.getPackage({ + pkgName: 'knowledge_base_test', + pkgVersion: '1.0.0', + }); + expect( + response.item.installationInfo?.installed_es.some( + (esAsset) => esAsset.type === 'knowledge_base' + ) + ).to.be(true); + }); }); } diff --git a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 7137e8360955d..6ca5cd5ac0337 100644 --- a/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/platform/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -179,6 +179,7 @@ export default function ({ getService }: FtrProviderContext) { 'fleet:policy-revisions-cleanup-task', 'fleet:privilege_level_change:retry', 'fleet:reassign_action:retry', + 'fleet:reindex_integration_knowledge', 'fleet:request_diagnostics:retry', 'fleet:setup', 'fleet:setup:upgrade_managed_package_policies', diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/retrieve_elastic_doc.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/retrieve_elastic_doc.spec.ts index 772ced501ca2e..06a4c4040a045 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/retrieve_elastic_doc.spec.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/retrieve_elastic_doc.spec.ts @@ -140,8 +140,8 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon expect(llmProxy.interceptedRequests.length).to.be(3); }); - it('emits 5 messageAdded events', () => { - expect(messageAddedEvents.length).to.be(5); + it('emits at least 3 messageAdded events', () => { + expect(messageAddedEvents.length).to.be.greaterThan(2); }); describe('The first request', () => { diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/utils/create_llm_proxy.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/utils/create_llm_proxy.ts index 55eebbbe53a82..dad8bfb4701ab 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/utils/create_llm_proxy.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/utils/create_llm_proxy.ts @@ -240,10 +240,16 @@ export class LlmProxy { interceptScoreToolChoice(log: ToolingLog) { function extractDocumentsToScore(source: string): KnowledgeBaseDocument[] { const [, raw] = source.match(/\s*(\[[\s\S]*?\])\s*<\/DocumentsToScore>/i)!; - const jsonString = raw.trim().replace(/\\"/g, '"'); - const documentsToScore = JSON.parse(jsonString); - log.debug(`Extracted documents to score: ${JSON.stringify(documentsToScore, null, 2)}`); - return documentsToScore; + const jsonString = raw.trim().replace(/\\"/g, '"').replace(/\n/g, ' '); + try { + const documentsToScore = JSON.parse(jsonString); + log.debug(`Extracted documents to score: ${JSON.stringify(documentsToScore, null, 2)}`); + return documentsToScore; + } catch (error) { + log.error(`Failed to extract documents to score: ${error}`); + log.debug(`Raw string: ${jsonString}`); + } + return []; } let documentsToScore: KnowledgeBaseDocument[] = []; diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts index 15758b95ca69d..6f0103b884b72 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts @@ -24,6 +24,9 @@ export default createStatefulTestConfig({ appenders: ['default'], }, ])}`, + `--xpack.fleet.experimentalFeatures=${JSON.stringify({ + integrationKnowledge: false, + })}`, ], }, }); diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.index.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.index.ts similarity index 100% rename from x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.index.ts rename to x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.index.ts diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.serverless.config.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.serverless.config.ts similarity index 57% rename from x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.serverless.config.ts rename to x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.serverless.config.ts index 900bcb03af291..8c6aa790f860f 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/configs/serverless/oblt.ai_assistant.serverless.config.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/feature_flag_configs/serverless/oblt.ai_assistant.serverless.config.ts @@ -5,15 +5,19 @@ * 2.0. */ -import { createServerlessTestConfig } from '@kbn/test-suites-xpack-platform/api_integration_deployment_agnostic/default_configs/serverless.config.base'; +import { createServerlessFeatureFlagTestConfig } from '@kbn/test-suites-xpack-platform/api_integration_deployment_agnostic/default_configs/feature_flag.serverless.config.base'; import { services } from '../../services'; -export default createServerlessTestConfig({ +export default createServerlessFeatureFlagTestConfig({ services, serverlessProject: 'oblt', testFiles: [require.resolve('./oblt.ai_assistant.index.ts')], junit: { reportName: 'Serverless Observability - Deployment-agnostic API Integration Tests', }, - indexRefreshInterval: '1s', + kbnServerArgs: [ + `--xpack.fleet.experimentalFeatures=${JSON.stringify({ + integrationKnowledge: false, + })}`, + ], }); diff --git a/x-pack/solutions/security/test/serverless/functional/configs/config.ts b/x-pack/solutions/security/test/serverless/functional/configs/config.ts index df20849abad83..cfe2629aee188 100644 --- a/x-pack/solutions/security/test/serverless/functional/configs/config.ts +++ b/x-pack/solutions/security/test/serverless/functional/configs/config.ts @@ -30,5 +30,8 @@ export default createTestConfig({ '--xpack.dataUsage.autoops.api.url=http://localhost:9000', `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`, `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`, + `--xpack.fleet.experimentalFeatures=${JSON.stringify({ + integrationKnowledge: false, + })}`, ], });