diff --git a/x-pack/platform/plugins/shared/fleet/server/mocks/index.ts b/x-pack/platform/plugins/shared/fleet/server/mocks/index.ts index db100c5fcf7ec..8aa3d96ae43b8 100644 --- a/x-pack/platform/plugins/shared/fleet/server/mocks/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/mocks/index.ts @@ -228,8 +228,10 @@ export const createPackagePolicyServiceMock = (): jest.Mocked => { return { + create: jest.fn().mockReturnValue(Promise.resolve()), get: jest.fn().mockReturnValue(Promise.resolve()), list: jest.fn().mockReturnValue(Promise.resolve()), + delete: jest.fn().mockReturnValue(Promise.resolve()), getFullAgentPolicy: jest.fn().mockReturnValue(Promise.resolve()), getByIds: jest.fn().mockReturnValue(Promise.resolve()), turnOffAgentTamperProtections: jest.fn().mockReturnValue(Promise.resolve()), diff --git a/x-pack/platform/plugins/shared/fleet/server/plugin.ts b/x-pack/platform/plugins/shared/fleet/server/plugin.ts index 1620df27b82c3..d1e4691fb48b5 100644 --- a/x-pack/platform/plugins/shared/fleet/server/plugin.ts +++ b/x-pack/platform/plugins/shared/fleet/server/plugin.ts @@ -816,16 +816,7 @@ export class FleetPlugin core.elasticsearch.client.asInternalUser, internalSoClient ), - agentPolicyService: { - get: agentPolicyService.get, - list: agentPolicyService.list, - getFullAgentPolicy: agentPolicyService.getFullAgentPolicy, - getByIds: agentPolicyService.getByIDs, - turnOffAgentTamperProtections: - agentPolicyService.turnOffAgentTamperProtections.bind(agentPolicyService), - fetchAllAgentPolicies: agentPolicyService.fetchAllAgentPolicies, - fetchAllAgentPolicyIds: agentPolicyService.fetchAllAgentPolicyIds, - }, + agentPolicyService, packagePolicyService, registerExternalCallback: (type: ExternalCallback[0], callback: ExternalCallback[1]) => { return appContextService.addExternalCallback(type, callback); diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts index 6d5587e283878..2233d9fc9fa3c 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts @@ -196,7 +196,7 @@ export const bulkGetAgentPoliciesHandler: FleetRequestHandler< 'full query parameter require agent policies read permissions' ); } - let items = await agentPolicyService.getByIDs(soClient, ids, { + let items = await agentPolicyService.getByIds(soClient, ids, { withPackagePolicies, ignoreMissing, }); @@ -687,7 +687,7 @@ export const GetListAgentPolicyOutputsHandler: FleetRequestHandler< body: { items: [] }, }); } - const agentPolicies = await agentPolicyService.getByIDs(soClient, ids, { + const agentPolicies = await agentPolicyService.getByIds(soClient, ids, { withPackagePolicies: true, }); diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/settings/enrollment_settings_handler.test.ts b/x-pack/platform/plugins/shared/fleet/server/routes/settings/enrollment_settings_handler.test.ts index 029efc146e09b..d8d2bdc2f2411 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/settings/enrollment_settings_handler.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/settings/enrollment_settings_handler.test.ts @@ -22,7 +22,7 @@ import { jest.mock('../../services', () => ({ agentPolicyService: { get: jest.fn(), - getByIDs: jest.fn(), + getByIds: jest.fn(), }, appContextService: { getInternalUserSOClientWithoutSpaceExtension: jest.fn(), diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.test.ts b/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.test.ts index 151a0a2ba2af8..2d263a4b66686 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.test.ts @@ -42,7 +42,7 @@ jest.mock('../../services', () => ({ }, agentPolicyService: { get: jest.fn(), - getByIDs: jest.fn(), + getByIds: jest.fn(), }, })); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts index 76d2727b65e85..68fde28107340 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts @@ -504,7 +504,7 @@ describe('Agent policy', () => { }); }); - describe('getByIDs', () => { + describe('getByIds', () => { it('should call audit logger', async () => { const soClient = savedObjectsClientMock.create(); @@ -525,7 +525,7 @@ describe('Agent policy', () => { ], }); - await agentPolicyService.getByIDs(soClient, ['test-agent-policy-1', 'test-agent-policy-2']); + await agentPolicyService.getByIds(soClient, ['test-agent-policy-1', 'test-agent-policy-2']); expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { action: 'get', diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts index 3a3273383ad38..5cf805008d967 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts @@ -507,7 +507,7 @@ class AgentPolicyService { return agentPolicy; } - public async getByIDs( + public async getByIds( soClient: SavedObjectsClientContract, ids: Array, options: { fields?: string[]; withPackagePolicies?: boolean; ignoreMissing?: boolean } = {} @@ -1345,7 +1345,7 @@ class AgentPolicyService { }); } - const policies = await agentPolicyService.getByIDs(soClient, agentPolicyIds); + const policies = await agentPolicyService.getByIds(soClient, agentPolicyIds); const policiesMap = keyBy(policies, 'id'); const fullPolicies = await pMap( agentPolicyIds, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.test.ts index 997bb0817351c..f92d2b00656d5 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.test.ts @@ -14,7 +14,7 @@ import { getHostedPolicies, isHostedAgent } from './hosted_agent'; jest.mock('../agent_policy', () => { return { agentPolicyService: { - getByIDs: jest.fn().mockResolvedValue([ + getByIds: jest.fn().mockResolvedValue([ { id: 'hosted-policy', is_managed: true }, { id: 'regular-policy', is_managed: false }, ]), diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.ts b/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.ts index 4acba8551e86d..06c85805d88dd 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.ts @@ -20,7 +20,7 @@ export async function getHostedPolicies( ); // get the agent policies for those ids - const agentPolicies = await agentPolicyService.getByIDs(soClient, Array.from(policyIdsToGet), { + const agentPolicies = await agentPolicyService.getByIds(soClient, Array.from(policyIdsToGet), { fields: ['is_managed'], ignoreMissing: true, }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agents/update_agent_tags.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/agents/update_agent_tags.test.ts index 6a2410dfa3c16..f1aa1a0944748 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agents/update_agent_tags.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agents/update_agent_tags.test.ts @@ -34,7 +34,7 @@ jest.mock('../agent_policy', () => { return { agentPolicyService: { getInactivityTimeouts: jest.fn().mockResolvedValue([]), - getByIDs: jest.fn().mockResolvedValue([{ id: 'hosted-agent-policy', is_managed: true }]), + getByIds: jest.fn().mockResolvedValue([{ id: 'hosted-agent-policy', is_managed: true }]), list: jest.fn().mockResolvedValue({ items: [] }), }, }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.test.ts index 5b9155a756645..1832ba78e8dd7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.test.ts @@ -204,7 +204,7 @@ describe('getFleetServerPolicies', () => { page: 1, perPage: mockPackagePolicies.length, }); - (mockedAgentPolicyService.getByIDs as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies); + (mockedAgentPolicyService.getByIds as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies); const result = await getFleetServerPolicies(soClient); expect(result).toEqual(mockFleetServerPolicies); }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.ts index e596709523351..034db5fdb0c1d 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.ts @@ -41,7 +41,7 @@ export const getFleetServerPolicies = async ( // Retrieve associated agent policies const fleetServerAgentPolicies = fleetServerAgentPolicyIds.length - ? await agentPolicyService.getByIDs( + ? await agentPolicyService.getByIds( soClient, uniqBy(fleetServerAgentPolicyIds, (p) => p.id) ) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/index.ts index deb902914a190..9db62cc61c2ed 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/index.ts @@ -15,10 +15,12 @@ export { getRegistryUrl } from './epm/registry/registry_url'; */ export interface AgentPolicyServiceInterface { + create: (typeof agentPolicyService)['create']; get: (typeof agentPolicyService)['get']; list: (typeof agentPolicyService)['list']; + delete: (typeof agentPolicyService)['delete']; getFullAgentPolicy: (typeof agentPolicyService)['getFullAgentPolicy']; - getByIds: (typeof agentPolicyService)['getByIDs']; + getByIds: (typeof agentPolicyService)['getByIds']; turnOffAgentTamperProtections: (typeof agentPolicyService)['turnOffAgentTamperProtections']; fetchAllAgentPolicyIds: (typeof agentPolicyService)['fetchAllAgentPolicyIds']; fetchAllAgentPolicies: (typeof agentPolicyService)['fetchAllAgentPolicies']; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/output.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/output.test.ts index f2dab8401e641..6b7da2744ffc5 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/output.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/output.test.ts @@ -316,7 +316,7 @@ describe('Output Service', () => { } as unknown as ReturnType; beforeEach(() => { - mockedAgentPolicyService.getByIDs.mockResolvedValue([]); + mockedAgentPolicyService.getByIds.mockResolvedValue([]); mockedAgentPolicyService.list.mockClear(); mockedPackagePolicyService.list.mockReset(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); @@ -334,7 +334,7 @@ describe('Output Service', () => { }); afterEach(() => { - mockedAgentPolicyService.getByIDs.mockClear(); + mockedAgentPolicyService.getByIds.mockClear(); }); describe('create', () => { @@ -688,7 +688,7 @@ describe('Output Service', () => { mockedPackagePolicyService.list.mockResolvedValue( mockedPackagePolicyWithFleetServerResolvedValue ); - mockedAgentPolicyService.getByIDs.mockResolvedValue( + mockedAgentPolicyService.getByIds.mockResolvedValue( (await mockedAgentPolicyWithFleetServerResolvedValue).items ); @@ -727,7 +727,7 @@ describe('Output Service', () => { mockedPackagePolicyService.list.mockResolvedValue( mockedPackagePolicyWithSyntheticsResolvedValue ); - mockedAgentPolicyService.getByIDs.mockResolvedValue( + mockedAgentPolicyService.getByIds.mockResolvedValue( (await mockedAgentPolicyWithSyntheticsResolvedValue).items ); @@ -845,7 +845,7 @@ describe('Output Service', () => { mockedPackagePolicyService.list.mockResolvedValue( mockedPackagePolicyWithFleetServerResolvedValue ); - mockedAgentPolicyService.getByIDs.mockResolvedValue( + mockedAgentPolicyService.getByIds.mockResolvedValue( (await mockedAgentPolicyWithFleetServerResolvedValue).items ); @@ -884,7 +884,7 @@ describe('Output Service', () => { mockedPackagePolicyService.list.mockResolvedValue( mockedPackagePolicyWithSyntheticsResolvedValue ); - mockedAgentPolicyService.getByIDs.mockResolvedValue( + mockedAgentPolicyService.getByIds.mockResolvedValue( (await mockedAgentPolicyWithSyntheticsResolvedValue).items ); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/output.ts b/x-pack/platform/plugins/shared/fleet/server/services/output.ts index 0b30890ee566d..f7f211862a896 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/output.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/output.ts @@ -176,7 +176,7 @@ async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean) }, []) ), ]; - const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( + const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIds( internalSoClientWithoutSpaceExtension, agentPolicyIdsFromPackagePolicies ); @@ -245,7 +245,7 @@ async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefa ); const agentPolicyIds = _.uniq(packagePolicies.flatMap((p) => p.policy_ids)); if (agentPolicyIds.length) { - agentPolicies = await agentPolicyService.getByIDs( + agentPolicies = await agentPolicyService.getByIds( internalSoClientWithoutSpaceExtension, agentPolicyIds ); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts index 7ea6ae290708b..29d9142d13ed8 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts @@ -227,7 +227,7 @@ const mockAgentPolicyGet = (spaceIds: string[] = ['default']) => { }); } ); - mockAgentPolicyService.getByIDs.mockImplementation( + mockAgentPolicyService.getByIds.mockImplementation( // @ts-ignore (_soClient: SavedObjectsClientContract, ids: string[]) => { return Promise.resolve( diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts index 3ff369994c5c7..7caba78ab862a 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts @@ -518,7 +518,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids)); - const agentPolicies = await agentPolicyService.getByIDs(soClient, [...agentPolicyIds]); + const agentPolicies = await agentPolicyService.getByIds(soClient, [...agentPolicyIds]); const agentPoliciesIndexById = indexBy('id', agentPolicies); for (const agentPolicy of agentPolicies) { validateIsNotHostedPolicy(agentPolicy, options?.force); @@ -1551,7 +1551,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { return acc; }, new Set()); - const agentPolicies = await agentPolicyService.getByIDs(soClient, uniquePolicyIdsR); + const agentPolicies = await agentPolicyService.getByIds(soClient, uniquePolicyIdsR); for (const policyId of uniquePolicyIdsR) { const agentPolicy = agentPolicies.find((p) => p.id === policyId); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.test.ts index d2daf3ff46ae5..e3f6f3369bfc0 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.test.ts @@ -217,7 +217,7 @@ describe('UninstallTokenService', () => { agentPolicyService.deployPolicies = jest.fn(); getAgentPoliciesByIDsMock = jest.fn().mockResolvedValue([]); - agentPolicyService.getByIDs = getAgentPoliciesByIDsMock; + agentPolicyService.getByIds = getAgentPoliciesByIDsMock; if (scoppedInSpace) { soClientMock.getCurrentNamespace.mockReturnValue(scoppedInSpace); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.ts b/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.ts index 275dcfc3a281d..4cbb0874a39a5 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.ts @@ -305,7 +305,7 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { } private async getPolicyIdNameDictionary(policyIds: string[]): Promise> { - const agentPolicies = await agentPolicyService.getByIDs(this.soClient, policyIds, { + const agentPolicies = await agentPolicyService.getByIds(this.soClient, policyIds, { ignoreMissing: true, }); @@ -615,7 +615,7 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { const batchSize = config?.setup?.agentPolicySchemaUpgradeBatchSize ?? 100; await asyncForEach(chunk(policyIds, batchSize), async (policyIdsBatch) => { - const policies = await agentPolicyService.getByIDs( + const policies = await agentPolicyService.getByIds( appContextService.getInternalUserSOClientWithoutSpaceExtension(), policyIds.map((id) => ({ id, spaceId: '*' })) ); diff --git a/x-pack/solutions/search/plugins/search_connectors/kibana.jsonc b/x-pack/solutions/search/plugins/search_connectors/kibana.jsonc index 45ae79ec2cb5c..fa0ecadfab86a 100644 --- a/x-pack/solutions/search/plugins/search_connectors/kibana.jsonc +++ b/x-pack/solutions/search/plugins/search_connectors/kibana.jsonc @@ -17,8 +17,11 @@ "search", "connectors" ], - "requiredPlugins": [], - "optionalPlugins": [], - "requiredBundles": [] + "requiredPlugins": [ + "licensing", + "taskManager", + "fleet" + ], + "optionalPlugins": [] } } diff --git a/x-pack/solutions/search/plugins/search_connectors/server/plugin.ts b/x-pack/solutions/search/plugins/search_connectors/server/plugin.ts index fe73afae20b9a..09ccbd31fe660 100644 --- a/x-pack/solutions/search/plugins/search_connectors/server/plugin.ts +++ b/x-pack/solutions/search/plugins/search_connectors/server/plugin.ts @@ -5,40 +5,94 @@ * 2.0. */ -import type { PluginInitializerContext, Plugin, CoreSetup } from '@kbn/core/server'; +import type { + PluginInitializerContext, + Plugin, + CoreStart, + CoreSetup, + Logger, +} from '@kbn/core/server'; import { ConnectorServerSideDefinition } from '@kbn/search-connectors'; +import { isAgentlessEnabled } from '@kbn/fleet-plugin/server/services/utils/agentless'; import { getConnectorTypes } from '../common/lib/connector_types'; import type { SearchConnectorsPluginSetup as SearchConnectorsPluginSetup, SearchConnectorsPluginStart as SearchConnectorsPluginStart, - SetupDependencies, - StartDependencies, + SearchConnectorsPluginSetupDependencies, + SearchConnectorsPluginStartDependencies, } from './types'; +import { AgentlessConnectorDeploymentsSyncService } from './task'; +import { SearchConnectorsConfig } from './config'; + export class SearchConnectorsPlugin implements Plugin< SearchConnectorsPluginSetup, SearchConnectorsPluginStart, - SetupDependencies, - StartDependencies + SearchConnectorsPluginSetupDependencies, + SearchConnectorsPluginStartDependencies > { private connectors: ConnectorServerSideDefinition[]; + private log: Logger; + private readonly config: SearchConnectorsConfig; + private agentlessConnectorDeploymentsSyncService: AgentlessConnectorDeploymentsSyncService; constructor(initializerContext: PluginInitializerContext) { this.connectors = []; + this.log = initializerContext.logger.get(); + this.config = initializerContext.config.get(); + this.agentlessConnectorDeploymentsSyncService = new AgentlessConnectorDeploymentsSyncService( + this.log + ); } - public setup({ getStartServices, http }: CoreSetup) { + public setup( + coreSetup: CoreSetup, + plugins: SearchConnectorsPluginSetupDependencies + ) { + const http = coreSetup.http; + this.connectors = getConnectorTypes(http.staticAssets); + const coreStartServices = coreSetup.getStartServices(); + + // There seems to be no way to check for agentless here + // So we register a task, but do not execute it in `start` method + this.log.debug('Registering agentless connectors infra sync task'); + + coreStartServices + .then(([coreStart, searchConnectorsPluginStartDependencies]) => { + this.agentlessConnectorDeploymentsSyncService.registerInfraSyncTask( + plugins, + coreStart, + searchConnectorsPluginStartDependencies + ); + }) + .catch((err) => { + this.log.error(`Error registering agentless connectors infra sync task`, err); + }); return { getConnectorTypes: () => this.connectors, }; } - public start() { + public start(coreStart: CoreStart, plugins: SearchConnectorsPluginStartDependencies) { + if (isAgentlessEnabled()) { + this.log.info( + 'Agentless is supported, scheduling initial agentless connectors infrastructure watcher task' + ); + this.agentlessConnectorDeploymentsSyncService + .scheduleInfraSyncTask(this.config, plugins.taskManager) + .catch((err) => { + this.log.error(`Error scheduling agentless connectors infra sync task`, err); + }); + } else { + this.log.info( + 'Agentless is not supported, skipping scheduling initial agentless connectors infrastructure watcher task' + ); + } return { getConnectors: () => this.connectors, }; diff --git a/x-pack/solutions/search/plugins/search_connectors/server/services/index.test.ts b/x-pack/solutions/search/plugins/search_connectors/server/services/index.test.ts new file mode 100644 index 0000000000000..e424a4db038b8 --- /dev/null +++ b/x-pack/solutions/search/plugins/search_connectors/server/services/index.test.ts @@ -0,0 +1,631 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { + ElasticsearchClientMock, + elasticsearchClientMock, +} from '@kbn/core-elasticsearch-client-server-mocks'; +import { + AgentlessConnectorsInfraService, + ConnectorMetadata, + PackagePolicyMetadata, + getConnectorsWithoutPolicies, + getPoliciesWithoutConnectors, +} from '.'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { MockedLogger, loggerMock } from '@kbn/logging-mocks'; +import { + createPackagePolicyServiceMock, + createMockAgentPolicyService, +} from '@kbn/fleet-plugin/server/mocks'; +import { AgentPolicyServiceInterface, PackagePolicyClient } from '@kbn/fleet-plugin/server'; +import { AgentPolicy, PackagePolicy, PackagePolicyInput } from '@kbn/fleet-plugin/common'; +import { createAgentPolicyMock, createPackagePolicyMock } from '@kbn/fleet-plugin/common/mocks'; + +jest.mock('@kbn/fleet-plugin/server/services/epm/packages', () => { + const mockedGetPackageInfo = ({ pkgName }: { pkgName: string }) => { + if (pkgName === 'elastic_connectors') { + const pkg = { + version: '0.0.5', + policy_templates: [ + { + name: 'github_elastic_connectors', + inputs: [ + { + type: 'connectors-py', + vars: [ + { + name: 'connector_id', + required: false, + type: 'string', + }, + { + name: 'connector_name', + required: false, + type: 'string', + }, + { + name: 'service_type', + required: false, + type: 'string', + }, + ], + }, + ], + }, + ], + }; + + return Promise.resolve(pkg); + } + }; + return { + getPackageInfo: jest.fn().mockImplementation(mockedGetPackageInfo), + }; +}); + +describe('AgentlessConnectorsInfraService', () => { + let soClient: jest.Mocked; + let esClient: ElasticsearchClientMock; + let packagePolicyService: jest.Mocked; + let agentPolicyInterface: jest.Mocked; + let logger: MockedLogger; + let service: AgentlessConnectorsInfraService; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchClientMock.createClusterClient().asInternalUser; + packagePolicyService = createPackagePolicyServiceMock(); + agentPolicyInterface = createMockAgentPolicyService(); + logger = loggerMock.create(); + + service = new AgentlessConnectorsInfraService( + soClient, + esClient, + packagePolicyService, + agentPolicyInterface, + logger + ); + + jest.clearAllMocks(); + }); + + describe('getNativeConnectors', () => { + test('Lists only native connectors', async () => { + const mockResult = { + results: [ + { + id: '00000001', + name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + is_native: false, + }, + { + id: '00000002', + name: 'Github Connector for ACME Organisation', + service_type: 'github', + is_native: true, + }, + ], + count: 2, + }; + esClient.transport.request.mockResolvedValue(mockResult); + + const nativeConnectors = await service.getNativeConnectors(); + expect(nativeConnectors.length).toBe(1); + expect(nativeConnectors[0].id).toBe(mockResult.results[1].id); + expect(nativeConnectors[0].name).toBe(mockResult.results[1].name); + expect(nativeConnectors[0].service_type).toBe(mockResult.results[1].service_type); + }); + + test('Lists only supported service types', async () => { + const mockResult = { + results: [ + { + id: '00000001', + name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + is_native: true, + }, + { + id: '00000002', + name: 'Github Connector for ACME Organisation', + service_type: 'github', + is_native: true, + }, + { + id: '00000003', + name: 'Connector with unexpected service_type', + service_type: 'crawler', + is_native: true, + }, + { + id: '00000004', + name: 'Connector with no service_type', + service_type: null, + is_native: true, + }, + ], + count: 4, + }; + esClient.transport.request.mockResolvedValue(mockResult); + + const nativeConnectors = await service.getNativeConnectors(); + expect(nativeConnectors.length).toBe(2); + expect(nativeConnectors[0].id).toBe(mockResult.results[0].id); + expect(nativeConnectors[0].name).toBe(mockResult.results[0].name); + expect(nativeConnectors[0].service_type).toBe(mockResult.results[0].service_type); + expect(nativeConnectors[1].id).toBe(mockResult.results[1].id); + expect(nativeConnectors[1].name).toBe(mockResult.results[1].name); + expect(nativeConnectors[1].service_type).toBe(mockResult.results[1].service_type); + }); + }); + describe('getConnectorPackagePolicies', () => { + const getMockPolicyFetchAllItems = (pages: PackagePolicy[][]) => { + return { + async *[Symbol.asyncIterator]() { + for (const page of pages) { + yield page; + } + }, + } as AsyncIterable; + }; + + test('Lists only policies with expected input', async () => { + const firstPackagePolicy = createPackagePolicyMock(); + firstPackagePolicy.id = 'this-is-package-policy-id'; + firstPackagePolicy.policy_ids = ['this-is-agent-policy-id']; + firstPackagePolicy.supports_agentless = true; + firstPackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + const secondPackagePolicy = createPackagePolicyMock(); + secondPackagePolicy.supports_agentless = true; + const thirdPackagePolicy = createPackagePolicyMock(); + thirdPackagePolicy.supports_agentless = true; + thirdPackagePolicy.inputs = [ + { + type: 'something-unsupported', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + + packagePolicyService.fetchAllItems.mockResolvedValue( + getMockPolicyFetchAllItems([[firstPackagePolicy, secondPackagePolicy, thirdPackagePolicy]]) + ); + + const policies = await service.getConnectorPackagePolicies(); + + expect(policies.length).toBe(1); + expect(policies[0].package_policy_id).toBe(firstPackagePolicy.id); + expect(policies[0].connector_metadata.id).toBe( + firstPackagePolicy.inputs[0].compiled_input.connector_id + ); + expect(policies[0].connector_metadata.name).toBe( + firstPackagePolicy.inputs[0].compiled_input.connector_name + ); + expect(policies[0].connector_metadata.service_type).toBe( + firstPackagePolicy.inputs[0].compiled_input.service_type + ); + expect(policies[0].agent_policy_ids).toBe(firstPackagePolicy.policy_ids); + }); + + test('Lists policies if they are returned over multiple pages', async () => { + const firstPackagePolicy = createPackagePolicyMock(); + firstPackagePolicy.id = 'this-is-package-policy-id'; + firstPackagePolicy.policy_ids = ['this-is-agent-policy-id']; + firstPackagePolicy.supports_agentless = true; + firstPackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + const secondPackagePolicy = createPackagePolicyMock(); + secondPackagePolicy.supports_agentless = true; + const thirdPackagePolicy = createPackagePolicyMock(); + thirdPackagePolicy.supports_agentless = true; + thirdPackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000003', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'github', + }, + } as PackagePolicyInput, + ]; + + packagePolicyService.fetchAllItems.mockResolvedValue( + getMockPolicyFetchAllItems([ + [firstPackagePolicy], + [secondPackagePolicy], + [thirdPackagePolicy], + ]) + ); + + const policies = await service.getConnectorPackagePolicies(); + + expect(policies.length).toBe(2); + expect(policies[0].package_policy_id).toBe(firstPackagePolicy.id); + expect(policies[0].connector_metadata.id).toBe( + firstPackagePolicy.inputs[0].compiled_input.connector_id + ); + expect(policies[0].connector_metadata.name).toBe( + firstPackagePolicy.inputs[0].compiled_input.connector_name + ); + expect(policies[0].connector_metadata.service_type).toBe( + firstPackagePolicy.inputs[0].compiled_input.service_type + ); + expect(policies[0].agent_policy_ids).toBe(firstPackagePolicy.policy_ids); + + expect(policies[1].package_policy_id).toBe(thirdPackagePolicy.id); + expect(policies[1].connector_metadata.id).toBe( + thirdPackagePolicy.inputs[0].compiled_input.connector_id + ); + expect(policies[1].connector_metadata.name).toBe( + thirdPackagePolicy.inputs[0].compiled_input.connector_name + ); + expect(policies[1].connector_metadata.service_type).toBe( + thirdPackagePolicy.inputs[0].compiled_input.service_type + ); + expect(policies[1].agent_policy_ids).toBe(thirdPackagePolicy.policy_ids); + }); + + test('Skips policies that have missing fields', async () => { + const firstPackagePolicy = createPackagePolicyMock(); + firstPackagePolicy.id = 'this-is-package-policy-id'; + firstPackagePolicy.policy_ids = ['this-is-agent-policy-id']; + firstPackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + }, + } as PackagePolicyInput, + ]; + const secondPackagePolicy = createPackagePolicyMock(); + secondPackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_name: 'Sharepoint Online Production Connector', + service_type: 'github', + }, + } as PackagePolicyInput, + ]; + + packagePolicyService.fetchAllItems.mockResolvedValue( + getMockPolicyFetchAllItems([[firstPackagePolicy], [secondPackagePolicy]]) + ); + + const policies = await service.getConnectorPackagePolicies(); + + expect(policies.length).toBe(0); + }); + }); + describe('deployConnector', () => { + let agentPolicy: AgentPolicy; + let sharepointOnlinePackagePolicy: PackagePolicy; + + beforeAll(() => { + agentPolicy = createAgentPolicyMock(); + + sharepointOnlinePackagePolicy = createPackagePolicyMock(); + sharepointOnlinePackagePolicy.id = 'this-is-package-policy-id'; + sharepointOnlinePackagePolicy.policy_ids = ['this-is-agent-policy-id']; + sharepointOnlinePackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + }); + + test('Raises an error if connector.id is missing', async () => { + const connector = { + id: '', + name: 'something', + service_type: 'github', + }; + + try { + await service.deployConnector(connector); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toContain('Connector id'); + } + }); + + test('Raises an error if connector.service_type is missing', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: '', + }; + + try { + await service.deployConnector(connector); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toContain('service_type'); + } + }); + + test('Raises an error if connector.service_type is unsupported', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: 'crawler', + }; + + try { + await service.deployConnector(connector); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toContain('service_type'); + expect(e.message).toContain('incompatible'); + } + }); + + test('Does not swallow an error if agent policy creation failed', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: 'github', + }; + const errorMessage = 'Failed to create an agent policy hehe'; + + agentPolicyInterface.create.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.deployConnector(connector); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Does not swallow an error if package policy creation failed', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: 'github', + }; + const errorMessage = 'Failed to create a package policy hehe'; + + agentPolicyInterface.create.mockResolvedValue(agentPolicy); + packagePolicyService.create.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.deployConnector(connector); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Returns a created package policy when all goes well', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: 'github', + }; + + agentPolicyInterface.create.mockResolvedValue(agentPolicy); + packagePolicyService.create.mockResolvedValue(sharepointOnlinePackagePolicy); + + const result = await service.deployConnector(connector); + expect(result).toBe(sharepointOnlinePackagePolicy); + }); + }); + describe('removeDeployment', () => { + const packagePolicyId = 'this-is-package-policy-id'; + const agentPolicyId = 'this-is-agent-policy-id'; + let sharepointOnlinePackagePolicy: PackagePolicy; + + beforeAll(() => { + sharepointOnlinePackagePolicy = createPackagePolicyMock(); + sharepointOnlinePackagePolicy.id = packagePolicyId; + sharepointOnlinePackagePolicy.policy_ids = [agentPolicyId]; + sharepointOnlinePackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + }); + + test('Calls for deletion of both agent policy and package policy', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + await service.removeDeployment(packagePolicyId); + + expect(agentPolicyInterface.delete).toBeCalledWith(soClient, esClient, agentPolicyId); + expect(packagePolicyService.delete).toBeCalledWith(soClient, esClient, [packagePolicyId]); + }); + + test('Raises an error if deletion of agent policy failed', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + const errorMessage = 'Failed to create a package policy hehe'; + + agentPolicyInterface.delete.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Raises an error if deletion of package policy failed', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + const errorMessage = 'Failed to create a package policy hehe'; + + packagePolicyService.delete.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Raises an error if a policy is not found', async () => { + packagePolicyService.get.mockResolvedValue(null); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toContain('Failed to delete policy'); + expect(e.message).toContain(packagePolicyId); + } + }); + }); +}); + +describe('module', () => { + const githubConnector: ConnectorMetadata = { + id: '000001', + name: 'Github Connector', + service_type: 'github', + }; + + const sharepointConnector: ConnectorMetadata = { + id: '000002', + name: 'Sharepoint Connector', + service_type: 'sharepoint_online', + }; + + const mysqlConnector: ConnectorMetadata = { + id: '000003', + name: 'MySQL Connector', + service_type: 'mysql', + }; + + const githubPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-001', + agent_policy_ids: ['agent-package-001'], + connector_metadata: githubConnector, + }; + + const sharepointPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-002', + agent_policy_ids: ['agent-package-002'], + connector_metadata: sharepointConnector, + }; + + const mysqlPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-003', + agent_policy_ids: ['agent-package-003'], + connector_metadata: mysqlConnector, + }; + + describe('getPoliciesWithoutConnectors', () => { + test('Returns a missing policy if one is missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector] + ); + + expect(missingPolicies.length).toBe(1); + expect(missingPolicies).toContain(mysqlPackagePolicy); + }); + + test('Returns empty array if no policies are missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingPolicies.length).toBe(0); + }); + + test('Returns all policies if all are missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [] + ); + + expect(missingPolicies.length).toBe(3); + expect(missingPolicies).toContain(githubPackagePolicy); + expect(missingPolicies).toContain(sharepointPackagePolicy); + expect(missingPolicies).toContain(mysqlPackagePolicy); + }); + }); + + describe('getConnectorsWithoutPolicies', () => { + test('Returns a missing policy if one is missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [githubPackagePolicy, sharepointPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(1); + expect(missingConnectors).toContain(mysqlConnector); + }); + + test('Returns empty array if no policies are missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(0); + }); + + test('Returns all policies if all are missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(3); + expect(missingConnectors).toContain(githubConnector); + expect(missingConnectors).toContain(sharepointConnector); + expect(missingConnectors).toContain(mysqlConnector); + }); + }); +}); diff --git a/x-pack/solutions/search/plugins/search_connectors/server/services/index.ts b/x-pack/solutions/search/plugins/search_connectors/server/services/index.ts new file mode 100644 index 0000000000000..2acb4143c14e9 --- /dev/null +++ b/x-pack/solutions/search/plugins/search_connectors/server/services/index.ts @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PackagePolicy } from '@kbn/fleet-plugin/common'; +import { AgentPolicyServiceInterface, PackagePolicyClient } from '@kbn/fleet-plugin/server'; +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { NATIVE_CONNECTOR_DEFINITIONS, fetchConnectors } from '@kbn/search-connectors'; +import { getPackageInfo } from '@kbn/fleet-plugin/server/services/epm/packages'; + +export interface ConnectorMetadata { + id: string; + name: string; + service_type: string; +} + +export interface PackagePolicyMetadata { + package_policy_id: string; + agent_policy_ids: string[]; + connector_metadata: ConnectorMetadata; +} + +const connectorsInputName = 'connectors-py'; +const pkgName = 'elastic_connectors'; +const pkgTitle = 'Elastic Connectors'; + +export class AgentlessConnectorsInfraService { + private logger: Logger; + private soClient: SavedObjectsClientContract; + private esClient: ElasticsearchClient; + private packagePolicyService: PackagePolicyClient; + private agentPolicyService: AgentPolicyServiceInterface; + + constructor( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + packagePolicyService: PackagePolicyClient, + agentPolicyService: AgentPolicyServiceInterface, + logger: Logger + ) { + this.logger = logger; + this.soClient = soClient; + this.esClient = esClient; + this.packagePolicyService = packagePolicyService; + this.agentPolicyService = agentPolicyService; + } + + public getNativeConnectors = async (): Promise => { + this.logger.debug(`Fetching all connectors and filtering only to native`); + const nativeConnectors: ConnectorMetadata[] = []; + const allConnectors = await fetchConnectors(this.esClient); + for (const connector of allConnectors) { + if (connector.is_native && connector.service_type != null) { + if (NATIVE_CONNECTOR_DEFINITIONS[connector.service_type] == null) { + this.logger.debug( + `Skipping connector ${connector.id}: unsupported service type ${connector.service_type}` + ); + continue; + } + + nativeConnectors.push({ + id: connector.id, + name: connector.name, + service_type: connector.service_type, + }); + } + } + return nativeConnectors; + }; + + public getConnectorPackagePolicies = async (): Promise => { + const kuery = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`; + this.logger.debug(`Fetching policies with kuery: "${kuery}"`); + const policiesIterator = await this.packagePolicyService.fetchAllItems(this.soClient, { + perPage: 50, + kuery, + }); + const policiesMetadata: PackagePolicyMetadata[] = []; + for await (const policyPage of policiesIterator) { + for (const policy of policyPage) { + for (const input of policy.inputs) { + if (input.type === connectorsInputName) { + if (input.compiled_input != null) { + if (input.compiled_input.service_type == null) { + this.logger.debug(`Policy ${policy.id} is missing service_type, skipping`); + continue; + } + + if (input.compiled_input.connector_id == null) { + this.logger.debug(`Policy ${policy.id} is missing connector_id, skipping`); + continue; + } + + if (input.compiled_input.connector_name == null) { + this.logger.debug(`Policy ${policy.id} is missing connector_name`); + // No need to skip, that's fine + } + + // TODO: We manage all policies here, not only agentless. + // Return this code back once this logic is ironed out + // if (policy.supports_agentless !== true) { + // this.logger.debug(`Policy ${policy.id} does not support agentless, skipping`); + // continue; + // } + + policiesMetadata.push({ + package_policy_id: policy.id, + agent_policy_ids: policy.policy_ids, + connector_metadata: { + id: input.compiled_input.connector_id, + name: input.compiled_input.connector_name || '', + service_type: input.compiled_input.service_type, + }, + }); + } + } + } + } + } + + return policiesMetadata; + }; + + public deployConnector = async (connector: ConnectorMetadata): Promise => { + this.logger.info( + `Connector ${connector.id} has no integration policy associated with it, creating` + ); + + if (connector.id == null || connector.id.trim().length === 0) { + throw new Error(`Connector id is null or empty`); + } + + if (connector.service_type == null || connector.service_type.trim().length === 0) { + throw new Error(`Connector ${connector.id} service_type is null or empty`); + } + + if (NATIVE_CONNECTOR_DEFINITIONS[connector.service_type] == null) { + throw new Error( + `Connector ${connector.id} service_type is incompatible with agentless or unsupported` + ); + } + this.logger.debug(`Getting package version for connectors package ${pkgName}`); + const pkgVersion = await this.getPackageVersion(); + this.logger.debug(`Latest package version for ${pkgName} is ${pkgVersion}`); + + const createdPolicy = await this.agentPolicyService.create(this.soClient, this.esClient, { + name: `${connector.service_type} connector: ${connector.id}`, + description: `Automatically generated on ${new Date(Date.now()).toISOString()}`, + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + inactivity_timeout: 1209600, + is_protected: false, + supports_agentless: true, + }); + + this.logger.info( + `Successfully created agent policy ${createdPolicy.id} for agentless connector ${connector.id}` + ); + this.logger.debug(`Creating a package policy for agentless connector ${connector.id}`); + const packagePolicy = await this.packagePolicyService.create(this.soClient, this.esClient, { + policy_ids: [createdPolicy.id], + package: { + title: pkgTitle, + name: pkgName, + version: pkgVersion, + }, + name: `${connector.service_type} connector ${connector.id}`, + description: '', + namespace: '', + enabled: true, + inputs: [ + { + type: connectorsInputName, + enabled: true, + vars: { + connector_id: { type: 'string', value: connector.id }, + connector_name: { type: 'string', value: connector.name }, + service_type: { type: 'string', value: connector.service_type }, + }, + streams: [], + }, + ], + }); + + this.logger.info( + `Successfully created package policy ${packagePolicy.id} for agentless connector ${connector.id}` + ); + + return packagePolicy; + }; + + public removeDeployment = async (packagePolicyId: string): Promise => { + this.logger.info(`Deleting package policy ${packagePolicyId}`); + + const policy = await this.packagePolicyService.get(this.soClient, packagePolicyId); + + if (policy == null) { + throw new Error(`Failed to delete policy ${packagePolicyId}: not found`); + } + + await this.packagePolicyService.delete(this.soClient, this.esClient, [policy.id]); + + this.logger.debug(`Deleting package policies with ids ${policy.policy_ids}`); + + // TODO: can we do it in one go? + // Why not use deleteFleetServerPoliciesForPolicyId? + for (const agentPolicyId of policy.policy_ids) { + this.logger.info(`Deleting agent policy ${agentPolicyId}`); + await this.agentPolicyService.delete(this.soClient, this.esClient, agentPolicyId); + } + }; + + private getPackageVersion = async (): Promise => { + this.logger.debug(`Fetching ${pkgName} version`); + + // This will raise an error if package is not there. + // Situation is exceptional, so we can just show + // the error message from getPackageInfo in this case + const packageInfo = await getPackageInfo({ + savedObjectsClient: this.soClient, + pkgName, + pkgVersion: '', + skipArchive: true, + ignoreUnverified: true, + prerelease: true, + }); + this.logger.debug(`Found ${pkgName} version: ${packageInfo.version}`); + + return packageInfo.version; + }; +} + +export const getConnectorsWithoutPolicies = ( + packagePolicies: PackagePolicyMetadata[], + connectors: ConnectorMetadata[] +): ConnectorMetadata[] => { + return connectors.filter( + (x) => packagePolicies.filter((y) => y.connector_metadata.id === x.id).length === 0 + ); +}; + +export const getPoliciesWithoutConnectors = ( + packagePolicies: PackagePolicyMetadata[], + connectors: ConnectorMetadata[] +): PackagePolicyMetadata[] => { + return packagePolicies.filter( + (x) => connectors.filter((y) => y.id === x.connector_metadata.id).length === 0 + ); +}; diff --git a/x-pack/solutions/search/plugins/search_connectors/server/task.test.ts b/x-pack/solutions/search/plugins/search_connectors/server/task.test.ts new file mode 100644 index 0000000000000..c77f443f9d7ed --- /dev/null +++ b/x-pack/solutions/search/plugins/search_connectors/server/task.test.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { infraSyncTaskRunner } from './task'; +import { ConcreteTaskInstance, TaskStatus } from '@kbn/task-manager-plugin/server'; +import { + AgentlessConnectorsInfraService, + ConnectorMetadata, + PackagePolicyMetadata, +} from './services'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { createPackagePolicyMock } from '@kbn/fleet-plugin/common/mocks'; + +const DATE_1970 = '1970-01-01T00:00:00.000Z'; + +describe('infraSyncTaskRunner', () => { + const githubConnector: ConnectorMetadata = { + id: '000001', + name: 'Github Connector', + service_type: 'github', + }; + + const sharepointConnector: ConnectorMetadata = { + id: '000002', + name: 'Sharepoint Connector', + service_type: 'sharepoint_online', + }; + + const mysqlConnector: ConnectorMetadata = { + id: '000003', + name: 'MySQL Connector', + service_type: 'mysql', + }; + + const githubPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-001', + agent_policy_ids: ['agent-package-001'], + connector_metadata: githubConnector, + }; + + const sharepointPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-002', + agent_policy_ids: ['agent-package-002'], + connector_metadata: sharepointConnector, + }; + + const mysqlPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-003', + agent_policy_ids: ['agent-package-003'], + connector_metadata: mysqlConnector, + }; + + let logger: MockedLogger; + let serviceMock: jest.Mocked; + let licensePluginStartMock: jest.Mocked; + + const taskInstanceStub: ConcreteTaskInstance = { + id: '', + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(DATE_1970), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: {}, + taskType: 'backfill', + timeoutOverride: '3m', + params: { + adHocRunParamsId: 'abc', + spaceId: 'default', + }, + ownerId: null, + }; + + const invalidLicenseMock = licensingMock.createLicenseMock(); + invalidLicenseMock.check.mockReturnValue({ state: 'invalid' }); + + const validLicenseMock = licensingMock.createLicenseMock(); + validLicenseMock.check.mockReturnValue({ state: 'valid' }); + + beforeAll(async () => { + logger = loggerMock.create(); + serviceMock = { + getNativeConnectors: jest.fn(), + getConnectorPackagePolicies: jest.fn(), + deployConnector: jest.fn(), + removeDeployment: jest.fn(), + } as unknown as jest.Mocked; + + licensePluginStartMock = { + getLicense: jest.fn(), + } as unknown as jest.Mocked; + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('Does nothing if no connectors or policies are configured', async () => { + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + }); + + test('Does nothing if connectors or policies requires deployment but license is not supported', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(invalidLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + expect(logger.warn).toBeCalledWith(expect.stringMatching(/.*not compatible.*/)); + expect(logger.warn).toBeCalledWith(expect.stringMatching(/.*license.*/)); + }); + + test('Does nothing if all connectors and package policies are in-sync', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([ + mysqlConnector, + githubConnector, + sharepointConnector, + ]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([ + mysqlPackagePolicy, + githubPackagePolicy, + sharepointPackagePolicy, + ]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + expect(logger.warn).not.toBeCalled(); + }); + + test('Deploys connectors if no policies has been created for these connectors', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).toBeCalledWith(mysqlConnector); + expect(serviceMock.deployConnector).toBeCalledWith(githubConnector); + }); + + test('Deploys connectors even if another connectors failed to be deployed', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([ + mysqlConnector, + githubConnector, + sharepointConnector, + ]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + serviceMock.deployConnector.mockImplementation(async (connector) => { + if (connector === mysqlConnector || connector === githubConnector) { + throw new Error('Cannot deploy these connectors'); + } + + return createPackagePolicyMock(); + }); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).toBeCalledWith(mysqlConnector); + expect(serviceMock.deployConnector).toBeCalledWith(githubConnector); + expect(serviceMock.deployConnector).toBeCalledWith(sharepointConnector); + }); + + test('Removes a package policy if no connectors match the policy', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.removeDeployment).toBeCalledWith(sharepointPackagePolicy.package_policy_id); + }); + + test('Removes deployments even if another connectors failed to be undeployed', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([ + sharepointPackagePolicy, + mysqlPackagePolicy, + githubPackagePolicy, + ]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + serviceMock.removeDeployment.mockImplementation(async (policyId) => { + if ( + policyId === sharepointPackagePolicy.package_policy_id || + policyId === mysqlPackagePolicy.package_policy_id + ) { + throw new Error('Cannot deploy these connectors'); + } + }); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.removeDeployment).toBeCalledWith(sharepointPackagePolicy.package_policy_id); + expect(serviceMock.removeDeployment).toBeCalledWith(mysqlPackagePolicy.package_policy_id); + expect(serviceMock.removeDeployment).toBeCalledWith(githubPackagePolicy.package_policy_id); + }); +}); diff --git a/x-pack/solutions/search/plugins/search_connectors/server/task.ts b/x-pack/solutions/search/plugins/search_connectors/server/task.ts new file mode 100644 index 0000000000000..7ecd538e0f7ce --- /dev/null +++ b/x-pack/solutions/search/plugins/search_connectors/server/task.ts @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, CoreStart, SavedObjectsClient } from '@kbn/core/server'; + +import type { + ConcreteTaskInstance, + TaskManagerStartContract, + TaskInstance, +} from '@kbn/task-manager-plugin/server'; + +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { + SearchConnectorsPluginStartDependencies, + SearchConnectorsPluginSetupDependencies, +} from './types'; +import { + AgentlessConnectorsInfraService, + getConnectorsWithoutPolicies, + getPoliciesWithoutConnectors, +} from './services'; + +import { SearchConnectorsConfig } from './config'; + +const AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID = 'search:agentless-connectors-manager-task'; +const AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_TYPE = 'search:agentless-connectors-manager'; + +const SCHEDULE = { interval: '1m' }; + +export function infraSyncTaskRunner( + logger: Logger, + service: AgentlessConnectorsInfraService, + licensingPluginStart: LicensingPluginStart +) { + return ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + try { + // We fetch some info even if license does not permit actual operations. + // This is done so that we could give a warning to the user only + // if they are actually using the feature. + logger.debug('Checking state of connectors and agentless policies'); + + // Fetch connectors + const nativeConnectors = await service.getNativeConnectors(); + + const policiesMetadata = await service.getConnectorPackagePolicies(); + + // Check license if any native connectors or agentless policies found + if (nativeConnectors.length > 0 || policiesMetadata.length > 0) { + const license = await licensingPluginStart.getLicense(); + + if (license.check('fleet', 'platinum').state !== 'valid') { + logger.warn( + 'Current license is not compatible with agentless connectors. Please upgrade the license to at least platinum' + ); + return; + } + } + + // Deploy Policies + const connectorsWithoutPolicies = getConnectorsWithoutPolicies( + policiesMetadata, + nativeConnectors + ); + + let agentlessConnectorsDeployed = 0; + for (const connectorMetadata of connectorsWithoutPolicies) { + // We try-catch to still be able to deploy other connectors if some fail + try { + await service.deployConnector(connectorMetadata); + + agentlessConnectorsDeployed += 1; + } catch (e) { + logger.warn( + `Error creating an agentless deployment for connector ${connectorMetadata.id}: ${e.message}` + ); + } + } + + // Delete policies + const policiesWithoutConnectors = getPoliciesWithoutConnectors( + policiesMetadata, + nativeConnectors + ); + let agentlessConnectorsRemoved = 0; + + for (const policyMetadata of policiesWithoutConnectors) { + // We try-catch to still be able to deploy other connectors if some fail + try { + await service.removeDeployment(policyMetadata.package_policy_id); + + agentlessConnectorsRemoved += 1; + } catch (e) { + logger.warn( + `Error when deleting a package policy ${policyMetadata.package_policy_id}: ${e.message}` + ); + } + } + return { + state: {}, + schedule: SCHEDULE, + }; + } catch (e) { + logger.warn(`Error executing agentless deployment sync task: ${e.message}`); + return { + state: {}, + schedule: SCHEDULE, + }; + } + }, + cancel: async () => { + logger.warn('timed out'); + }, + }; + }; +} + +export class AgentlessConnectorDeploymentsSyncService { + private logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + public registerInfraSyncTask( + plugins: SearchConnectorsPluginSetupDependencies, + coreStart: CoreStart, + searchConnectorsPluginStartDependencies: SearchConnectorsPluginStartDependencies + ) { + const taskManager = plugins.taskManager; + + const esClient = coreStart.elasticsearch.client.asInternalUser; + const savedObjects = coreStart.savedObjects; + + const agentPolicyService = searchConnectorsPluginStartDependencies.fleet.agentPolicyService; + const packagePolicyService = searchConnectorsPluginStartDependencies.fleet.packagePolicyService; + + const soClient = new SavedObjectsClient(savedObjects.createInternalRepository()); + + const service = new AgentlessConnectorsInfraService( + soClient, + esClient, + packagePolicyService, + agentPolicyService, + this.logger + ); + + taskManager.registerTaskDefinitions({ + [AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_TYPE]: { + title: 'Agentless Connector Deployment Manager', + description: + 'This task peridocally checks native connectors, agent policies and syncs them if they are out of sync', + timeout: '1m', + maxAttempts: 3, + createTaskRunner: infraSyncTaskRunner( + this.logger, + service, + searchConnectorsPluginStartDependencies.licensing + ), + }, + }); + } + + public async scheduleInfraSyncTask( + config: SearchConnectorsConfig, + taskManager: TaskManagerStartContract + ): Promise { + this.logger.info(`Scheduling ${AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID}`); + try { + await taskManager.removeIfExists(AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID); + const taskInstance = await taskManager.ensureScheduled({ + id: AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID, + taskType: AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_TYPE, + schedule: SCHEDULE, + params: {}, + state: {}, + scope: ['search', 'connectors'], + }); + + this.logger.info( + `Scheduled ${AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID} with interval ${taskInstance.schedule?.interval}` + ); + + return taskInstance; + } catch (e) { + this.logger.error( + `Error scheduling ${AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_ID}, received ${e.message}` + ); + return null; + } + } +} diff --git a/x-pack/solutions/search/plugins/search_connectors/server/types.ts b/x-pack/solutions/search/plugins/search_connectors/server/types.ts index 36b5aa877fd1e..cfd29370fee5f 100644 --- a/x-pack/solutions/search/plugins/search_connectors/server/types.ts +++ b/x-pack/solutions/search/plugins/search_connectors/server/types.ts @@ -6,6 +6,13 @@ */ import { ConnectorServerSideDefinition } from '@kbn/search-connectors'; +import type { FleetStartContract, FleetSetupContract } from '@kbn/fleet-plugin/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { SavedObjectsServiceSetup, SavedObjectsServiceStart } from '@kbn/core-saved-objects-server'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; /* eslint-disable @typescript-eslint/no-empty-interface */ @@ -15,5 +22,14 @@ export interface SearchConnectorsPluginSetup { export interface SearchConnectorsPluginStart {} -export interface StartDependencies {} -export interface SetupDependencies {} +export interface SearchConnectorsPluginStartDependencies { + fleet: FleetStartContract; + taskManager: TaskManagerStartContract; + soClient: SavedObjectsServiceStart; + licensing: LicensingPluginStart; +} +export interface SearchConnectorsPluginSetupDependencies { + fleet: FleetSetupContract; + taskManager: TaskManagerSetupContract; + soClient: SavedObjectsServiceSetup; +} diff --git a/x-pack/solutions/search/plugins/search_connectors/tsconfig.json b/x-pack/solutions/search/plugins/search_connectors/tsconfig.json index ba6f6d85f5102..db177b77269ab 100644 --- a/x-pack/solutions/search/plugins/search_connectors/tsconfig.json +++ b/x-pack/solutions/search/plugins/search_connectors/tsconfig.json @@ -19,5 +19,13 @@ "@kbn/config-schema", "@kbn/core-http-browser", "@kbn/search-connectors", + "@kbn/task-manager-plugin", + "@kbn/fleet-plugin", + "@kbn/core-saved-objects-api-server", + "@kbn/core-elasticsearch-client-server-mocks", + "@kbn/logging-mocks", + "@kbn/core-elasticsearch-server", + "@kbn/licensing-plugin", + "@kbn/core-saved-objects-server", ] } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 682b2b9ec5f7d..1f9664907b40c 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -161,6 +161,7 @@ export default function ({ getService }: FtrProviderContext) { 'osquery:telemetry-saved-queries', 'report:execute', 'risk_engine:risk_scoring', + 'search:agentless-connectors-manager', 'security-solution-ea-asset-criticality-ecs-migration', 'security:endpoint-diagnostics', 'security:endpoint-meta-telemetry',