diff --git a/x-pack/platform/plugins/shared/fleet/server/plugin.ts b/x-pack/platform/plugins/shared/fleet/server/plugin.ts index 0e5659134fa97..efed058030e27 100644 --- a/x-pack/platform/plugins/shared/fleet/server/plugin.ts +++ b/x-pack/platform/plugins/shared/fleet/server/plugin.ts @@ -28,7 +28,7 @@ import type { } from '@kbn/core/server'; import { DEFAULT_APP_CATEGORIES, SavedObjectsClient, ServiceStatusLevels } from '@kbn/core/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; - +import { LockManagerService } from '@kbn/lock-manager'; import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-plugin/server'; import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; @@ -49,9 +49,7 @@ import type { } from '@kbn/task-manager-plugin/server'; import type { CloudSetup } from '@kbn/cloud-plugin/server'; - import type { SpacesPluginStart } from '@kbn/spaces-plugin/server'; - import type { SavedObjectTaggingStart } from '@kbn/saved-objects-tagging-plugin/server'; import { SECURITY_EXTENSION_ID } from '@kbn/core-saved-objects-server'; @@ -207,6 +205,7 @@ export interface FleetAppContext { taskManagerStart?: TaskManagerStartContract; fetchUsage?: (abortController: AbortController) => Promise; syncIntegrationsTask: SyncIntegrationsTask; + lockManagerService?: LockManagerService; } export type FleetSetupContract = void; @@ -317,6 +316,7 @@ export class FleetPlugin private packagePolicyService?: PackagePolicyService; private policyWatcher?: PolicyWatcher; private fetchUsage?: (abortController: AbortController) => Promise; + private lockManagerService?: LockManagerService; constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = this.initializerContext.config.create(); @@ -673,6 +673,7 @@ export class FleetPlugin taskManager: deps.taskManager, logFactory: this.initializerContext.logger, }); + this.lockManagerService = new LockManagerService(core, this.initializerContext.logger.get()); // Register fields metadata extractors registerFieldsMetadataExtractors({ core, fieldsMetadata: deps.fieldsMetadata }); @@ -725,6 +726,7 @@ export class FleetPlugin taskManagerStart: plugins.taskManager, fetchUsage: this.fetchUsage, syncIntegrationsTask: this.syncIntegrationsTask!, + lockManagerService: this.lockManagerService, }); licenseService.start(plugins.licensing.license$); this.telemetryEventsSender.start(plugins.telemetry, core).catch(() => {}); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/app_context.ts b/x-pack/platform/plugins/shared/fleet/server/services/app_context.ts index e1cd44736fc1e..5d7418b0afaf7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/app_context.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/app_context.ts @@ -11,7 +11,6 @@ import { kibanaPackageJson } from '@kbn/repo-info'; import type { HttpServiceSetup, KibanaRequest } from '@kbn/core-http-server'; import { kibanaRequestFactory } from '@kbn/core-http-server-utils'; - import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import type { EncryptedSavedObjectsClient, @@ -28,6 +27,7 @@ import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { SecurityServiceStart } from '@kbn/core-security-server'; import type { Logger } from '@kbn/logging'; +import type { LockManagerService } from '@kbn/lock-manager'; import type { FleetConfigType } from '../../common/types'; import { @@ -49,10 +49,10 @@ import type { FleetAppContext } from '../plugin'; import type { TelemetryEventsSender } from '../telemetry/sender'; import { UNINSTALL_TOKENS_SAVED_OBJECT_TYPE } from '../constants'; import type { MessageSigningServiceInterface } from '..'; +import type { FleetUsage } from '../collectors/register'; import type { BulkActionsResolver } from './agents/bulk_actions_resolver'; import { type UninstallTokenServiceInterface } from './security/uninstall_token_service'; -import type { FleetUsage } from '../collectors/register'; class AppContextService { private encryptedSavedObjects: EncryptedSavedObjectsClient | undefined; @@ -82,6 +82,7 @@ class AppContextService { private uninstallTokenService: UninstallTokenServiceInterface | undefined; private taskManagerStart: TaskManagerStartContract | undefined; private fetchUsage?: (abortController: AbortController) => Promise; + private lockManagerService: LockManagerService | undefined; public start(appContext: FleetAppContext) { this.data = appContext.data; @@ -108,6 +109,7 @@ class AppContextService { this.uninstallTokenService = appContext.uninstallTokenService; this.taskManagerStart = appContext.taskManagerStart; this.fetchUsage = appContext.fetchUsage; + this.lockManagerService = appContext.lockManagerService; if (appContext.config$) { this.config$ = appContext.config$; @@ -351,6 +353,10 @@ class AppContextService { public getFetchUsage() { return this.fetchUsage; } + + public getLockManagerService() { + return this.lockManagerService; + } } export const appContextService = new AppContextService(); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/setup.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/setup.test.ts index 6dbad62ec71ac..04e144d1e2aa8 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/setup.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/setup.test.ts @@ -8,6 +8,7 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import type { ElasticsearchClientMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { LockAcquisitionError } from '@kbn/lock-manager'; import type { Logger } from '@kbn/core/server'; @@ -20,7 +21,7 @@ import { appContextService } from './app_context'; import { getInstallations } from './epm/packages'; import { setupUpgradeManagedPackagePolicies } from './setup/managed_package_policies'; import { getPreconfiguredDeleteUnenrolledAgentsSettingFromConfig } from './preconfiguration/delete_unenrolled_agent_setting'; -import { setupFleet } from './setup'; +import { _runSetupWithLock, setupFleet } from './setup'; import { isPackageInstalled } from './epm/packages/install'; import { upgradeAgentPolicySchemaVersion } from './setup/upgrade_agent_policy_schema_version'; import { createOrUpdateFleetSyncedIntegrationsIndex } from './setup/fleet_synced_integrations'; @@ -141,69 +142,6 @@ describe('setupFleet', () => { }); }); - it('should create and delete lock if not exists', async () => { - const soClient = getMockedSoClient(); - - soClient.get.mockRejectedValue({ isBoom: true, output: { statusCode: 404 } } as any); - - const result = await setupFleet(soClient, esClient, { useLock: true }); - - expect(result).toEqual({ - isInitialized: true, - nonFatalErrors: [], - }); - expect(soClient.create).toHaveBeenCalledWith('fleet-setup-lock', expect.anything(), { - id: 'fleet-setup-lock', - }); - expect(soClient.delete).toHaveBeenCalledWith('fleet-setup-lock', 'fleet-setup-lock', { - refresh: true, - }); - }); - - it('should return not initialized if lock exists', async () => { - const soClient = getMockedSoClient(); - - const result = await setupFleet(soClient, esClient, { useLock: true }); - - expect(result).toEqual({ - isInitialized: false, - nonFatalErrors: [], - }); - expect(soClient.create).not.toHaveBeenCalled(); - expect(soClient.delete).not.toHaveBeenCalled(); - }); - - it('should return not initialized if lock could not be created', async () => { - const soClient = getMockedSoClient(); - - soClient.get.mockRejectedValue({ isBoom: true, output: { statusCode: 404 } } as any); - soClient.create.mockRejectedValue({ isBoom: true, output: { statusCode: 409 } } as any); - const result = await setupFleet(soClient, esClient, { useLock: true }); - - expect(result).toEqual({ - isInitialized: false, - nonFatalErrors: [], - }); - expect(soClient.delete).not.toHaveBeenCalled(); - }); - - it('should delete previous lock if created more than 1 hour ago', async () => { - const soClient = getMockedSoClient(); - - soClient.get.mockResolvedValue({ - attributes: { started_at: new Date(Date.now() - 60 * 60 * 1000 - 1000).toISOString() }, - } as any); - - const result = await setupFleet(soClient, esClient, { useLock: true }); - - expect(result).toEqual({ - isInitialized: true, - nonFatalErrors: [], - }); - expect(soClient.create).toHaveBeenCalled(); - expect(soClient.delete).toHaveBeenCalledTimes(2); - }); - it('should return non fatal errors when generateKeyPair result has errors', async () => { const soClient = getMockedSoClient(); @@ -228,3 +166,42 @@ describe('setupFleet', () => { }); }); }); + +describe('_runSetupWithLock', () => { + let mockedWithLock: jest.Mock; + beforeEach(() => { + mockedWithLock = jest.fn(); + mockedAppContextService.getLockManagerService.mockReturnValue({ + withLock: mockedWithLock as any, + } as any); + }); + it('should retry on lock acquisition error', async () => { + mockedWithLock + .mockImplementationOnce(async () => { + throw new LockAcquisitionError('test'); + }) + .mockImplementationOnce(async (id, fn) => { + return fn(); + }); + + const setupFn = jest.fn(); + await _runSetupWithLock(setupFn); + + expect(setupFn).toHaveBeenCalled(); + expect(mockedWithLock).toHaveBeenCalledTimes(2); + }); + + it('should not retry on setupFn error', async () => { + mockedWithLock.mockImplementation(async (id, fn) => { + return fn(); + }); + + const setupFn = jest.fn(); + setupFn.mockRejectedValue(new Error('test')); + + await expect(_runSetupWithLock(setupFn)).rejects.toThrow(/test/); + + expect(setupFn).toHaveBeenCalled(); + expect(mockedWithLock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/setup.ts b/x-pack/platform/plugins/shared/fleet/server/services/setup.ts index bc04ac4ad7782..0a3f323b19780 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/setup.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/setup.ts @@ -11,15 +11,17 @@ import apm from 'elastic-apm-node'; import { compact } from 'lodash'; import pMap from 'p-map'; -import { v4 as uuidv4 } from 'uuid'; +import pRetry from 'p-retry'; + import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import { LockAcquisitionError } from '@kbn/lock-manager'; import { MessageSigningError } from '../../common/errors'; -import { AUTO_UPDATE_PACKAGES, FLEET_SETUP_LOCK_TYPE } from '../../common/constants'; +import { AUTO_UPDATE_PACKAGES } from '../../common/constants'; import type { PreconfigurationError } from '../../common/constants'; -import type { DefaultPackagesInstallationError, FleetSetupLock } from '../../common/types'; +import type { DefaultPackagesInstallationError } from '../../common/types'; import { MAX_CONCURRENT_EPM_PACKAGES_INSTALLATIONS } from '../constants'; @@ -78,6 +80,20 @@ export interface SetupStatus { >; } +export async function _runSetupWithLock(setupFn: () => Promise) { + return await pRetry( + () => appContextService.getLockManagerService()!.withLock('fleet-setup', () => setupFn()), + { + onFailedAttempt: async (error) => { + if (!(error instanceof LockAcquisitionError)) { + throw error; + } + }, + maxRetryTime: 5 * 60 * 1000, // Retry for 5 minute to get the lock + } + ); +} + export async function setupFleet( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, @@ -86,87 +102,20 @@ export async function setupFleet( } = { useLock: false } ): Promise { const t = apm.startTransaction('fleet-setup', 'fleet'); - let created = false; try { if (options.useLock) { - const { created: isCreated, toReturn } = await createLock(soClient); - created = isCreated; - if (toReturn) return toReturn; + return _runSetupWithLock(() => + awaitIfPending(async () => createSetupSideEffects(soClient, esClient)) + ); + } else { + return await awaitIfPending(async () => createSetupSideEffects(soClient, esClient)); } - return await awaitIfPending(async () => createSetupSideEffects(soClient, esClient)); } catch (error) { apm.captureError(error); t.setOutcome('failure'); throw error; } finally { t.end(); - // only delete lock if it was created by this instance - if (options.useLock && created) { - await deleteLock(soClient); - } - } -} - -async function createLock( - soClient: SavedObjectsClientContract -): Promise<{ created: boolean; toReturn?: SetupStatus }> { - const logger = appContextService.getLogger(); - let created; - try { - // check if fleet setup is already started - const fleetSetupLock = await soClient.get( - FLEET_SETUP_LOCK_TYPE, - FLEET_SETUP_LOCK_TYPE - ); - - const LOCK_TIMEOUT = 60 * 60 * 1000; // 1 hour - - // started more than 1 hour ago, delete previous lock - if ( - fleetSetupLock.attributes.started_at && - new Date(fleetSetupLock.attributes.started_at).getTime() < Date.now() - LOCK_TIMEOUT - ) { - await deleteLock(soClient); - } else { - logger.info('Fleet setup already in progress, abort setup'); - return { created: false, toReturn: { isInitialized: false, nonFatalErrors: [] } }; - } - } catch (error) { - if (error.isBoom && error.output.statusCode === 404) { - logger.debug('Fleet setup lock does not exist, continue setup'); - } - } - - try { - created = await soClient.create( - FLEET_SETUP_LOCK_TYPE, - { - status: 'in_progress', - uuid: uuidv4(), - started_at: new Date().toISOString(), - }, - { id: FLEET_SETUP_LOCK_TYPE } - ); - if (logger.isLevelEnabled('debug')) { - logger.debug(`Fleet setup lock created: ${JSON.stringify(created)}`); - } - } catch (error) { - logger.info(`Could not create fleet setup lock, abort setup: ${error}`); - return { created: false, toReturn: { isInitialized: false, nonFatalErrors: [] } }; - } - return { created: !!created }; -} - -async function deleteLock(soClient: SavedObjectsClientContract) { - const logger = appContextService.getLogger(); - try { - await soClient.delete(FLEET_SETUP_LOCK_TYPE, FLEET_SETUP_LOCK_TYPE, { refresh: true }); - logger.debug(`Fleet setup lock deleted`); - } catch (error) { - // ignore 404 errors - if (error.statusCode !== 404) { - logger.error('Could not delete fleet setup lock', error); - } } } diff --git a/x-pack/platform/plugins/shared/fleet/tsconfig.json b/x-pack/platform/plugins/shared/fleet/tsconfig.json index 2f4442e24f158..9d43c51fc1d3a 100644 --- a/x-pack/platform/plugins/shared/fleet/tsconfig.json +++ b/x-pack/platform/plugins/shared/fleet/tsconfig.json @@ -120,5 +120,6 @@ "@kbn/core-http-server-utils", "@kbn/core-notifications-browser-mocks", "@kbn/handlebars", + "@kbn/lock-manager", ] }