diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index ba781c18ad9c..e66c25e407ff 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -16,6 +16,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { SavedObjectsUtils, SavedObjectsErrorHelpers } from '@kbn/core/server'; import minVersion from 'semver/ranges/min-version'; +import { chunk } from 'lodash'; + import { updateIndexSettings } from '../elasticsearch/index/update_settings'; import { @@ -30,6 +32,8 @@ import type { EsAssetReference, KibanaAssetReference, Installation, + ArchivePackage, + RegistryPackage, } from '../../../types'; import { deletePipeline } from '../elasticsearch/ingest_pipeline'; import { removeUnusedIndexPatterns } from '../kibana/index_pattern/install'; @@ -48,6 +52,8 @@ import * as Registry from '../registry'; import { getInstallation, kibanaSavedObjectTypes } from '.'; +const MAX_ASSETS_TO_DELETE = 1000; + export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; @@ -115,61 +121,70 @@ export async function removeInstallation(options: { } /** - * This method resolves saved objects before deleting them. It is needed when - * deleting assets that were installed in 7.x to mitigate the breaking change - * that occurred in 8.0. This is a memory-intensive operation as it requires - * loading all the saved objects into memory. It is generally better to delete - * assets directly if the package is known to be installed in 8.x or later. + * This method deletes saved objects resolving them whenever necessary. + * + * Resolving is needed when deleting assets that were installed in 7.x to + * mitigate the breaking change that occurred in 8.0. This is a memory-intensive + * operation as it requires loading all the saved objects into memory. It is + * generally better to delete assets directly if the package is known to be + * installed in 8.x or later. */ -async function resolveAndDeleteKibanaAssets( - installedObjects: KibanaAssetReference[], - spaceId: string = DEFAULT_SPACE_ID -) { +async function deleteKibanaAssets({ + installedObjects, + packageInfo, + spaceId = DEFAULT_SPACE_ID, +}: { + installedObjects: KibanaAssetReference[]; + spaceId?: string; + packageInfo: RegistryPackage | ArchivePackage; +}) { const savedObjectsClient = new SavedObjectsClient( appContextService.getSavedObjects().createInternalRepository() ); + const namespace = SavedObjectsUtils.namespaceStringToId(spaceId); - const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve( - installedObjects, - { namespace } - ); - for (const { saved_object: savedObject } of resolvedObjects) { - auditLoggingService.writeCustomSoAuditLog({ - action: 'get', - id: savedObject.id, - savedObjectType: savedObject.type, - }); - } + const minKibana = packageInfo.conditions?.kibana?.version + ? minVersion(packageInfo.conditions.kibana.version) + : null; - const foundObjects = resolvedObjects.filter( - ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404 - ); + // Compare Kibana versions to determine if the package could been installed + // only in 8.x or later. If so, we can skip SO resolution step altogether + // and delete the assets directly. Otherwise, we need to resolve the assets + // which might create high memory pressure if a package has a lot of assets. + if (minKibana && minKibana.major >= 8) { + await bulkDeleteSavedObjects(installedObjects, namespace, savedObjectsClient); + } else { + const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve( + installedObjects, + { namespace } + ); - // in the case of a partial install, it is expected that some assets will be not found - // we filter these out before calling delete - const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type })); + for (const { saved_object: savedObject } of resolvedObjects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: savedObject.id, + savedObjectType: savedObject.type, + }); + } - for (const asset of assetsToDelete) { - auditLoggingService.writeCustomSoAuditLog({ - action: 'delete', - id: asset.id, - savedObjectType: asset.type, - }); - } + const foundObjects = resolvedObjects.filter( + ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404 + ); - return savedObjectsClient.bulkDelete(assetsToDelete, { namespace }); + // in the case of a partial install, it is expected that some assets will be not found + // we filter these out before calling delete + const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type })); + + await bulkDeleteSavedObjects(assetsToDelete, namespace, savedObjectsClient); + } } -async function deleteKibanaAssets( - assetsToDelete: KibanaAssetReference[], - spaceId: string = DEFAULT_SPACE_ID +async function bulkDeleteSavedObjects( + assetsToDelete: Array<{ id: string; type: string }>, + namespace: string | undefined, + savedObjectsClient: SavedObjectsClientContract ) { - const savedObjectsClient = new SavedObjectsClient( - appContextService.getSavedObjects().createInternalRepository() - ); - const namespace = SavedObjectsUtils.namespaceStringToId(spaceId); - for (const asset of assetsToDelete) { auditLoggingService.writeCustomSoAuditLog({ action: 'delete', @@ -178,7 +193,14 @@ async function deleteKibanaAssets( }); } - return savedObjectsClient.bulkDelete(assetsToDelete, { namespace }); + // Delete assets in chunks to avoid high memory pressure. This is mostly + // relevant for packages containing many assets, as large payload and response + // objects are created in memory during the delete operation. While chunking + // may work slower, it allows garbage collection to clean up memory between + // requests. + for (const assetsChunk of chunk(assetsToDelete, MAX_ASSETS_TO_DELETE)) { + await savedObjectsClient.bulkDelete(assetsChunk, { namespace }); + } } function deleteESAssets( @@ -211,6 +233,8 @@ async function deleteAssets( installed_kibana: installedKibana, installed_kibana_space_id: spaceId = DEFAULT_SPACE_ID, additional_spaces_installed_kibana: installedInAdditionalSpacesKibana = {}, + name, + version, }: Installation, savedObjectsClient: SavedObjectsClientContract, esClient: ElasticsearchClient @@ -263,12 +287,18 @@ async function deleteAssets( // then delete index templates and pipelines await Promise.all(deleteESAssets(indexTemplatesAndPipelines, esClient)); + + const packageInfo = await Registry.fetchInfo(name, version); // then the other asset types await Promise.all([ ...deleteESAssets(otherAssets, esClient), - resolveAndDeleteKibanaAssets(installedKibana, spaceId), + deleteKibanaAssets({ installedObjects: installedKibana, spaceId, packageInfo }), Object.entries(installedInAdditionalSpacesKibana).map(([additionalSpaceId, kibanaAssets]) => - resolveAndDeleteKibanaAssets(kibanaAssets, additionalSpaceId) + deleteKibanaAssets({ + installedObjects: kibanaAssets, + spaceId: additionalSpaceId, + packageInfo, + }) ), ]); } catch (err) { @@ -328,25 +358,17 @@ export async function deleteKibanaSavedObjectsAssets({ .filter(({ type }) => kibanaSavedObjectTypes.includes(type)) .map(({ id, type }) => ({ id, type } as KibanaAssetReference)); - const registryInfo = await Registry.fetchInfo( - installedPkg.attributes.name, - installedPkg.attributes.version - ); - - const minKibana = registryInfo.conditions?.kibana?.version - ? minVersion(registryInfo.conditions.kibana.version) - : null; - try { - // Compare Kibana versions to determine if the package could been installed - // only in 8.x or later. If so, we can skip SO resolution step altogether - // and delete the assets directly. Otherwise, we need to resolve the assets - // which might create high memory pressure if a package has a lot of assets. - if (minKibana && minKibana.major >= 8) { - await deleteKibanaAssets(assetsToDelete, spaceIdToDelete); - } else { - await resolveAndDeleteKibanaAssets(assetsToDelete, spaceIdToDelete); - } + const packageInfo = await Registry.fetchInfo( + installedPkg.attributes.name, + installedPkg.attributes.version + ); + + await deleteKibanaAssets({ + installedObjects: assetsToDelete, + spaceId: spaceIdToDelete, + packageInfo, + }); } catch (err) { // in the rollback case, partial installs are likely, so missing assets are not an error if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {