[Fleet] Delete installed package assets in chunks to avoid high memor…

…y consumption (#188248)

**Resolves: #188208

## Summary

This PR limits the number of assets that can be deleted in one request.
Before, the bulk delete method was called with all package assets:

```ts
await savedObjectsClient.bulkDelete(assetsToDelete, { namespace });
```

This led to significant request and response objects being created and
stored in memory at the same time before being garbage collected. We
don't want those objects to grow with package size indefinitely, so
splitting that work into smaller chunks to reduce memory pressure. This
slows the installation of larger packages but ensures we do not reach
OOM errors.

For packages with ~5000 saved objects, the installation time was ~50%
slower but with memory consumption not exceeding 800-850Mb during the
package removing phase vs. ~1Gb without the optimization.

**Before**

```
Summary:
  Total:	190.7639 secs
  Slowest:	10.1355 secs
  Fastest:	9.0772 secs
  Average:	9.5382 secs
  Requests/sec:	0.1048
```

![Screenshot 2024-07-12 at 12 40
54](https://github.com/user-attachments/assets/a10aaaa9-ac3d-4a5a-b624-8b4a36400a6a)


**After**
```
Summary:
  Total:	303.8411 secs
  Slowest:	19.1417 secs
  Fastest:	13.0912 secs
  Average:	15.1921 secs
  Requests/sec:	0.0658
```

![Screenshot 2024-07-12 at 12 44
58](https://github.com/user-attachments/assets/2c7bb609-f107-46bd-bd98-43a0c58107e8)

x-pack/plugins/fleet/server/services/epm/packages/remove.ts

@@ -16,6 +16,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 import { SavedObjectsUtils, SavedObjectsErrorHelpers } from '@kbn/core/server';
 import minVersion from 'semver/ranges/min-version';
 
+import { chunk } from 'lodash';
+
 import { updateIndexSettings } from '../elasticsearch/index/update_settings';
 
 import {
@@ -30,6 +32,8 @@ import type {
   EsAssetReference,
   KibanaAssetReference,
   Installation,
+  ArchivePackage,
+  RegistryPackage,
 } from '../../../types';
 import { deletePipeline } from '../elasticsearch/ingest_pipeline';
 import { removeUnusedIndexPatterns } from '../kibana/index_pattern/install';
@@ -48,6 +52,8 @@ import * as Registry from '../registry';
 
 import { getInstallation, kibanaSavedObjectTypes } from '.';
 
+const MAX_ASSETS_TO_DELETE = 1000;
+
 export async function removeInstallation(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
@@ -115,61 +121,70 @@ export async function removeInstallation(options: {
 }
 
 /**
- * This method resolves saved objects before deleting them. It is needed when
- * deleting assets that were installed in 7.x to mitigate the breaking change
- * that occurred in 8.0. This is a memory-intensive operation as it requires
- * loading all the saved objects into memory. It is generally better to delete
- * assets directly if the package is known to be installed in 8.x or later.
+ * This method deletes saved objects resolving them whenever necessary.
+ *
+ * Resolving is needed when deleting assets that were installed in 7.x to
+ * mitigate the breaking change that occurred in 8.0. This is a memory-intensive
+ * operation as it requires loading all the saved objects into memory. It is
+ * generally better to delete assets directly if the package is known to be
+ * installed in 8.x or later.
  */
-async function resolveAndDeleteKibanaAssets(
-  installedObjects: KibanaAssetReference[],
-  spaceId: string = DEFAULT_SPACE_ID
-) {
+async function deleteKibanaAssets({
+  installedObjects,
+  packageInfo,
+  spaceId = DEFAULT_SPACE_ID,
+}: {
+  installedObjects: KibanaAssetReference[];
+  spaceId?: string;
+  packageInfo: RegistryPackage | ArchivePackage;
+}) {
   const savedObjectsClient = new SavedObjectsClient(
     appContextService.getSavedObjects().createInternalRepository()
   );
+
   const namespace = SavedObjectsUtils.namespaceStringToId(spaceId);
-  const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve(
-    installedObjects,
-    { namespace }
-  );
 
-  for (const { saved_object: savedObject } of resolvedObjects) {
-    auditLoggingService.writeCustomSoAuditLog({
-      action: 'get',
-      id: savedObject.id,
-      savedObjectType: savedObject.type,
-    });
-  }
+  const minKibana = packageInfo.conditions?.kibana?.version
+    ? minVersion(packageInfo.conditions.kibana.version)
+    : null;
 
-  const foundObjects = resolvedObjects.filter(
-    ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404
-  );
+  // Compare Kibana versions to determine if the package could been installed
+  // only in 8.x or later. If so, we can skip SO resolution step altogether
+  // and delete the assets directly. Otherwise, we need to resolve the assets
+  // which might create high memory pressure if a package has a lot of assets.
+  if (minKibana && minKibana.major >= 8) {
+    await bulkDeleteSavedObjects(installedObjects, namespace, savedObjectsClient);
+  } else {
+    const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve(
+      installedObjects,
+      { namespace }
+    );
 
-  // in the case of a partial install, it is expected that some assets will be not found
-  // we filter these out before calling delete
-  const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type }));
+    for (const { saved_object: savedObject } of resolvedObjects) {
+      auditLoggingService.writeCustomSoAuditLog({
+        action: 'get',
+        id: savedObject.id,
+        savedObjectType: savedObject.type,
+      });
+    }
 
-  for (const asset of assetsToDelete) {
-    auditLoggingService.writeCustomSoAuditLog({
-      action: 'delete',
-      id: asset.id,
-      savedObjectType: asset.type,
-    });
-  }
+    const foundObjects = resolvedObjects.filter(
+      ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404
+    );
 
-  return savedObjectsClient.bulkDelete(assetsToDelete, { namespace });
+    // in the case of a partial install, it is expected that some assets will be not found
+    // we filter these out before calling delete
+    const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type }));
+
+    await bulkDeleteSavedObjects(assetsToDelete, namespace, savedObjectsClient);
+  }
 }
 
-async function deleteKibanaAssets(
-  assetsToDelete: KibanaAssetReference[],
-  spaceId: string = DEFAULT_SPACE_ID
+async function bulkDeleteSavedObjects(
+  assetsToDelete: Array<{ id: string; type: string }>,
+  namespace: string | undefined,
+  savedObjectsClient: SavedObjectsClientContract
 ) {
-  const savedObjectsClient = new SavedObjectsClient(
-    appContextService.getSavedObjects().createInternalRepository()
-  );
-  const namespace = SavedObjectsUtils.namespaceStringToId(spaceId);
-
   for (const asset of assetsToDelete) {
     auditLoggingService.writeCustomSoAuditLog({
       action: 'delete',
@@ -178,7 +193,14 @@ async function deleteKibanaAssets(
     });
   }
 
-  return savedObjectsClient.bulkDelete(assetsToDelete, { namespace });
+  // Delete assets in chunks to avoid high memory pressure. This is mostly
+  // relevant for packages containing many assets, as large payload and response
+  // objects are created in memory during the delete operation. While chunking
+  // may work slower, it allows garbage collection to clean up memory between
+  // requests.
+  for (const assetsChunk of chunk(assetsToDelete, MAX_ASSETS_TO_DELETE)) {
+    await savedObjectsClient.bulkDelete(assetsChunk, { namespace });
+  }
 }
 
 function deleteESAssets(
@@ -211,6 +233,8 @@ async function deleteAssets(
     installed_kibana: installedKibana,
     installed_kibana_space_id: spaceId = DEFAULT_SPACE_ID,
     additional_spaces_installed_kibana: installedInAdditionalSpacesKibana = {},
+    name,
+    version,
   }: Installation,
   savedObjectsClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient
@@ -263,12 +287,18 @@ async function deleteAssets(
 
     // then delete index templates and pipelines
     await Promise.all(deleteESAssets(indexTemplatesAndPipelines, esClient));
+
+    const packageInfo = await Registry.fetchInfo(name, version);
     // then the other asset types
     await Promise.all([
       ...deleteESAssets(otherAssets, esClient),
-      resolveAndDeleteKibanaAssets(installedKibana, spaceId),
+      deleteKibanaAssets({ installedObjects: installedKibana, spaceId, packageInfo }),
       Object.entries(installedInAdditionalSpacesKibana).map(([additionalSpaceId, kibanaAssets]) =>
-        resolveAndDeleteKibanaAssets(kibanaAssets, additionalSpaceId)
+        deleteKibanaAssets({
+          installedObjects: kibanaAssets,
+          spaceId: additionalSpaceId,
+          packageInfo,
+        })
       ),
     ]);
   } catch (err) {
@@ -328,25 +358,17 @@ export async function deleteKibanaSavedObjectsAssets({
     .filter(({ type }) => kibanaSavedObjectTypes.includes(type))
     .map(({ id, type }) => ({ id, type } as KibanaAssetReference));
 
-  const registryInfo = await Registry.fetchInfo(
-    installedPkg.attributes.name,
-    installedPkg.attributes.version
-  );
-
-  const minKibana = registryInfo.conditions?.kibana?.version
-    ? minVersion(registryInfo.conditions.kibana.version)
-    : null;
-
   try {
-    // Compare Kibana versions to determine if the package could been installed
-    // only in 8.x or later. If so, we can skip SO resolution step altogether
-    // and delete the assets directly. Otherwise, we need to resolve the assets
-    // which might create high memory pressure if a package has a lot of assets.
-    if (minKibana && minKibana.major >= 8) {
-      await deleteKibanaAssets(assetsToDelete, spaceIdToDelete);
-    } else {
-      await resolveAndDeleteKibanaAssets(assetsToDelete, spaceIdToDelete);
-    }
+    const packageInfo = await Registry.fetchInfo(
+      installedPkg.attributes.name,
+      installedPkg.attributes.version
+    );
+
+    await deleteKibanaAssets({
+      installedObjects: assetsToDelete,
+      spaceId: spaceIdToDelete,
+      packageInfo,
+    });
   } catch (err) {
     // in the rollback case, partial installs are likely, so missing assets are not an error
     if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {