[EDR Workflows][Artifact transfer 3] Import API validator

x-pack/solutions/security/packages/kbn-securitysolution-io-ts-list-types/src/response/import_exceptions_schema/index.ts

@@ -20,6 +20,8 @@ export const bulkErrorErrorSchema = t.exact(
   })
 );
 
+export type BulkErrorErrorSchema = t.TypeOf<typeof bulkErrorErrorSchema>;
+
 export const bulkErrorSchema = t.intersection([
   t.exact(
     t.type({

x-pack/solutions/security/plugins/lists/moon.yml

@@ -40,7 +40,6 @@ dependsOn:
   - '@kbn/core-http-server'
   - '@kbn/securitysolution-es-utils'
   - '@kbn/securitysolution-io-ts-types'
-  - '@kbn/std'
   - '@kbn/utils'
   - '@kbn/logging-mocks'
   - '@kbn/utility-types'

x-pack/solutions/security/plugins/lists/server/exception_item_import_error.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { BulkErrorErrorSchema } from '@kbn/securitysolution-io-ts-list-types';
+
+import { ListsErrorWithStatusCode } from '.';
+
+export class ExceptionItemImportError extends Error implements BulkErrorErrorSchema {
+  public readonly status_code: number;
+
+  constructor(error: Error, public readonly listId: string, public readonly itemId: string) {
+    super(error.message);
+    this.status_code = error instanceof ListsErrorWithStatusCode ? error.getStatusCode() : 400;
+  }
+}

x-pack/solutions/security/plugins/lists/server/services/exception_lists/bulk_delete_exception_list_items.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { NamespaceType } from '@kbn/securitysolution-io-ts-list-types';
+import { getSavedObjectType } from '@kbn/securitysolution-list-utils';
+import type { SavedObjectsBulkDeleteObject, SavedObjectsClientContract } from '@kbn/core/server';
+
+interface BulkDeleteExceptionListItemsOptions {
+  ids: string[];
+  namespaceType: NamespaceType;
+  savedObjectsClient: SavedObjectsClientContract;
+}
+
+export const bulkDeleteExceptionListItems = async ({
+  ids,
+  namespaceType,
+  savedObjectsClient,
+}: BulkDeleteExceptionListItemsOptions): Promise<void> => {
+  const savedObjectType = getSavedObjectType({ namespaceType });
+
+  const bulkDeleteObjects = ids.map<SavedObjectsBulkDeleteObject>((id) => ({
+    id,
+    type: savedObjectType,
+  }));
+
+  await savedObjectsClient.bulkDelete(bulkDeleteObjects);
+};

x-pack/solutions/security/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts

@@ -10,11 +10,10 @@ import type {
   ListId,
   NamespaceType,
 } from '@kbn/securitysolution-io-ts-list-types';
-import { getSavedObjectType } from '@kbn/securitysolution-list-utils';
-import { asyncForEach } from '@kbn/std';
 import type { SavedObjectsClientContract } from '@kbn/core/server';
 
 import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder';
+import { bulkDeleteExceptionListItems } from './bulk_delete_exception_list_items';
 
 interface DeleteExceptionListItemByListOptions {
   listId: ListId;
@@ -28,7 +27,7 @@ export const deleteExceptionListItemByList = async ({
   namespaceType,
 }: DeleteExceptionListItemByListOptions): Promise<void> => {
   const ids = await getExceptionListItemIds({ listId, namespaceType, savedObjectsClient });
-  await deleteFoundExceptionListItems({ ids, namespaceType, savedObjectsClient });
+  await bulkDeleteExceptionListItems({ ids, namespaceType, savedObjectsClient });
 };
 
 export const getExceptionListItemIds = async ({
@@ -56,30 +55,3 @@ export const getExceptionListItemIds = async ({
   });
   return ids;
 };
-
-/**
- * NOTE: This is slow and terrible as we are deleting everything one at a time.
- * TODO: Replace this with a bulk call or a delete by query would be more useful
- */
-export const deleteFoundExceptionListItems = async ({
-  ids,
-  savedObjectsClient,
-  namespaceType,
-}: {
-  ids: string[];
-  savedObjectsClient: SavedObjectsClientContract;
-  namespaceType: NamespaceType;
-}): Promise<void> => {
-  const savedObjectType = getSavedObjectType({ namespaceType });
-  await asyncForEach(ids, async (id) => {
-    try {
-      await savedObjectsClient.delete(savedObjectType, id);
-      // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    } catch (err) {
-      // This can happen from race conditions or networking issues so deleting the id's
-      // like this is considered "best effort" and it is possible to get dangling pieces
-      // of data sitting around in which case the user has to manually clean up the data
-      // I am very hopeful this does not happen often or at all.
-    }
-  });
-};

x-pack/solutions/security/plugins/lists/server/services/exception_lists/exception_list_client.test.ts

@@ -338,6 +338,18 @@ describe('exception_list_client', () => {
           return extensionPointStorageContext.exceptionPreDelete.callback;
         },
       ],
+      [
+        'bulkDeleteExceptionListItems',
+        (): ReturnType<ExceptionListClient['bulkDeleteExceptionListItems']> => {
+          return exceptionListClient.bulkDeleteExceptionListItems({
+            ids: ['1', '2', '3'],
+            namespaceType: 'agnostic',
+          });
+        },
+        (): ExtensionPointStorageContextMock['exceptionPreDelete']['callback'] => {
+          return extensionPointStorageContext.exceptionPreDelete.callback;
+        },
+      ],
       [
         'getEndpointListItem',
         (): ReturnType<ExceptionListClient['getEndpointListItem']> => {

x-pack/solutions/security/plugins/lists/server/services/exception_lists/exception_list_client.ts

@@ -33,6 +33,7 @@ import type {
 } from '../extension_points';
 
 import type {
+  BulkDeleteExceptionListItemsOptions,
   ClosePointInTimeOptions,
   ConstructorOptions,
   CreateEndpointListItemOptions,
@@ -100,6 +101,7 @@ import { findValueListExceptionListItemsPointInTimeFinder } from './find_value_l
 import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder';
 import { duplicateExceptionListAndItems } from './duplicate_exception_list';
 import { updateOverwriteExceptionListItem } from './update_overwrite_exception_list_item';
+import { bulkDeleteExceptionListItems } from './bulk_delete_exception_list_items';
 
 /**
  * Class for use for exceptions that are with trusted applications or
@@ -831,6 +833,33 @@ export class ExceptionListClient {
     });
   };
 
+  /**
+   * Bulk delete exception list items by an `id` array
+   * @param options
+   * @param options.ids the array of ids of exception list items to delete
+   * @param options.namespaceType saved object namespace (single | agnostic)
+   */
+  public bulkDeleteExceptionListItems = async ({
+    ids,
+    namespaceType,
+  }: BulkDeleteExceptionListItemsOptions): Promise<void> => {
+    const { savedObjectsClient } = this;
+
+    if (this.enableServerExtensionPoints) {
+      // todo: this is not ideal, items will be checked one-by-one. we'd need an `exceptionsListPreBulkDeleteItems`
+      // callback, but then that also needs a bulkGet function in exceptionListClient, which we don't have yet.
+      for (const id of ids) {
+        await this.serverExtensionsClient.pipeRun(
+          'exceptionsListPreDeleteItem',
+          { id, itemId: undefined, namespaceType },
+          this.getServerExtensionCallbackContext()
+        );
+      }
+    }
+
+    return bulkDeleteExceptionListItems({ ids, namespaceType, savedObjectsClient });
+  };
+
   /**
    * This is the same as "deleteExceptionListItem" except it applies specifically to the endpoint list.
    * Either id or itemId has to be defined to delete but not both is required. If both are provided, the id
@@ -1168,18 +1197,21 @@ export class ExceptionListClient {
       ...readStream,
     ]);
 
+    let shouldListApiPerformOverwrite = overwrite;
     if (this.enableServerExtensionPoints) {
-      await this.serverExtensionsClient.pipeRun(
+      const result = await this.serverExtensionsClient.pipeRun(
         'exceptionsListPreImport',
-        parsedObjects,
+        { data: parsedObjects, overwrite },
         this.getServerExtensionCallbackContext()
       );
+
+      shouldListApiPerformOverwrite = result.overwrite;
     }
 
     return importExceptions({
       exceptions: parsedObjects,
       generateNewListId,
-      overwrite,
+      overwrite: shouldListApiPerformOverwrite,
       savedObjectsClient,
       user,
     });
@@ -1203,18 +1235,21 @@ export class ExceptionListClient {
     // validation of import and sorting of lists and items
     const parsedObjects = exceptionsChecksFromArray(exceptionsToImport, maxExceptionsImportSize);
 
+    let shouldListApiPerformOverwrite = overwrite;
     if (this.enableServerExtensionPoints) {
-      await this.serverExtensionsClient.pipeRun(
+      const result = await this.serverExtensionsClient.pipeRun(
         'exceptionsListPreImport',
-        parsedObjects,
+        { data: parsedObjects, overwrite },
         this.getServerExtensionCallbackContext()
       );
+
+      shouldListApiPerformOverwrite = result.overwrite;
     }
 
     return importExceptions({
       exceptions: parsedObjects,
       generateNewListId: false,
-      overwrite,
+      overwrite: shouldListApiPerformOverwrite,
       savedObjectsClient,
       user,
     });

x-pack/solutions/security/plugins/lists/server/services/exception_lists/exception_list_client_types.ts

@@ -202,6 +202,18 @@ export interface DeleteExceptionListItemByIdOptions {
   namespaceType: NamespaceType;
 }
 
+/**
+ * ExceptionListClient.bulkDeleteExceptionListItems
+ * {@link ExceptionListClient.bulkDeleteExceptionListItems}
+ */
+export interface BulkDeleteExceptionListItemsOptions {
+  /** the "ids" of the exception list items */
+  ids: Id[];
+
+  /** saved object namespace (single | agnostic) */
+  namespaceType: NamespaceType;
+}
+
 /**
  * ExceptionListClient.deleteEndpointListItem
  * {@link ExceptionListClient.deleteEndpointListItem}

x-pack/solutions/security/plugins/lists/server/services/exception_lists/utils/import/dedupe_incoming_items.ts

@@ -6,11 +6,13 @@
  */
 
 import { v4 as uuidv4 } from 'uuid';
-import type {
-  BulkErrorSchema,
-  ImportExceptionListItemSchemaDecoded,
+import {
+  type BulkErrorSchema,
+  type ImportExceptionListItemSchemaDecoded,
 } from '@kbn/securitysolution-io-ts-list-types';
 
+import { ExceptionItemImportError } from '../../../../exception_item_import_error';
+
 /**
  * Reports on duplicates and returns unique array of items
  * @param items - exception items to be checked for duplicate list_ids
@@ -21,7 +23,14 @@ export const getTupleErrorsAndUniqueExceptionListItems = (
 ): [BulkErrorSchema[], ImportExceptionListItemSchemaDecoded[]] => {
   const { errors, itemsAcc } = items.reduce(
     (acc, parsedExceptionItem) => {
-      if (parsedExceptionItem instanceof Error) {
+      if (parsedExceptionItem instanceof ExceptionItemImportError) {
+        acc.errors.set(uuidv4(), {
+          error: parsedExceptionItem,
+
+          item_id: parsedExceptionItem.itemId,
+          list_id: parsedExceptionItem.listId,
+        });
+      } else if (parsedExceptionItem instanceof Error) {
         acc.errors.set(uuidv4(), {
           error: {
             message: `Error found importing exception list item: ${parsedExceptionItem.message}`,

x-pack/solutions/security/plugins/lists/server/services/extension_points/types.ts

@@ -75,7 +75,7 @@ interface ServerExtensionPointDefinition<
  */
 export type ExceptionsListPreImportServerExtension = ServerExtensionPointDefinition<
   'exceptionsListPreImport',
-  PromiseFromStreams
+  { data: PromiseFromStreams; overwrite: boolean }
 >;
 
 /**

x-pack/solutions/security/plugins/lists/tsconfig.json

@@ -34,7 +34,6 @@
     "@kbn/core-http-server",
     "@kbn/securitysolution-es-utils",
     "@kbn/securitysolution-io-ts-types",
-    "@kbn/std",
     "@kbn/utils",
     "@kbn/logging-mocks",
     "@kbn/utility-types",

x-pack/solutions/security/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts

@@ -13,6 +13,7 @@ import {
   ListOperatorTypeEnum,
   type ListOperatorType,
 } from '@kbn/securitysolution-io-ts-list-types';
+import type { ENDPOINT_ARTIFACT_LIST_IDS } from '@kbn/securitysolution-list-constants';
 import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import { ConditionEntryField } from '@kbn/securitysolution-utils';
 import { LIST_ITEM_ENTRY_OPERATOR_TYPES } from './common/artifact_list_item_entry_values';
@@ -453,4 +454,32 @@ export class ExceptionsListItemGenerator extends BaseDataGenerator<ExceptionList
       ...overrides,
     };
   }
+
+  generateEndpointArtifact = (
+    listId: (typeof ENDPOINT_ARTIFACT_LIST_IDS)[number],
+    overrides: Partial<ExceptionListItemSchema> = {}
+  ) => {
+    switch (listId) {
+      case ENDPOINT_ARTIFACT_LISTS.endpointExceptions.id:
+        return this.generateEndpointException(overrides);
+
+      case ENDPOINT_ARTIFACT_LISTS.blocklists.id:
+        return this.generateBlocklist(overrides);
+
+      case ENDPOINT_ARTIFACT_LISTS.eventFilters.id:
+        return this.generateEventFilter(overrides);
+
+      case ENDPOINT_ARTIFACT_LISTS.hostIsolationExceptions.id:
+        return this.generateHostIsolationException(overrides);
+
+      case ENDPOINT_ARTIFACT_LISTS.trustedApps.id:
+        return this.generateTrustedApp(overrides);
+
+      case ENDPOINT_ARTIFACT_LISTS.trustedDevices.id:
+        return this.generateTrustedDevice(overrides);
+
+      default:
+        throw new Error(`Unknown listId: ${listId}. Unable to generate exception list item.`);
+    }
+  };
 }

x-pack/solutions/security/plugins/security_solution/common/endpoint/service/artifacts/constants.ts

@@ -13,6 +13,8 @@ export const BY_POLICY_ARTIFACT_TAG_PREFIX = 'policy:';
 
 export const GLOBAL_ARTIFACT_TAG = `${BY_POLICY_ARTIFACT_TAG_PREFIX}all`;
 
+export const IMPORTED_ARTIFACT_TAG = 'imported_artifact';
+
 export const ADVANCED_MODE_TAG = 'form_mode:advanced';
 
 /** The tag name for process descendants in event filters */

x-pack/solutions/security/plugins/security_solution/common/endpoint/service/artifacts/index.ts

@@ -16,6 +16,7 @@ export {
 export {
   BY_POLICY_ARTIFACT_TAG_PREFIX,
   GLOBAL_ARTIFACT_TAG,
+  IMPORTED_ARTIFACT_TAG,
   ADVANCED_MODE_TAG,
   FILTER_PROCESS_DESCENDANTS_TAG,
   TRUSTED_PROCESS_DESCENDANTS_TAG,