diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 94753297ce85f..9404a7aab18c0 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -521,6 +521,8 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts diff --git a/.buildkite/pipelines/security_solution/api_integration.yml b/.buildkite/pipelines/security_solution/api_integration.yml index 5b3394c05ffc7..b0c7832b967d5 100644 --- a/.buildkite/pipelines/security_solution/api_integration.yml +++ b/.buildkite/pipelines/security_solution/api_integration.yml @@ -146,6 +146,17 @@ steps: - exit_status: "1" limit: 2 + - label: Running genai:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh genai:qa:serverless + key: genai:qa:serverless + agents: + queue: n2-4-spot + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: "1" + limit: 2 + - label: Running prebuilt_rules_management:qa:serverless command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_management:qa:serverless key: prebuilt_rules_management:qa:serverless diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f45cb00fa1bbd..dcfce13388c7a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1501,6 +1501,9 @@ x-pack/plugins/security_solution/public/flyout/entity_details @elastic/security- x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-entity-analytics x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score @elastic/security-entity-analytics +## Security Solution sub teams - GenAI +x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai + # Security Defend Workflows - OSQuery Ownership /x-pack/plugins/security_solution/common/api/detection_engine/model/rule_response_actions @elastic/security-defend-workflows /x-pack/plugins/security_solution/public/detection_engine/rule_response_actions @elastic/security-defend-workflows diff --git a/x-pack/test/security_solution_api_integration/package.json b/x-pack/test/security_solution_api_integration/package.json index 1e2b5e4c7a509..fd8751b8c1820 100644 --- a/x-pack/test/security_solution_api_integration/package.json +++ b/x-pack/test/security_solution_api_integration/package.json @@ -5,6 +5,9 @@ "private": true, "license": "Elastic License 2.0", "scripts": { + "initialize-server:genai": "node ./scripts/index.js server genai trial_license_complete_tier", + "run-tests:genai": "node ./scripts/index.js runner genai trial_license_complete_tier", + "initialize-server:ea": "node ./scripts/index.js server entity_analytics trial_license_complete_tier", "run-tests:ea": "node ./scripts/index.js runner entity_analytics trial_license_complete_tier", @@ -26,6 +29,12 @@ "initialize-server:lists:complete": "node ./scripts/index.js server lists_and_exception_lists trial_license_complete_tier", "run-tests:lists:complete": "node ./scripts/index.js runner lists_and_exception_lists trial_license_complete_tier", + "genai:server:serverless": "npm run initialize-server:genai invoke_ai serverless", + "genai:runner:serverless": "npm run run-tests:genai invoke_ai serverless serverlessEnv", + "genai:qa:serverless": "npm run run-tests:genai invoke_ai serverless qaEnv", + "genai:server:ess": "npm run initialize-server:genai invoke_ai ess", + "genai:runner:ess": "npm run run-tests:genai invoke_ai ess essEnv", + "entity_analytics:server:serverless": "npm run initialize-server:ea risk_engine serverless", "entity_analytics:runner:serverless": "npm run run-tests:ea risk_engine serverless serverlessEnv", "entity_analytics:qa:serverless": "npm run run-tests:ea risk_engine serverless qaEnv", diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/basic.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/basic.ts new file mode 100644 index 0000000000000..bcac95217c280 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/basic.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { BedrockSimulator } from '@kbn/actions-simulators-plugin/server/bedrock_simulation'; +import { OpenAISimulator } from '@kbn/actions-simulators-plugin/server/openai_simulation'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; +import { postActionsClientExecute } from '../utils/post_actions_client_execute'; +import { ObjectRemover } from '../utils/object_remover'; +import { createConnector } from '../utils/create_connector'; + +const mockRequest = { + params: { + subActionParams: { + messages: [ + { role: 'user', content: '\\n\\n\\n\\nWhat is my name?' }, + { + role: 'assistant', + content: + "I'm sorry, but I don't have the information about your name. You can tell me your name if you'd like, and we can continue our conversation from there.", + }, + { role: 'user', content: '\\n\\nMy name is Andrew' }, + { + role: 'assistant', + content: "Hello, Andrew! It's nice to meet you. What would you like to talk about today?", + }, + { role: 'user', content: '\\n\\nDo you know my name?' }, + ], + }, + subAction: 'invokeAI', + }, + isEnabledKnowledgeBase: false, + isEnabledRAGAlerts: false, + llmType: 'bedrock', +}; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const objectRemover = new ObjectRemover(supertest); + const configService = getService('config'); + + // @skipInQA tag because the simulators do not work in the QA env + describe('@ess @serverless @skipInQA Basic Security AI Assistant Invoke AI [non-streaming, non-LangChain]', async () => { + after(() => { + objectRemover.removeAll(); + }); + + describe('With Bedrock connector', () => { + const simulator = new BedrockSimulator({ + proxy: { + config: configService.get('kbnTestServer.serverArgs'), + }, + }); + let apiUrl: string; + let bedrockActionId: string; + + before(async () => { + apiUrl = await simulator.start(); + bedrockActionId = await createConnector(supertest, objectRemover, apiUrl, 'bedrock'); + }); + + after(() => { + simulator.close(); + }); + it('should execute a chat completion', async () => { + const response = await postActionsClientExecute(bedrockActionId, mockRequest, supertest); + + const expected = { + connector_id: bedrockActionId, + data: 'Hello there! How may I assist you today?', + status: 'ok', + }; + + expect(response.body).to.eql(expected); + }); + }); + + describe('With OpenAI connector', () => { + const simulator = new OpenAISimulator({ + returnError: false, + proxy: { + config: configService.get('kbnTestServer.serverArgs'), + }, + }); + let apiUrl: string; + let openaiActionId: string; + + before(async () => { + apiUrl = await simulator.start(); + openaiActionId = await createConnector(supertest, objectRemover, apiUrl, 'openai'); + }); + + after(() => { + simulator.close(); + }); + it('should execute a chat completion', async () => { + const response = await postActionsClientExecute( + openaiActionId, + { ...mockRequest, llmType: 'openai' }, + supertest + ); + + const expected = { + connector_id: openaiActionId, + data: 'Hello there! How may I assist you today?', + status: 'ok', + }; + + expect(response.body).to.eql(expected); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts new file mode 100644 index 0000000000000..722b39a700026 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import getPort from 'get-port'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../config/ess/config.base.trial') + ); + + const proxyPort = await getPort({ port: getPort.makeRange(6200, 6299) }); + + return { + ...functionalConfig.getAll(), + kbnTestServer: { + ...functionalConfig.get('kbnTestServer'), + serverArgs: [ + ...functionalConfig.get('kbnTestServer.serverArgs'), + // used for connector simulators + `--xpack.actions.proxyUrl=http://localhost:${proxyPort}`, + `--xpack.actions.enabledActionTypes=${JSON.stringify(['.bedrock', '.gen-ai'])}`, + ], + }, + testFiles: [require.resolve('..')], + junit: { + reportName: 'GenAI - Invoke AI Tests - ESS Env - Trial License', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts new file mode 100644 index 0000000000000..d0666e224e4bf --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../../config/serverless/config.base'; + +export default createTestConfig({ + kbnTestServerArgs: [ + // used for connector simulators + `--xpack.actions.proxyUrl=http://localhost:6200`, + ], + testFiles: [require.resolve('..')], + junit: { + reportName: 'GenAI - Invoke AI Tests - Serverless Env - Complete Tier', + }, +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/index.ts new file mode 100644 index 0000000000000..e6ff5bbbfe667 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + // this is the test suite for the inaptly named post_actions_connector_execute route + describe('GenAI - Invoke AI', function () { + loadTestFile(require.resolve('./basic')); + }); +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/create_connector.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/create_connector.ts new file mode 100644 index 0000000000000..ac2d60ad631e2 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/create_connector.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; +import { getUrlPrefix } from './space_test_utils'; +import { ObjectRemover } from './object_remover'; + +const connectorSetup = { + bedrock: { + connectorTypeId: '.bedrock', + name: 'A bedrock action', + secrets: { + accessKey: 'bedrockAccessKey', + secret: 'bedrockSecret', + }, + config: { + defaultModel: 'anthropic.claude-v2', + }, + }, + openai: { + connectorTypeId: '.gen-ai', + name: 'An openai action', + secrets: { + apiKey: 'genAiApiKey', + }, + config: { + apiProvider: 'OpenAI', + }, + }, +}; + +/** + * Creates a connector + * @param supertest The supertest agent. + * @param apiUrl The url of the api + * @param connectorType The type of connector to create + * @param spaceId The space id + */ +export const createConnector = async ( + supertest: SuperTest.SuperTest, + objectRemover: ObjectRemover, + apiUrl: string, + connectorType: 'bedrock' | 'openai', + spaceId?: string +) => { + const { connectorTypeId, config, name, secrets } = connectorSetup[connectorType]; + const result = await supertest + .post(`${getUrlPrefix(spaceId ?? 'default')}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send({ + name, + connector_type_id: connectorTypeId, + config: { ...config, apiUrl }, + secrets, + }) + .expect(200); + + const { body } = result; + + objectRemover.add(spaceId ?? 'default', body.id, 'connector', 'actions'); + + return body.id; +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/object_remover.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/object_remover.ts new file mode 100644 index 0000000000000..93eaa85e802fc --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/object_remover.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; +import { getUrlPrefix } from './space_test_utils'; + +interface ObjectToRemove { + spaceId: string; + id: string; + type: string; + plugin: string; + isInternal?: boolean; +} + +export class ObjectRemover { + private readonly supertest: any; + private objectsToRemove: ObjectToRemove[] = []; + + constructor(supertest: any) { + this.supertest = supertest; + } + + add( + spaceId: ObjectToRemove['spaceId'], + id: ObjectToRemove['id'], + type: ObjectToRemove['type'], + plugin: ObjectToRemove['plugin'], + isInternal?: ObjectToRemove['isInternal'] + ) { + this.objectsToRemove.push({ spaceId, id, type, plugin, isInternal }); + } + + async removeAll() { + await Promise.all( + this.objectsToRemove.map(({ spaceId, id, type, plugin, isInternal }) => { + return this.supertest + .delete( + `${getUrlPrefix(spaceId)}/${isInternal ? 'internal' : 'api'}/${plugin}/${type}/${id}` + ) + .set('kbn-xsrf', 'foo') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .expect(plugin === 'saved_objects' ? 200 : 204); + }) + ); + this.objectsToRemove = []; + } +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/post_actions_client_execute.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/post_actions_client_execute.ts new file mode 100644 index 0000000000000..0477401a26533 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/post_actions_client_execute.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; +import { Response } from 'superagent'; + +/** + * Executes an invoke AI action + * @param connectorId The connector id + * @param args The arguments to pass to the action + * @param supertest The supertest agent + */ +export const postActionsClientExecute = async ( + connectorId: string, + args: any, + supertest: SuperTest.SuperTest +): Promise => { + const response = await supertest + .post(`/internal/elastic_assistant/actions/connector/${connectorId}/_execute`) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(args); + + return response; +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/space_test_utils.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/space_test_utils.ts new file mode 100644 index 0000000000000..6ffbdc492aee4 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/space_test_utils.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getUrlPrefix(spaceId: string) { + return spaceId && spaceId !== 'default' ? `/s/${spaceId}` : ``; +}