diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index dbcf0b92b2409..b9f9fcdb399be 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -48,6 +48,7 @@ disabled: # Gen AI suites, running with their own pipeline - x-pack/platform/test/functional_gen_ai/inference/config.ts + - x-pack/platform/test/onechat/smoke_tests/config.stateful.ts defaultQueue: 'n2-4-spot' enabled: diff --git a/.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml b/.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml new file mode 100644 index 0000000000000..958f7878b00b3 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml @@ -0,0 +1,56 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-agent-builder-smoke-tests-daily + description: Runs the Agent Builder smoke tests daily + links: + - url: 'https://buildkite.com/elastic/kibana-agent-builder-smoke-tests-daily' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:workchat-eng' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / agent-builder / smoke tests / daily + description: Runs the Agent Builder smoke tests daily + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#agent-builder-eng' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main + cancel_intermediate_builds: true + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/agent_builder/smoke_tests.yml + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + workchat-eng: + access_level: MANAGE_BUILD_AND_READ + appex-ai-infra: + access_level: MANAGE_BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build: + cronline: 0 0 * * * America/New_York + message: Daily build + branch: main + tags: + - kibana diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index 8a20ee1059f1b..8f146d639296f 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -7,6 +7,7 @@ metadata: spec: type: url targets: + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-apis-capacity-testing-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml diff --git a/.buildkite/pipelines/agent_builder/smoke_tests.yml b/.buildkite/pipelines/agent_builder/smoke_tests.yml new file mode 100644 index 0000000000000..55f25c098aa52 --- /dev/null +++ b/.buildkite/pipelines/agent_builder/smoke_tests.yml @@ -0,0 +1,46 @@ +env: + FTR_GEN_AI: '1' +steps: + - label: '👨‍🔧 Pre-Build' + command: .buildkite/scripts/lifecycle/pre_build.sh + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + + - wait + + - label: '🧑‍🏭 Build Kibana Distribution' + command: .buildkite/scripts/steps/build_kibana.sh + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + key: build + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + + - wait + + - command: .buildkite/scripts/steps/test/ftr_configs.sh + env: + FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts' + FTR_CONFIG_GROUP_KEY: 'ftr-agent-builder-smoke-tests' + FTR_GEN_AI: '1' + label: Agent Builder API Smoke Tests + key: ftr-agent-builder-smoke-tests + timeout_in_minutes: 50 + parallelism: 1 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 diff --git a/.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml b/.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml new file mode 100644 index 0000000000000..e93ad229ed8d8 --- /dev/null +++ b/.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml @@ -0,0 +1,30 @@ +steps: + - group: Agent Builder Smoke Tests + key: agent-builder-smoke-tests + depends_on: + - build + - quick_checks + - checks + - linting + - linting_with_types + - check_types + - check_oas_snapshot + steps: + - command: .buildkite/scripts/steps/test/ftr_configs.sh + env: + FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts' + FTR_CONFIG_GROUP_KEY: 'ftr-agent-builder-smoke-tests' + FTR_GEN_AI: '1' + label: Agent Builder API Smoke Tests + key: ftr-agent-builder-smoke-tests + timeout_in_minutes: 50 + parallelism: 1 + agents: + machineType: n2-standard-4 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts index 7ac3ae69f9c9f..295ccb1810fed 100644 --- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts +++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts @@ -132,20 +132,37 @@ const getPipeline = (filename: string, removeSteps = true) => { pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml')); } + const aiInfraPaths = [ + /^x-pack\/platform\/packages\/shared\/ai-infra/, + /^x-pack\/platform\/plugins\/shared\/ai_infra/, + /^x-pack\/platform\/plugins\/shared\/inference/, + ]; + const aiConnectorPaths = [ + /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/, + /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/, + /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/, + /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/inference/, + ]; + const agentBuilderPaths = [ + /^x-pack\/platform\/plugins\/shared\/onechat/, + /^x-pack\/platform\/packages\/shared\/onechat/, + ]; + if ( - (await doAnyChangesMatch([ - /^x-pack\/platform\/packages\/shared\/ai-infra/, - /^x-pack\/platform\/plugins\/shared\/ai_infra/, - /^x-pack\/platform\/plugins\/shared\/inference/, - /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/, - /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/, - /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/, - ])) || + (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths])) || GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites') ) { pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ai_infra_gen_ai.yml')); } + if ( + (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths, ...agentBuilderPaths])) || + GITHUB_PR_LABELS.includes('agent-builder:run-smoke-tests') || + GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites') + ) { + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml')); + } + if ( GITHUB_PR_LABELS.includes('ci:build-cloud-image') && !GITHUB_PR_LABELS.includes('ci:deploy-cloud') && diff --git a/x-pack/platform/test/onechat/smoke_tests/config.stateful.ts b/x-pack/platform/test/onechat/smoke_tests/config.stateful.ts new file mode 100644 index 0000000000000..ef538b2250297 --- /dev/null +++ b/x-pack/platform/test/onechat/smoke_tests/config.stateful.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getPreconfiguredConnectorConfig } from '@kbn/gen-ai-functional-testing'; +import type { FtrConfigProviderContext } from '@kbn/test'; +import { createStatefulTestConfig } from '../../api_integration_deployment_agnostic/default_configs/stateful.config.base'; +import { oneChatApiServices } from '../services/api'; + +// eslint-disable-next-line import/no-default-export +export default async function (ftrContext: FtrConfigProviderContext) { + const preconfiguredConnectors = getPreconfiguredConnectorConfig(); + + return createStatefulTestConfig({ + services: oneChatApiServices, + testFiles: [require.resolve('./tests')], + junit: { + reportName: 'Agent Builder - Smoke Tests - API Integration', + }, + // @ts-expect-error + kbnTestServer: { + serverArgs: [ + '--uiSettings.overrides.agentBuilder:enabled=true', + `--xpack.actions.preconfigured=${JSON.stringify(preconfiguredConnectors)}`, + ], + }, + })(ftrContext); +} diff --git a/x-pack/platform/test/onechat/smoke_tests/ftr_provider_context.ts b/x-pack/platform/test/onechat/smoke_tests/ftr_provider_context.ts new file mode 100644 index 0000000000000..72d6af3477b8b --- /dev/null +++ b/x-pack/platform/test/onechat/smoke_tests/ftr_provider_context.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { GenericFtrProviderContext } from '@kbn/test'; +import { oneChatApiServices as services } from '../services/api'; +import { pageObjects } from '../../functional/page_objects'; + +export type FtrProviderContext = GenericFtrProviderContext; + +export { services, pageObjects }; diff --git a/x-pack/platform/test/onechat/smoke_tests/tests/converse.ts b/x-pack/platform/test/onechat/smoke_tests/tests/converse.ts new file mode 100644 index 0000000000000..1a5938c007a56 --- /dev/null +++ b/x-pack/platform/test/onechat/smoke_tests/tests/converse.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { AvailableConnectorWithId } from '@kbn/gen-ai-functional-testing'; +import { isToolCallStep, platformCoreTools } from '@kbn/onechat-common'; +import type { + ChatRequestBodyPayload, + ChatResponse, +} from '@kbn/onechat-plugin/common/http_api/chat'; +import type { FtrProviderContext } from '../ftr_provider_context'; + +export const converseApiSuite = ( + { id: connectorId }: AvailableConnectorWithId, + { getService }: FtrProviderContext +) => { + const supertest = getService('supertest'); + + const converse = async ( + payload: ChatRequestBodyPayload, + statusCode = 200 + ): Promise => { + const res = await supertest + .post('/api/agent_builder/converse') + .set('kbn-xsrf', 'true') + .send(payload) + .expect(statusCode); + return res.body as T; + }; + + describe('Converse API', () => { + describe('sync', () => { + it('returns an answer for a simple message', async () => { + const response = await converse({ + input: 'Hello', + connector_id: connectorId, + }); + + expect(response.response.message.length).to.be.greaterThan(0); + }); + + it('can execute a tool', async () => { + const response = await converse({ + input: `Using the "platform_core_list_indices" tool, please list my indices. Only call the tool once.`, + connector_id: connectorId, + }); + + expect(response.response.message.length).to.be.greaterThan(0); + + const toolCalls = response.steps.filter(isToolCallStep); + expect(toolCalls.length).to.eql(1); + + const toolCall = toolCalls[0]; + expect(toolCall.tool_id).to.eql(platformCoreTools.listIndices); + }); + + it('can continue a text conversation', async () => { + const response1 = await converse({ + input: 'Please say "hello"', + connector_id: connectorId, + }); + + expect(response1.response.message.length).to.be.greaterThan(0); + + const response2 = await converse({ + conversation_id: response1.conversation_id, + input: 'Please say it again.', + connector_id: connectorId, + }); + + expect(response2.response.message.length).to.be.greaterThan(0); + }); + }); + }); +}; diff --git a/x-pack/platform/test/onechat/smoke_tests/tests/index.ts b/x-pack/platform/test/onechat/smoke_tests/tests/index.ts new file mode 100644 index 0000000000000..8d9bd5db5ffc3 --- /dev/null +++ b/x-pack/platform/test/onechat/smoke_tests/tests/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getAvailableConnectors } from '@kbn/gen-ai-functional-testing'; +import type { FtrProviderContext } from '../ftr_provider_context'; +import { converseApiSuite } from './converse'; + +// eslint-disable-next-line import/no-default-export +export default function (providerContext: FtrProviderContext) { + describe('Agent Builder - LLM Smoke tests', async () => { + const connectors = getAvailableConnectors(); + + connectors.forEach((connector) => { + describe(`Connector "${connector.id}"`, () => { + converseApiSuite(connector, providerContext); + }); + }); + }); +}