[Agent Builder] add basic smoke tests

.buildkite/ftr_platform_stateful_configs.yml

@@ -48,6 +48,7 @@ disabled:
 
   # Gen AI suites, running with their own pipeline
   - x-pack/platform/test/functional_gen_ai/inference/config.ts
+  - x-pack/platform/test/onechat/smoke_tests/config.stateful.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml

@@ -0,0 +1,56 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-agent-builder-smoke-tests-daily
+  description: Runs the Agent Builder smoke tests daily
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-agent-builder-smoke-tests-daily'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:workchat-eng'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / agent-builder / smoke tests / daily
+      description: Runs the Agent Builder smoke tests daily
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#agent-builder-eng'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      cancel_intermediate_builds: true
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/agent_builder/smoke_tests.yml
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        workchat-eng:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-ai-infra:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 0 * * * America/New_York
+          message: Daily build
+          branch: main
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/locations.yml

@@ -7,6 +7,7 @@ metadata:
 spec:
   type: url
   targets:
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-agent-builder-smoke-tests-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-apis-capacity-testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml

.buildkite/pipelines/agent_builder/smoke_tests.yml

@@ -0,0 +1,46 @@
+env:
+  FTR_GEN_AI: '1'
+steps:
+  - label: '👨‍🔧 Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - label: '🧑‍🏭 Build Kibana Distribution'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts'
+      FTR_CONFIG_GROUP_KEY: 'ftr-agent-builder-smoke-tests'
+      FTR_GEN_AI: '1'
+    label: Agent Builder API Smoke Tests
+    key: ftr-agent-builder-smoke-tests
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml

@@ -0,0 +1,30 @@
+steps:
+  - group: Agent Builder Smoke Tests
+    key: agent-builder-smoke-tests
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+      - check_oas_snapshot
+    steps:
+      - command: .buildkite/scripts/steps/test/ftr_configs.sh
+        env:
+          FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts'
+          FTR_CONFIG_GROUP_KEY: 'ftr-agent-builder-smoke-tests'
+          FTR_GEN_AI: '1'
+        label: Agent Builder API Smoke Tests
+        key: ftr-agent-builder-smoke-tests
+        timeout_in_minutes: 50
+        parallelism: 1
+        agents:
+          machineType: n2-standard-4
+          preemptible: true
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+            - exit_status: '*'
+              limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -132,20 +132,37 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml'));
     }
 
+    const aiInfraPaths = [
+      /^x-pack\/platform\/packages\/shared\/ai-infra/,
+      /^x-pack\/platform\/plugins\/shared\/ai_infra/,
+      /^x-pack\/platform\/plugins\/shared\/inference/,
+    ];
+    const aiConnectorPaths = [
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/inference/,
+    ];
+    const agentBuilderPaths = [
+      /^x-pack\/platform\/plugins\/shared\/onechat/,
+      /^x-pack\/platform\/packages\/shared\/onechat/,
+    ];
+
     if (
-      (await doAnyChangesMatch([
-        /^x-pack\/platform\/packages\/shared\/ai-infra/,
-        /^x-pack\/platform\/plugins\/shared\/ai_infra/,
-        /^x-pack\/platform\/plugins\/shared\/inference/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/,
-      ])) ||
+      (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths])) ||
       GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ai_infra_gen_ai.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths, ...agentBuilderPaths])) ||
+      GITHUB_PR_LABELS.includes('agent-builder:run-smoke-tests') ||
+      GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml'));
+    }
+
     if (
       GITHUB_PR_LABELS.includes('ci:build-cloud-image') &&
       !GITHUB_PR_LABELS.includes('ci:deploy-cloud') &&

x-pack/platform/test/onechat/smoke_tests/config.stateful.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getPreconfiguredConnectorConfig } from '@kbn/gen-ai-functional-testing';
+import type { FtrConfigProviderContext } from '@kbn/test';
+import { createStatefulTestConfig } from '../../api_integration_deployment_agnostic/default_configs/stateful.config.base';
+import { oneChatApiServices } from '../services/api';
+
+// eslint-disable-next-line import/no-default-export
+export default async function (ftrContext: FtrConfigProviderContext) {
+  const preconfiguredConnectors = getPreconfiguredConnectorConfig();
+
+  return createStatefulTestConfig({
+    services: oneChatApiServices,
+    testFiles: [require.resolve('./tests')],
+    junit: {
+      reportName: 'Agent Builder - Smoke Tests - API Integration',
+    },
+    // @ts-expect-error
+    kbnTestServer: {
+      serverArgs: [
+        '--uiSettings.overrides.agentBuilder:enabled=true',
+        `--xpack.actions.preconfigured=${JSON.stringify(preconfiguredConnectors)}`,
+      ],
+    },
+  })(ftrContext);
+}

x-pack/platform/test/onechat/smoke_tests/ftr_provider_context.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { GenericFtrProviderContext } from '@kbn/test';
+import { oneChatApiServices as services } from '../services/api';
+import { pageObjects } from '../../functional/page_objects';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
+
+export { services, pageObjects };

x-pack/platform/test/onechat/smoke_tests/tests/converse.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import type { AvailableConnectorWithId } from '@kbn/gen-ai-functional-testing';
+import { isToolCallStep, platformCoreTools } from '@kbn/onechat-common';
+import type {
+  ChatRequestBodyPayload,
+  ChatResponse,
+} from '@kbn/onechat-plugin/common/http_api/chat';
+import type { FtrProviderContext } from '../ftr_provider_context';
+
+export const converseApiSuite = (
+  { id: connectorId }: AvailableConnectorWithId,
+  { getService }: FtrProviderContext
+) => {
+  const supertest = getService('supertest');
+
+  const converse = async <T = ChatResponse>(
+    payload: ChatRequestBodyPayload,
+    statusCode = 200
+  ): Promise<T> => {
+    const res = await supertest
+      .post('/api/agent_builder/converse')
+      .set('kbn-xsrf', 'true')
+      .send(payload)
+      .expect(statusCode);
+    return res.body as T;
+  };
+
+  describe('Converse API', () => {
+    describe('sync', () => {
+      it('returns an answer for a simple message', async () => {
+        const response = await converse({
+          input: 'Hello',
+          connector_id: connectorId,
+        });
+
+        expect(response.response.message.length).to.be.greaterThan(0);
+      });
+
+      it('can execute a tool', async () => {
+        const response = await converse({
+          input: `Using the "platform_core_list_indices" tool, please list my indices. Only call the tool once.`,
+          connector_id: connectorId,
+        });
+
+        expect(response.response.message.length).to.be.greaterThan(0);
+
+        const toolCalls = response.steps.filter(isToolCallStep);
+        expect(toolCalls.length).to.eql(1);
+
+        const toolCall = toolCalls[0];
+        expect(toolCall.tool_id).to.eql(platformCoreTools.listIndices);
+      });
+
+      it('can continue a text conversation', async () => {
+        const response1 = await converse({
+          input: 'Please say "hello"',
+          connector_id: connectorId,
+        });
+
+        expect(response1.response.message.length).to.be.greaterThan(0);
+
+        const response2 = await converse({
+          conversation_id: response1.conversation_id,
+          input: 'Please say it again.',
+          connector_id: connectorId,
+        });
+
+        expect(response2.response.message.length).to.be.greaterThan(0);
+      });
+    });
+  });
+};

x-pack/platform/test/onechat/smoke_tests/tests/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getAvailableConnectors } from '@kbn/gen-ai-functional-testing';
+import type { FtrProviderContext } from '../ftr_provider_context';
+import { converseApiSuite } from './converse';
+
+// eslint-disable-next-line import/no-default-export
+export default function (providerContext: FtrProviderContext) {
+  describe('Agent Builder - LLM Smoke tests', async () => {
+    const connectors = getAvailableConnectors();
+
+    connectors.forEach((connector) => {
+      describe(`Connector "${connector.id}"`, () => {
+        converseApiSuite(connector, providerContext);
+      });
+    });
+  });
+}