[Agent Builder] add basic smoke tests

.buildkite/ftr_platform_stateful_configs.yml

@@ -48,6 +48,7 @@ disabled:
 
   # Gen AI suites, running with their own pipeline
   - x-pack/platform/test/functional_gen_ai/inference/config.ts
+  - x-pack/platform/test/onechat/smoke_tests/config.stateful.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

.buildkite/pipelines/agent_builder/smoke_tests.yml

@@ -0,0 +1,46 @@
+env:
+  FTR_GEN_AI: '1'
+steps:
+  - label: '👨‍🔧 Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - label: '🧑‍🏭 Build Kibana Distribution'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts'
+      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+      FTR_GEN_AI: '1'
+    label: Agent Builder API Smoke Tests
+    key: ftr-agent-builder-smoke-tests
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml

@@ -0,0 +1,30 @@
+steps:
+  - group: Agent Builder Smoke Tests
+    key: agent-builder-smoke-tests
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+      - check_oas_snapshot
+    steps:
+      - command: .buildkite/scripts/steps/test/ftr_configs.sh
+        env:
+          FTR_CONFIG: 'x-pack/platform/test/onechat/smoke_tests/config.stateful.ts'
+          FTR_CONFIG_GROUP_KEY: 'ftr-agent-builder-smoke-tests'
+          FTR_GEN_AI: '1'
+        label: Agent Builder API Smoke Tests
+        key: ftr-agent-builder-smoke-tests
+        timeout_in_minutes: 50
+        parallelism: 1
+        agents:
+          machineType: n2-standard-4
+          preemptible: true
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+            - exit_status: '*'
+              limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -132,20 +132,37 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml'));
     }
 
+    const aiInfraPaths = [
+      /^x-pack\/platform\/packages\/shared\/ai-infra/,
+      /^x-pack\/platform\/plugins\/shared\/ai_infra/,
+      /^x-pack\/platform\/plugins\/shared\/inference/,
+    ];
+    const aiConnectorPaths = [
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/,
+      /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/inference/,
+    ];
+    const agentBuilderPaths = [
+      /^x-pack\/platform\/plugins\/shared\/onechat/,
+      /^x-pack\/platform\/packages\/shared\/onechat/,
+    ];
+
     if (
-      (await doAnyChangesMatch([
-        /^x-pack\/platform\/packages\/shared\/ai-infra/,
-        /^x-pack\/platform\/plugins\/shared\/ai_infra/,
-        /^x-pack\/platform\/plugins\/shared\/inference/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/bedrock/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/gemini/,
-        /^x-pack\/platform\/plugins\/shared\/stack_connectors\/server\/connector_types\/openai/,
-      ])) ||
+      (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths])) ||
       GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ai_infra_gen_ai.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([...aiInfraPaths, ...aiConnectorPaths, ...agentBuilderPaths])) ||
+      GITHUB_PR_LABELS.includes('agent-builder:run-smoke-tests') ||
+      GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/agent_builder_smoke_tests.yml'));
+    }
+
     if (
       GITHUB_PR_LABELS.includes('ci:build-cloud-image') &&
       !GITHUB_PR_LABELS.includes('ci:deploy-cloud') &&

x-pack/platform/test/onechat/smoke_tests/config.stateful.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getPreconfiguredConnectorConfig } from '@kbn/gen-ai-functional-testing';
+import type { FtrConfigProviderContext } from '@kbn/test';
+import { createStatefulTestConfig } from '../../api_integration_deployment_agnostic/default_configs/stateful.config.base';
+import { oneChatApiServices } from '../services/api';
+
+// eslint-disable-next-line import/no-default-export
+export default async function (ftrContext: FtrConfigProviderContext) {
+  const preconfiguredConnectors = getPreconfiguredConnectorConfig();
+
+  return createStatefulTestConfig({
+    services: oneChatApiServices,
+    testFiles: [require.resolve('./tests')],
+    junit: {
+      reportName: 'Agent Builder - Smoke Tests - API Integration',
+    },
+    // @ts-expect-error
+    kbnTestServer: {
+      serverArgs: [
+        '--uiSettings.overrides.agentBuilder:enabled=true',
+        `--xpack.actions.preconfigured=${JSON.stringify(preconfiguredConnectors)}`,
+      ],
+    },
+  })(ftrContext);
+}

x-pack/platform/test/onechat/smoke_tests/ftr_provider_context.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { GenericFtrProviderContext } from '@kbn/test';
+import { oneChatApiServices as services } from '../services/api';
+import { pageObjects } from '../../functional/page_objects';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
+
+export { services, pageObjects };

x-pack/platform/test/onechat/smoke_tests/tests/converse.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import type { AvailableConnectorWithId } from '@kbn/gen-ai-functional-testing';
+import { isToolCallStep, platformCoreTools } from '@kbn/onechat-common';
+import type {
+  ChatRequestBodyPayload,
+  ChatResponse,
+} from '@kbn/onechat-plugin/common/http_api/chat';
+import type { FtrProviderContext } from '../ftr_provider_context';
+
+export const converseApiSuite = (
+  { id: connectorId }: AvailableConnectorWithId,
+  { getService }: FtrProviderContext
+) => {
+  const supertest = getService('supertest');
+
+  const converse = async <T = ChatResponse>(
+    payload: ChatRequestBodyPayload,
+    statusCode = 200
+  ): Promise<T> => {
+    const res = await supertest
+      .post('/api/agent_builder/converse')
+      .set('kbn-xsrf', 'true')
+      .send(payload)
+      .expect(statusCode);
+    return res.body as T;
+  };
+
+  describe('Converse API', () => {
+    describe('sync', () => {
+      it('returns a answer for a simple message', async () => {
+        const response = await converse({
+          input: 'Hello',
+          connector_id: connectorId,
+        });
+
+        expect(response.response.message.length).to.be.greaterThan(0);
+      });
+
+      it('can execute a tool', async () => {
+        const response = await converse({
+          input: 'Please list my indices',
+          connector_id: connectorId,
+        });
+
+        expect(response.response.message.length).to.be.greaterThan(0);
+
+        const toolCalls = response.steps.filter(isToolCallStep);
+        expect(toolCalls.length).to.eql(1);
+
+        const toolCall = toolCalls[0];
+        expect(toolCall.tool_id).to.eql(platformCoreTools.listIndices);
+      });
+    });
+  });
+};

x-pack/platform/test/onechat/smoke_tests/tests/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getAvailableConnectors } from '@kbn/gen-ai-functional-testing';
+import type { FtrProviderContext } from '../ftr_provider_context';
+import { converseApiSuite } from './converse';
+
+// eslint-disable-next-line import/no-default-export
+export default function (providerContext: FtrProviderContext) {
+  describe('Agent Builder - LLM Smoke tests', async () => {
+    const connectors = getAvailableConnectors();
+
+    connectors.forEach((connector) => {
+      describe(`Connector "${connector.id}"`, () => {
+        converseApiSuite(connector, providerContext);
+      });
+    });
+  });
+}