[Security Solution][Elastic AI Assistant] Adds Model Evaluation Tooling

x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx

@@ -13,6 +13,7 @@ import type { Conversation, Message } from '../assistant_context/types';
 import { API_ERROR } from './translations';
 import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector';
 import { getFormattedMessageContent } from './helpers';
+import { PerformEvaluationParams } from './settings/evaluation_settings/use_perform_evaluation';
 
 export interface FetchConnectorExecuteAction {
   assistantLangChain: boolean;
@@ -199,3 +200,57 @@ export const deleteKnowledgeBase = async ({
     return error as IHttpFetchError;
   }
 };
+
+export interface PostEvaluationParams {
+  http: HttpSetup;
+  evalParams?: PerformEvaluationParams;
+  signal?: AbortSignal | undefined;
+}
+
+export interface PostEvaluationResponse {
+  success: boolean;
+}
+
+/**
+ * API call for evaluating models.
+ *
+ * @param {Object} options - The options object.
+ * @param {HttpSetup} options.http - HttpSetup
+ * @param {string} [options.evalParams] - Params necessary for evaluation
+ * @param {AbortSignal} [options.signal] - AbortSignal
+ *
+ * @returns {Promise<PostEvaluationResponse | IHttpFetchError>}
+ */
+export const postEvaluation = async ({
+  http,
+  evalParams,
+  signal,
+}: PostEvaluationParams): Promise<PostEvaluationResponse | IHttpFetchError> => {
+  try {
+    const path = `/internal/elastic_assistant/evaluate`;
+    const query = {
+      models: evalParams?.models.sort()?.join(','),
+      agents: evalParams?.agents.sort()?.join(','),
+      evaluationType: evalParams?.evaluationType.sort()?.join(','),
+      evalModel: evalParams?.evalModel.sort()?.join(','),
+      outputIndex: evalParams?.outputIndex,
+    };
+
+    const response = await http.fetch(path, {
+      method: 'POST',
+      body: JSON.stringify({
+        dataset: JSON.parse(evalParams?.dataset ?? '[]'),
+        evalPrompt: evalParams?.evalPrompt ?? '',
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      query,
+      signal,
+    });
+
+    return response as PostEvaluationResponse;
+  } catch (error) {
+    return error as IHttpFetchError;
+  }
+};

x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx

@@ -34,6 +34,7 @@ import { AdvancedSettings } from './advanced_settings/advanced_settings';
 import { ConversationSettings } from '../conversations/conversation_settings/conversation_settings';
 import { TEST_IDS } from '../constants';
 import { useSettingsUpdater } from './use_settings_updater/use_settings_updater';
+import { EvaluationSettings } from './evaluation_settings/evaluation_settings';
 
 const StyledEuiModal = styled(EuiModal)`
   width: 800px;
@@ -45,13 +46,15 @@ export const QUICK_PROMPTS_TAB = 'QUICK_PROMPTS_TAB' as const;
 export const SYSTEM_PROMPTS_TAB = 'SYSTEM_PROMPTS_TAB' as const;
 export const ANONYMIZATION_TAB = 'ANONYMIZATION_TAB' as const;
 export const ADVANCED_TAB = 'ADVANCED_TAB' as const;
+export const EVALUATION_TAB = 'EVALUATION_TAB' as const;
 
 export type SettingsTabs =
   | typeof CONVERSATIONS_TAB
   | typeof QUICK_PROMPTS_TAB
   | typeof SYSTEM_PROMPTS_TAB
   | typeof ANONYMIZATION_TAB
-  | typeof ADVANCED_TAB;
+  | typeof ADVANCED_TAB
+  | typeof EVALUATION_TAB;
 interface Props {
   defaultConnectorId?: string;
   defaultProvider?: OpenAiProviderType;
@@ -243,6 +246,16 @@ export const AssistantSettings: React.FC<Props> = React.memo(
                   <EuiIcon type="advancedSettingsApp" size="l" />
                 </EuiKeyPadMenuItem>
               )}
+              {assistantLangChain && (
+                <EuiKeyPadMenuItem
+                  id={EVALUATION_TAB}
+                  label={i18n.EVALUATION_MENU_ITEM}
+                  isSelected={selectedSettingsTab === EVALUATION_TAB}
+                  onClick={() => setSelectedSettingsTab(EVALUATION_TAB)}
+                >
+                  <EuiIcon type="crossClusterReplicationApp" size="l" />
+                </EuiKeyPadMenuItem>
+              )}
             </EuiKeyPadMenu>
           </EuiPageSidebar>
           <EuiPageBody paddingSize="none" panelled={true}>
@@ -295,6 +308,7 @@ export const AssistantSettings: React.FC<Props> = React.memo(
                   />
                 )}
                 {selectedSettingsTab === ADVANCED_TAB && <AdvancedSettings />}
+                {selectedSettingsTab === EVALUATION_TAB && <EvaluationSettings />}
               </EuiSplitPanel.Inner>
               <EuiSplitPanel.Inner
                 grow={false}