From 7368339e001f097602cbbbf4270dd491df309443 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 12 Mar 2025 13:17:07 -0600 Subject: [PATCH 1/4] wip --- .../shared/kbn-langchain/server/language_models/llm.ts | 7 +++++++ .../automatic_import/server/routes/analyze_api_route.ts | 3 +++ .../automatic_import/server/routes/analyze_logs_routes.ts | 3 +++ .../server/routes/categorization_routes.ts | 3 +++ .../shared/automatic_import/server/routes/cel_routes.ts | 3 +++ .../shared/automatic_import/server/routes/ecs_routes.ts | 3 +++ .../automatic_import/server/routes/related_routes.ts | 3 +++ .../server/connector_types/inference/inference.ts | 8 ++++++++ .../evaluation/helpers/get_evaluator_llm/index.ts | 3 +++ .../server/lib/attack_discovery/evaluation/index.ts | 3 +++ .../lib/langchain/graphs/default_assistant_graph/index.ts | 3 +++ .../post/helpers/invoke_attack_discovery_graph/index.tsx | 3 +++ .../server/routes/defend_insights/helpers.ts | 6 ++++++ .../server/routes/evaluate/post_evaluate.ts | 3 +++ 14 files changed, 54 insertions(+) diff --git a/x-pack/platform/packages/shared/kbn-langchain/server/language_models/llm.ts b/x-pack/platform/packages/shared/kbn-langchain/server/language_models/llm.ts index 787c4e85b1358..166a8db644870 100644 --- a/x-pack/platform/packages/shared/kbn-langchain/server/language_models/llm.ts +++ b/x-pack/platform/packages/shared/kbn-langchain/server/language_models/llm.ts @@ -11,6 +11,7 @@ import { LLM } from '@langchain/core/language_models/llms'; import { get } from 'lodash/fp'; import { v4 as uuidv4 } from 'uuid'; import { PublicMethodsOf } from '@kbn/utility-types'; +import type { TelemetryMetadata } from '@kbn/actions-plugin/server/lib'; import { DEFAULT_TIMEOUT, getDefaultArguments } from './constants'; import { getMessageContentAndRole } from './helpers'; @@ -28,6 +29,7 @@ interface ActionsClientLlmParams { timeout?: number; traceId?: string; traceOptions?: TraceOptions; + telemetryMetadata?: TelemetryMetadata; } export class ActionsClientLlm extends LLM { @@ -36,6 +38,7 @@ export class ActionsClientLlm extends LLM { #logger: Logger; #traceId: string; #timeout?: number; + telemetryMetadata?: TelemetryMetadata; // Local `llmType` as it can change and needs to be accessed by abstract `_llmType()` method // Not using getter as `this._llmType()` is called in the constructor via `super({})` @@ -54,6 +57,7 @@ export class ActionsClientLlm extends LLM { temperature, timeout, traceOptions, + telemetryMetadata, }: ActionsClientLlmParams) { super({ callbacks: [...(traceOptions?.tracers ?? [])], @@ -67,6 +71,7 @@ export class ActionsClientLlm extends LLM { this.#timeout = timeout; this.model = model; this.temperature = temperature; + this.telemetryMetadata = telemetryMetadata; } _llmType() { @@ -102,6 +107,7 @@ export class ActionsClientLlm extends LLM { model: this.model, messages: [assistantMessage], // the assistant message }, + telemetryMetadata: this.telemetryMetadata, }, } : { @@ -113,6 +119,7 @@ export class ActionsClientLlm extends LLM { ...getDefaultArguments(this.llmType, this.temperature), // This timeout is large because LangChain prompts can be complicated and take a long time timeout: this.#timeout ?? DEFAULT_TIMEOUT, + telemetryMetadata: this.telemetryMetadata, }, }, }; diff --git a/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts b/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts index f73d79ed28cad..c74b3407fa881 100644 --- a/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts +++ b/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts @@ -77,6 +77,9 @@ export function registerApiAnalysisRoutes(router: IRouter { * @signal abort signal */ public async performApiUnifiedCompletionStream(params: UnifiedChatCompleteParams) { + const productUseCase = params.telemetryMetadata?.pluginId + ? params.telemetryMetadata?.pluginId === 'siem_migrations' + ? 'security_migration' + : params.telemetryMetadata?.pluginId + : 'unknown'; const response = await this.esClient.transport.request( { method: 'POST', @@ -196,6 +201,9 @@ export class InferenceConnector extends SubActionConnector { asStream: true, meta: true, signal: params.signal, + headers: { + 'X-Elastic-Product-Use-Case': productUseCase, + }, } ); // errors should be thrown as it will not be a stream response diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/helpers/get_evaluator_llm/index.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/helpers/get_evaluator_llm/index.ts index 236def9670d07..968a21122361a 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/helpers/get_evaluator_llm/index.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/helpers/get_evaluator_llm/index.ts @@ -61,5 +61,8 @@ export const getEvaluatorLlm = async ({ temperature: 0, // zero temperature for evaluation timeout: connectorTimeout, traceOptions, + telemetryMetadata: { + pluginId: 'security_attack_discovery', + }, }); }; diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/index.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/index.ts index 2de3eb0bdf9d3..ce5b2ab71d177 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/index.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/index.ts @@ -92,6 +92,9 @@ export const evaluateAttackDiscovery = async ({ temperature: 0, // zero temperature for attack discovery, because we want structured JSON output timeout: connectorTimeout, traceOptions, + telemetryMetadata: { + pluginId: 'security_attack_discovery', + }, }); const graph = getDefaultAttackDiscoveryGraph({ diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts index da1d2244e5c5e..10039f1b1039d 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts @@ -89,6 +89,9 @@ export const callAssistantGraph: AgentExecutor = async ({ // failure could be due to bad connector, we should deliver that result to the client asap maxRetries: 0, convertSystemMessageToHumanContent: false, + telemetryMetadata: { + pluginId: 'security_ai_assistant', + }, }); const anonymizationFieldsRes = diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/attack_discovery/post/helpers/invoke_attack_discovery_graph/index.tsx b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/attack_discovery/post/helpers/invoke_attack_discovery_graph/index.tsx index 850bb06829d13..f0be667a81800 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/attack_discovery/post/helpers/invoke_attack_discovery_graph/index.tsx +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/attack_discovery/post/helpers/invoke_attack_discovery_graph/index.tsx @@ -87,6 +87,9 @@ export const invokeAttackDiscoveryGraph = async ({ temperature: 0, // zero temperature for attack discovery, because we want structured JSON output timeout: connectorTimeout, traceOptions, + telemetryMetadata: { + pluginId: 'security_attack_discovery', + }, }); if (llm == null) { diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/defend_insights/helpers.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/defend_insights/helpers.ts index 66363db3ae7fc..47fab77b675c8 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/defend_insights/helpers.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/defend_insights/helpers.ts @@ -156,6 +156,9 @@ export function getAssistantToolParams({ temperature: 0, // zero temperature because we want structured JSON output timeout: connectorTimeout, traceOptions, + telemetryMetadata: { + pluginId: 'security_defend_insights', + }, }); return { @@ -443,6 +446,9 @@ export const invokeDefendInsightsGraph = async ({ temperature: 0, timeout: connectorTimeout, traceOptions, + telemetryMetadata: { + pluginId: 'security_defend_insights', + }, }); if (llm == null) { diff --git a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index ae82fec6ceeca..1056b9bb29a5b 100644 --- a/x-pack/solutions/security/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/solutions/security/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -250,6 +250,9 @@ export const postEvaluateRoute = ( streaming: false, maxRetries: 0, convertSystemMessageToHumanContent: false, + telemetryMetadata: { + pluginId: 'security_ai_assistant', + }, }); const llm = createLlmInstance(); const anonymizationFieldsRes = From 15d24f37fdc55a60ae364fe56c13bf86a842b9b7 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Wed, 12 Mar 2025 13:20:42 -0600 Subject: [PATCH 2/4] security_automatic_import => automatic_import --- .../shared/automatic_import/server/routes/analyze_api_route.ts | 2 +- .../automatic_import/server/routes/analyze_logs_routes.ts | 2 +- .../automatic_import/server/routes/categorization_routes.ts | 2 +- .../plugins/shared/automatic_import/server/routes/cel_routes.ts | 2 +- .../plugins/shared/automatic_import/server/routes/ecs_routes.ts | 2 +- .../shared/automatic_import/server/routes/related_routes.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts b/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts index c74b3407fa881..862ac30cfaf51 100644 --- a/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts +++ b/x-pack/platform/plugins/shared/automatic_import/server/routes/analyze_api_route.ts @@ -78,7 +78,7 @@ export function registerApiAnalysisRoutes(router: IRouter Date: Wed, 12 Mar 2025 15:08:36 -0600 Subject: [PATCH 3/4] fix tests --- .../inference/inference.test.ts | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts index f3e1a43063599..ae2eb3ebf0894 100644 --- a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts +++ b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts @@ -70,6 +70,7 @@ describe('InferenceConnector', () => { const response = await connector.performApiUnifiedCompletion({ body: { messages: [{ content: 'What is Elastic?', role: 'user' }] }, + telemetryMetadata: { pluginId: 'security_ai_assistant' }, }); expect(mockEsClient.transport.request).toBeCalledTimes(1); expect(mockEsClient.transport.request).toHaveBeenCalledWith( @@ -86,7 +87,13 @@ describe('InferenceConnector', () => { method: 'POST', path: '_inference/chat_completion/test/_stream', }, - { asStream: true, meta: true } + { + asStream: true, + meta: true, + headers: { + 'X-Elastic-Product-Use-Case': 'security_ai_assistant', + }, + } ); expect(response.choices[0].message.content).toEqual(' you'); }); @@ -290,7 +297,13 @@ describe('InferenceConnector', () => { method: 'POST', path: '_inference/chat_completion/test/_stream', }, - { asStream: true, meta: true } + { + asStream: true, + meta: true, + headers: { + 'X-Elastic-Product-Use-Case': 'unknown', + }, + } ); }); @@ -312,7 +325,14 @@ describe('InferenceConnector', () => { method: 'POST', path: '_inference/chat_completion/test/_stream', }, - { asStream: true, meta: true, signal } + { + asStream: true, + meta: true, + signal, + headers: { + 'X-Elastic-Product-Use-Case': 'unknown', + }, + } ); }); From ac3c3707368630d6d620cf94cace048c86d58427 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Thu, 13 Mar 2025 08:00:26 -0600 Subject: [PATCH 4/4] updates from slack --- .../connector_types/inference/inference.test.ts | 6 ------ .../server/connector_types/inference/inference.ts | 15 +++++++-------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts index ae2eb3ebf0894..d975b6aafb208 100644 --- a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts +++ b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.test.ts @@ -300,9 +300,6 @@ describe('InferenceConnector', () => { { asStream: true, meta: true, - headers: { - 'X-Elastic-Product-Use-Case': 'unknown', - }, } ); }); @@ -329,9 +326,6 @@ describe('InferenceConnector', () => { asStream: true, meta: true, signal, - headers: { - 'X-Elastic-Product-Use-Case': 'unknown', - }, } ); }); diff --git a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.ts b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.ts index 509aa40f1a6fa..46c63814058c0 100644 --- a/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.ts +++ b/x-pack/platform/plugins/shared/stack_connectors/server/connector_types/inference/inference.ts @@ -186,11 +186,6 @@ export class InferenceConnector extends SubActionConnector { * @signal abort signal */ public async performApiUnifiedCompletionStream(params: UnifiedChatCompleteParams) { - const productUseCase = params.telemetryMetadata?.pluginId - ? params.telemetryMetadata?.pluginId === 'siem_migrations' - ? 'security_migration' - : params.telemetryMetadata?.pluginId - : 'unknown'; const response = await this.esClient.transport.request( { method: 'POST', @@ -201,9 +196,13 @@ export class InferenceConnector extends SubActionConnector { asStream: true, meta: true, signal: params.signal, - headers: { - 'X-Elastic-Product-Use-Case': productUseCase, - }, + ...(params.telemetryMetadata?.pluginId + ? { + headers: { + 'X-Elastic-Product-Use-Case': params.telemetryMetadata?.pluginId, + }, + } + : {}), } ); // errors should be thrown as it will not be a stream response