diff --git a/x-pack/plugins/actions/server/lib/action_executor.test.ts b/x-pack/plugins/actions/server/lib/action_executor.test.ts index 015cc09fc383f..dfb0ccda5c45a 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.test.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.test.ts @@ -10,7 +10,7 @@ import { schema } from '@kbn/config-schema'; import { ActionExecutor } from './action_executor'; import { actionTypeRegistryMock } from '../action_type_registry.mock'; import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks'; -import { httpServerMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { httpServerMock, loggingSystemMock, analyticsServiceMock } from '@kbn/core/server/mocks'; import { eventLoggerMock } from '@kbn/event-log-plugin/server/mocks'; import { spacesServiceMock } from '@kbn/spaces-plugin/server/spaces_service/spaces_service.mock'; import { ActionType as ConnectorType } from '../types'; @@ -26,6 +26,7 @@ import { PassThrough } from 'stream'; import { SecurityConnectorFeatureId } from '../../common'; import { TaskErrorSource } from '@kbn/task-manager-plugin/common'; import { createTaskRunError, getErrorSource } from '@kbn/task-manager-plugin/server/task_running'; +import { GEN_AI_TOKEN_COUNT_EVENT } from './event_based_telemetry'; const actionExecutor = new ActionExecutor({ isESOCanEncrypt: true }); const services = actionsMock.createServices(); @@ -61,7 +62,6 @@ const securityMockStart = securityMock.createStart(); const authorizationMock = actionsAuthorizationMock.create(); const getActionsAuthorizationWithRequest = jest.fn(); - const actionExecutorInitializationParams = { logger: loggerMock, spaces: spacesMock, @@ -69,6 +69,7 @@ const actionExecutorInitializationParams = { getServices: () => services, getUnsecuredServices: () => unsecuredServices, actionTypeRegistry: connectorTypeRegistry, + analyticsService: analyticsServiceMock.createAnalyticsServiceStart(), encryptedSavedObjectsClient, eventLogger, getActionsAuthorizationWithRequest, @@ -1355,163 +1356,160 @@ describe('System actions', () => { }); }); }); - -test('writes to event log for execute timeout', async () => { - setupActionExecutorMock(); - - await actionExecutor.logCancellation({ - actionId: 'action1', - executionId: '123abc', - consumer: 'test-consumer', - relatedSavedObjects: [], - request: {} as KibanaRequest, - actionExecutionId: '2', - }); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(1); - expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { - event: { - action: 'execute-timeout', - kind: 'action', - }, - kibana: { - action: { - execution: { - uuid: '2', - }, - name: undefined, - id: 'action1', +describe('Event log', () => { + test('writes to event log for execute timeout', async () => { + setupActionExecutorMock(); + + await actionExecutor.logCancellation({ + actionId: 'action1', + executionId: '123abc', + consumer: 'test-consumer', + relatedSavedObjects: [], + request: {} as KibanaRequest, + actionExecutionId: '2', + }); + expect(eventLogger.logEvent).toHaveBeenCalledTimes(1); + expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { + event: { + action: 'execute-timeout', + kind: 'action', }, - alert: { - rule: { - consumer: 'test-consumer', + kibana: { + action: { execution: { - uuid: '123abc', + uuid: '2', }, - }, - }, - saved_objects: [ - { + name: undefined, id: 'action1', - namespace: 'some-namespace', - rel: 'primary', - type: 'action', - type_id: 'test', }, - ], - space_ids: ['some-namespace'], - }, - message: - 'action: test:action1: \'action-1\' execution cancelled due to timeout - exceeded default timeout of "5m"', + alert: { + rule: { + consumer: 'test-consumer', + execution: { + uuid: '123abc', + }, + }, + }, + saved_objects: [ + { + id: 'action1', + namespace: 'some-namespace', + rel: 'primary', + type: 'action', + type_id: 'test', + }, + ], + space_ids: ['some-namespace'], + }, + message: + 'action: test:action1: \'action-1\' execution cancelled due to timeout - exceeded default timeout of "5m"', + }); }); -}); -test('writes to event log for execute and execute start', async () => { - const executorMock = setupActionExecutorMock(); - executorMock.mockResolvedValue({ - actionId: '1', - status: 'ok', - }); - await actionExecutor.execute(executeParams); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); - expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { - event: { - action: 'execute-start', - kind: 'action', - }, - kibana: { - action: { - execution: { - uuid: '2', - }, - name: 'action-1', - id: '1', + test('writes to event log for execute and execute start', async () => { + const executorMock = setupActionExecutorMock(); + executorMock.mockResolvedValue({ + actionId: '1', + status: 'ok', + }); + await actionExecutor.execute(executeParams); + expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); + expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { + event: { + action: 'execute-start', + kind: 'action', }, - alert: { - rule: { + kibana: { + action: { execution: { - uuid: '123abc', + uuid: '2', }, - }, - }, - saved_objects: [ - { + name: 'action-1', id: '1', - namespace: 'some-namespace', - rel: 'primary', - type: 'action', - type_id: 'test', }, - ], - space_ids: ['some-namespace'], - }, - message: 'action started: test:1: action-1', - }); -}); - -test('writes to event log for execute and execute start when consumer and related saved object are defined', async () => { - const executorMock = setupActionExecutorMock(); - executorMock.mockResolvedValue({ - actionId: '1', - status: 'ok', - }); - await actionExecutor.execute({ - ...executeParams, - consumer: 'test-consumer', - relatedSavedObjects: [ - { - typeId: '.rule-type', - type: 'alert', - id: '12', + alert: { + rule: { + execution: { + uuid: '123abc', + }, + }, + }, + saved_objects: [ + { + id: '1', + namespace: 'some-namespace', + rel: 'primary', + type: 'action', + type_id: 'test', + }, + ], + space_ids: ['some-namespace'], }, - ], + message: 'action started: test:1: action-1', + }); }); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); - expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { - event: { - action: 'execute-start', - kind: 'action', - }, - kibana: { - action: { - execution: { - uuid: '2', + + test('writes to event log for execute and execute start when consumer and related saved object are defined', async () => { + const executorMock = setupActionExecutorMock(); + executorMock.mockResolvedValue({ + actionId: '1', + status: 'ok', + }); + await actionExecutor.execute({ + ...executeParams, + consumer: 'test-consumer', + relatedSavedObjects: [ + { + typeId: '.rule-type', + type: 'alert', + id: '12', }, - name: 'action-1', - id: '1', + ], + }); + expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); + expect(eventLogger.logEvent).toHaveBeenNthCalledWith(1, { + event: { + action: 'execute-start', + kind: 'action', }, - alert: { - rule: { - consumer: 'test-consumer', + kibana: { + action: { execution: { - uuid: '123abc', + uuid: '2', }, - rule_type_id: '.rule-type', - }, - }, - saved_objects: [ - { + name: 'action-1', id: '1', - namespace: 'some-namespace', - rel: 'primary', - type: 'action', - type_id: 'test', }, - { - id: '12', - namespace: undefined, - rel: 'primary', - type: 'alert', - type_id: '.rule-type', + alert: { + rule: { + consumer: 'test-consumer', + execution: { + uuid: '123abc', + }, + rule_type_id: '.rule-type', + }, }, - ], - space_ids: ['some-namespace'], - }, - message: 'action started: test:1: action-1', + saved_objects: [ + { + id: '1', + namespace: 'some-namespace', + rel: 'primary', + type: 'action', + type_id: 'test', + }, + { + id: '12', + namespace: undefined, + rel: 'primary', + type: 'alert', + type_id: '.rule-type', + }, + ], + space_ids: ['some-namespace'], + }, + message: 'action started: test:1: action-1', + }); }); -}); - -test('writes usage data to event log for OpenAI events', async () => { - const executorMock = setupActionExecutorMock('.gen-ai'); const mockGenAi = { id: 'chatcmpl-7LztF5xsJl2z5jcNpJKvaPm4uWt8x', object: 'chat.completion', @@ -1533,150 +1531,167 @@ test('writes usage data to event log for OpenAI events', async () => { }, ], }; - executorMock.mockResolvedValue({ - actionId: '1', - status: 'ok', - // @ts-ignore - data: mockGenAi, - }); - await actionExecutor.execute(executeParams); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); - expect(eventLogger.logEvent).toHaveBeenNthCalledWith(2, { - event: { - action: 'execute', - kind: 'action', - outcome: 'success', - }, - kibana: { - action: { - execution: { - uuid: '2', - gen_ai: { - usage: mockGenAi.usage, - }, - }, - name: 'action-1', - id: '1', + test('writes usage data to event log for OpenAI events', async () => { + const executorMock = setupActionExecutorMock('.gen-ai'); + + executorMock.mockResolvedValue({ + actionId: '1', + status: 'ok', + // @ts-ignore + data: mockGenAi, + }); + await actionExecutor.execute(executeParams); + expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); + expect(eventLogger.logEvent).toHaveBeenNthCalledWith(2, { + event: { + action: 'execute', + kind: 'action', + outcome: 'success', }, - alert: { - rule: { + kibana: { + action: { execution: { - uuid: '123abc', + uuid: '2', + gen_ai: { + usage: mockGenAi.usage, + }, }, - }, - }, - saved_objects: [ - { + name: 'action-1', id: '1', - namespace: 'some-namespace', - rel: 'primary', - type: 'action', - type_id: '.gen-ai', }, - ], - space_ids: ['some-namespace'], - }, - message: 'action executed: .gen-ai:1: action-1', - user: { name: 'coolguy', id: '123' }, + alert: { + rule: { + execution: { + uuid: '123abc', + }, + }, + }, + saved_objects: [ + { + id: '1', + namespace: 'some-namespace', + rel: 'primary', + type: 'action', + type_id: '.gen-ai', + }, + ], + space_ids: ['some-namespace'], + }, + message: 'action executed: .gen-ai:1: action-1', + user: { name: 'coolguy', id: '123' }, + }); }); -}); -test('writes usage data to event log for streaming OpenAI events', async () => { - const executorMock = setupActionExecutorMock('.gen-ai', { - params: { schema: schema.any() }, - config: { schema: schema.any() }, - secrets: { schema: schema.any() }, - }); + test('writes usage data to event log for streaming OpenAI events', async () => { + const executorMock = setupActionExecutorMock('.gen-ai', { + params: { schema: schema.any() }, + config: { schema: schema.any() }, + secrets: { schema: schema.any() }, + }); - const stream = new PassThrough(); + const stream = new PassThrough(); - executorMock.mockResolvedValue({ - actionId: '1', - status: 'ok', - // @ts-ignore - data: stream, - }); + executorMock.mockResolvedValue({ + actionId: '1', + status: 'ok', + // @ts-ignore + data: stream, + }); - await actionExecutor.execute({ - ...executeParams, - params: { - subActionParams: { - body: JSON.stringify({ - messages: [ - { - role: 'system', - content: 'System message', - }, - { - role: 'user', - content: 'User message', - }, - ], - }), + await actionExecutor.execute({ + ...executeParams, + params: { + subActionParams: { + body: JSON.stringify({ + messages: [ + { + role: 'system', + content: 'System message', + }, + { + role: 'user', + content: 'User message', + }, + ], + }), + }, }, - }, - }); + }); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(1); - stream.write( - `data: ${JSON.stringify({ - object: 'chat.completion.chunk', - choices: [{ delta: { content: 'Single' } }], - })}\n` - ); - stream.write(`data: [DONE]`); + expect(eventLogger.logEvent).toHaveBeenCalledTimes(1); + stream.write( + `data: ${JSON.stringify({ + object: 'chat.completion.chunk', + choices: [{ delta: { content: 'Single' } }], + })}\n` + ); + stream.write(`data: [DONE]`); - stream.end(); + stream.end(); - await finished(stream); + await finished(stream); - await new Promise(process.nextTick); + await new Promise(process.nextTick); - expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); - expect(eventLogger.logEvent).toHaveBeenNthCalledWith(2, { - event: { - action: 'execute', - kind: 'action', - outcome: 'success', - }, - kibana: { - action: { - execution: { - uuid: '2', - gen_ai: { - usage: { - completion_tokens: 5, - prompt_tokens: 30, - total_tokens: 35, + expect(eventLogger.logEvent).toHaveBeenCalledTimes(2); + expect(eventLogger.logEvent).toHaveBeenNthCalledWith(2, { + event: { + action: 'execute', + kind: 'action', + outcome: 'success', + }, + kibana: { + action: { + execution: { + uuid: '2', + gen_ai: { + usage: { + completion_tokens: 5, + prompt_tokens: 30, + total_tokens: 35, + }, }, }, + name: 'action-1', + id: '1', }, - name: 'action-1', - id: '1', - }, - alert: { - rule: { - execution: { - uuid: '123abc', + alert: { + rule: { + execution: { + uuid: '123abc', + }, }, }, + saved_objects: [ + { + id: '1', + namespace: 'some-namespace', + rel: 'primary', + type: 'action', + type_id: '.gen-ai', + }, + ], + space_ids: ['some-namespace'], }, - saved_objects: [ - { - id: '1', - namespace: 'some-namespace', - rel: 'primary', - type: 'action', - type_id: '.gen-ai', - }, - ], - space_ids: ['some-namespace'], - }, - message: 'action executed: .gen-ai:1: action-1', - user: { name: 'coolguy', id: '123' }, + message: 'action executed: .gen-ai:1: action-1', + user: { name: 'coolguy', id: '123' }, + }); + }); + test('reports telemetry for token count events', async () => { + const executorMock = setupActionExecutorMock('.gen-ai'); + executorMock.mockResolvedValue({ + actionId: '1', + status: 'ok', + // @ts-ignore + data: mockGenAi, + }); + await actionExecutor.execute(executeParams); + expect(actionExecutorInitializationParams.analyticsService.reportEvent).toHaveBeenCalledWith( + GEN_AI_TOKEN_COUNT_EVENT.eventType, + { actionTypeId: '.gen-ai', completion_tokens: 9, prompt_tokens: 10, total_tokens: 19 } + ); }); }); - function setupActionExecutorMock( actionTypeId = 'test', validationOverride?: ConnectorType['validate'] diff --git a/x-pack/plugins/actions/server/lib/action_executor.ts b/x-pack/plugins/actions/server/lib/action_executor.ts index c06e33bf3df6a..5ca37f6ee871c 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.ts @@ -6,7 +6,12 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { KibanaRequest, Logger, SavedObjectsErrorHelpers } from '@kbn/core/server'; +import { + AnalyticsServiceStart, + KibanaRequest, + Logger, + SavedObjectsErrorHelpers, +} from '@kbn/core/server'; import { cloneDeep } from 'lodash'; import { set } from '@kbn/safer-lodash-set'; import { withSpan } from '@kbn/apm-utils'; @@ -16,6 +21,7 @@ import { IEventLogger, SAVED_OBJECT_REL_PRIMARY } from '@kbn/event-log-plugin/se import { AuthenticatedUser, SecurityPluginStart } from '@kbn/security-plugin/server'; import { createTaskRunError, TaskErrorSource } from '@kbn/task-manager-plugin/server'; import { getErrorSource } from '@kbn/task-manager-plugin/server/task_running'; +import { GEN_AI_TOKEN_COUNT_EVENT } from './event_based_telemetry'; import { getGenAiTokenTracking, shouldTrackGenAiToken } from './gen_ai_token_tracking'; import { validateConfig, @@ -58,6 +64,7 @@ export interface ActionExecutorContext { getUnsecuredServices: GetUnsecuredServicesFunction; encryptedSavedObjectsClient: EncryptedSavedObjectsClient; actionTypeRegistry: ActionTypeRegistryContract; + analyticsService: AnalyticsServiceStart; eventLogger: IEventLogger; inMemoryConnectors: InMemoryConnector[]; getActionsAuthorizationWithRequest: (request: KibanaRequest) => ActionsAuthorization; @@ -380,7 +387,7 @@ export class ActionExecutor { }, }, async (span) => { - const { actionTypeRegistry, eventLogger } = this.actionExecutorContext!; + const { actionTypeRegistry, analyticsService, eventLogger } = this.actionExecutorContext!; const actionInfo = await this.getActionInfoInternal(actionId, namespace.namespace); @@ -587,6 +594,15 @@ export class ActionExecutor { prompt_tokens: tokenTracking.prompt_tokens, completion_tokens: tokenTracking.completion_tokens, }); + analyticsService.reportEvent(GEN_AI_TOKEN_COUNT_EVENT.eventType, { + actionTypeId, + total_tokens: tokenTracking.total_tokens, + prompt_tokens: tokenTracking.prompt_tokens, + completion_tokens: tokenTracking.completion_tokens, + ...(actionTypeId === '.gen-ai' && config?.apiProvider != null + ? { provider: config?.apiProvider } + : {}), + }); } }) .catch((err) => { diff --git a/x-pack/plugins/actions/server/lib/event_based_telemetry.ts b/x-pack/plugins/actions/server/lib/event_based_telemetry.ts new file mode 100644 index 0000000000000..eb7ad8dbdfb65 --- /dev/null +++ b/x-pack/plugins/actions/server/lib/event_based_telemetry.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EventTypeOpts } from '@kbn/core/server'; + +export const GEN_AI_TOKEN_COUNT_EVENT: EventTypeOpts<{ + actionTypeId: string; + total_tokens: number; + prompt_tokens: number; + completion_tokens: number; + provider?: string; +}> = { + eventType: 'gen_ai_token_count', + schema: { + actionTypeId: { + type: 'keyword', + _meta: { + description: 'Kibana connector type', + optional: false, + }, + }, + total_tokens: { + type: 'integer', + _meta: { + description: 'Total token count', + optional: false, + }, + }, + prompt_tokens: { + type: 'integer', + _meta: { + description: 'Prompt token count', + optional: false, + }, + }, + completion_tokens: { + type: 'integer', + _meta: { + description: 'Completion token count', + optional: false, + }, + }, + provider: { + type: 'keyword', + _meta: { + description: 'OpenAI provider', + optional: true, + }, + }, + }, +}; + +export const events: Array> = [GEN_AI_TOKEN_COUNT_EVENT]; diff --git a/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts b/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts index 96a3059ef852a..c5f10f728065e 100644 --- a/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts +++ b/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts @@ -17,6 +17,7 @@ import { loggingSystemMock, httpServiceMock, savedObjectsRepositoryMock, + analyticsServiceMock, } from '@kbn/core/server/mocks'; import { eventLoggerMock } from '@kbn/event-log-plugin/server/mocks'; import { ActionTypeDisabledError } from './errors'; @@ -96,6 +97,7 @@ const actionExecutorInitializerParams = { encryptedSavedObjectsClient: mockedEncryptedSavedObjectsClient, eventLogger, inMemoryConnectors: [], + analyticsService: analyticsServiceMock.createAnalyticsServiceStart(), }; const taskRunnerFactoryInitializerParams = { diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts index 651de2ea04137..111e0509f81ac 100644 --- a/x-pack/plugins/actions/server/plugin.ts +++ b/x-pack/plugins/actions/server/plugin.ts @@ -44,6 +44,7 @@ import { MonitoringCollectionSetup } from '@kbn/monitoring-collection-plugin/ser import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/server'; import { ActionsConfig, AllowedHosts, EnabledConnectorTypes, getValidatedConfig } from './config'; import { resolveCustomHosts } from './lib/custom_host_settings'; +import { events } from './lib/event_based_telemetry'; import { ActionsClient } from './actions_client/actions_client'; import { ActionTypeRegistry } from './action_type_registry'; import { @@ -249,7 +250,7 @@ export class ActionsPlugin implements Plugin core.analytics.registerEventType(eventConfig)); const actionExecutor = new ActionExecutor({ isESOCanEncrypt: this.isESOCanEncrypt, }); @@ -571,6 +572,7 @@ export class ActionsPlugin implements Plugin