From 05dd29e99b84fffafeade9c60a65ffaa889f606a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Mon, 5 Jan 2026 13:30:04 +0100 Subject: [PATCH 01/17] add observability get_trace_change_points tool --- .../agent-builder-server/allow_lists.ts | 1 + .../tools/get_trace_change_points/README.md | 77 +++++++ .../tools/get_trace_change_points/handler.ts | 209 ++++++++++++++++++ .../tools/get_trace_change_points/tool.ts | 122 ++++++++++ .../server/tools/register_tools.ts | 6 + .../apis/observability_agent_builder/index.ts | 1 + .../tools/get_trace_change_points.spec.ts | 80 +++++++ .../create_trace_change_points_data.ts | 75 +++++++ 8 files changed, 571 insertions(+) create mode 100644 x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md create mode 100644 x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts create mode 100644 x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts create mode 100644 x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts create mode 100644 x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts diff --git a/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts b/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts index b8984e012c037..e9124016c4e29 100644 --- a/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts +++ b/x-pack/platform/packages/shared/agent-builder/agent-builder-server/allow_lists.ts @@ -29,6 +29,7 @@ export const AGENT_BUILDER_BUILTIN_TOOLS: string[] = [ `${internalNamespaces.observability}.get_trace_metrics`, `${internalNamespaces.observability}.get_log_change_points`, `${internalNamespaces.observability}.get_metric_change_points`, + `${internalNamespaces.observability}.get_trace_change_points`, // Dashboards 'platform.dashboard.create_dashboard', diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md new file mode 100644 index 0000000000000..17e20f6ab0723 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md @@ -0,0 +1,77 @@ +# get_trace_change_points + +Detects statistically significant changes (e.g., "spike", "dip", "trend_change", "step_change", "distribution_change", "non_stationary", "stationary", or "indeterminable") in trace data over time. Returns the top 25 most significant change points ordered by p-value. + +## Examples + +### Basic time range + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-1h", + "end": "now" + } +} +``` + +### With explicit index pattern + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-1h", + "end": "now", + "index": "index-example-*" + } +} +``` + +### With custom aggregation (span latency p99) + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-24h", + "end": "now", + "aggregation": { + "field": "span.duration.us", + "type": "p99" + } + } +} +``` + +### With groupBy fields + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-24h", + "end": "now", + "groupBy": ["service.name", "service.environment"] + } +} +``` + +### With KQL filter + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-24h", + "end": "now", + "kqlFilter": "service.name:my-service AND transaction.type:request" + } +} +``` diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts new file mode 100644 index 0000000000000..3531a6d30c6f4 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { IScopedClusterClient } from '@kbn/core/server'; +import { orderBy } from 'lodash'; +import { type Bucket, getChangePoints } from '../../utils/get_change_points'; +import { parseDatemath } from '../../utils/time'; +import { timeRangeFilter, kqlFilter } from '../../utils/dsl_filters'; +import { getTypedSearch } from '../../utils/get_typed_search'; + +type TraceType = 'min' | 'max' | 'sum' | 'count' | 'avg' | 'p95' | 'p99'; + +interface AggregationConfig { + field: string; + type: TraceType; +} + +function getTraceAggregation(aggregation: AggregationConfig | undefined): { + agg: AggregationsAggregationContainer; + buckets_path: string; +} { + if (!aggregation) { + return { + agg: { + percentiles: { + field: 'transaction.duration.us', + percents: [95.0], + keyed: true, + }, + }, + buckets_path: 'metric[95.0]', + }; + } + + if (aggregation.type === 'count') { + return { + agg: { + filter: { + match_all: {}, + }, + }, + buckets_path: '_count', + }; + } + + if (['min', 'max', 'sum', 'avg'].includes(aggregation.type)) { + return { + agg: { + [aggregation.type]: { + field: aggregation.field, + }, + } as Record, { field: string }>, + buckets_path: 'metric', + }; + } + + const percentile = `${aggregation.type.split('p')[1]}.0`; + + return { + agg: { + percentiles: { + field: aggregation.field, + percents: [Number(percentile)], + keyed: true, + }, + }, + buckets_path: `metric[${percentile}]`, + }; +} + +function getGroupingAggregation(groupingFields: string[]) { + if (groupingFields.length === 0) { + return { + filters: { + filters: [ + { + match_all: {}, + }, + ], + }, + }; + } + + if (groupingFields.length === 1) { + return { terms: { field: groupingFields[0] } }; + } + + return { + multi_terms: { + terms: groupingFields.map((groupingField) => ({ field: groupingField })), + size: 10, + }, + }; +} + +async function getTraceChangePoints({ + index, + start, + end, + kqlFilter: kuery, + groupBy, + aggregation, + esClient, +}: { + esClient: IScopedClusterClient; + start: string; + end: string; + index: string; + aggregation: AggregationConfig | undefined; + kqlFilter?: string; + groupBy: string[]; +}) { + const { agg: traceAgg, buckets_path: bucketsPathTrace } = getTraceAggregation(aggregation); + + const groupAgg = getGroupingAggregation(groupBy); + + const aggregations = { + groups: { + ...groupAgg, + aggs: { + time_series: { + auto_date_histogram: { + field: '@timestamp', + buckets: 100, + }, + aggs: { + metric: traceAgg, + value: { + bucket_script: { + buckets_path: { + metric: bucketsPathTrace, + }, + script: 'params.metric', + }, + }, + }, + }, + changes: { + change_point: { + buckets_path: 'time_series>value', + }, + // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 + } as AggregationsAggregationContainer, + }, + }, + }; + + const search = getTypedSearch(esClient.asCurrentUser); + + const response = await search({ + index, + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [ + ...timeRangeFilter('@timestamp', { + start: parseDatemath(start), + end: parseDatemath(end), + }), + ...kqlFilter(kuery), + ], + }, + }, + aggs: aggregations as Record, + }); + + const buckets = (response.aggregations as { groups?: { buckets?: Bucket[] } })?.groups?.buckets; + if (!buckets) { + return []; + } + + return await getChangePoints({ + buckets, + }); +} + +export async function getToolHandler({ + esClient, + start, + end, + index, + aggregation, + kqlFilter: kqlFilterValue, + groupBy, +}: { + esClient: IScopedClusterClient; + start: string; + end: string; + index: string; + aggregation: AggregationConfig | undefined; + kqlFilter?: string; + groupBy: string[]; +}) { + const traceChangePoints = await getTraceChangePoints({ + esClient, + index, + start, + end, + aggregation, + kqlFilter: kqlFilterValue, + groupBy, + }); + return orderBy(traceChangePoints.flat(), [(item) => item.changes.p_value]).slice(0, 25); +} diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts new file mode 100644 index 0000000000000..e9cdeab840dbe --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { z } from '@kbn/zod'; +import { ToolType } from '@kbn/agent-builder-common'; +import { ToolResultType } from '@kbn/agent-builder-common/tools/tool_result'; +import type { BuiltinToolDefinition } from '@kbn/agent-builder-server'; +import type { CoreSetup, Logger } from '@kbn/core/server'; +import type { + ObservabilityAgentBuilderPluginSetupDependencies, + ObservabilityAgentBuilderPluginStart, + ObservabilityAgentBuilderPluginStartDependencies, +} from '../../types'; +import { timeRangeSchemaRequired } from '../../utils/tool_schemas'; +import { getApmIndices } from '../../utils/get_apm_indices'; + +import { getToolHandler } from './handler'; + +export const OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID = + 'observability.get_trace_change_points'; + +const getTraceChangePointsSchema = z.object({ + ...timeRangeSchemaRequired, + index: z.string().describe('The index or index pattern to find the traces').optional(), + kqlFilter: z + .string() + .describe( + 'A KQL query to filter the trace documents. Examples: trace.id:"abc123", service.name:"my-service"' + ) + .optional(), + aggregation: z + .object({ + field: z + .string() + .describe( + `Numeric field to aggregate and observe for changes (e.g., 'transaction.duration.us', 'span.duration.us').` + ), + type: z + .enum(['avg', 'sum', 'min', 'max', 'p95', 'p99', 'count']) + .describe('Aggregation to apply to the specified field.'), + }) + .optional(), + groupBy: z + .array(z.string()) + .describe( + `Optional keyword fields to break down metrics by to identify which specific group experienced a change. +Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. +common fields to group by include: +- Service level: 'service.name', 'service.environment', 'service.version' +- Transaction level: 'transaction.name', 'transaction.type' +- Dependency level: 'span.destination.service.resource', 'span.type' (external, db, cache) +- Infrastructure level: 'host.name', 'container.id', 'kubernetes.pod.name' +` + ) + .optional(), +}); + +export function createGetTraceChangePointsTool({ + core, + plugins, + logger, +}: { + core: CoreSetup< + ObservabilityAgentBuilderPluginStartDependencies, + ObservabilityAgentBuilderPluginStart + >; + plugins: ObservabilityAgentBuilderPluginSetupDependencies; + logger: Logger; +}) { + const toolDefinition: BuiltinToolDefinition = { + id: OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, + type: ToolType.builtin, + description: `Detects statistically significant changes (spikes/dips) in traces across groups (e.g., by service, host, or custom fields).`, + schema: getTraceChangePointsSchema, + tags: ['observability', 'traces'], + handler: async ({ start, end, index, kqlFilter, aggregation, groupBy = [] }, { esClient }) => { + try { + const { transaction, span } = await getApmIndices({ core, plugins, logger }); + + const topTraceChangePoints = await getToolHandler({ + esClient, + start, + end, + index: index || transaction + ',' + span, + kqlFilter, + aggregation, + groupBy, + }); + + return { + results: [ + { + type: ToolResultType.other, + data: { + changePoints: topTraceChangePoints, + }, + }, + ], + }; + } catch (error) { + logger.error(`Error getting trace change points: ${error.message}`); + logger.debug(error); + return { + results: [ + { + type: ToolResultType.error, + data: { + message: `Error getting trace change points: ${error.message}`, + stack: error.stack, + }, + }, + ], + }; + } + }, + }; + + return toolDefinition; +} diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts index 66205629a19b7..a7e3af11bc457 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts @@ -53,6 +53,10 @@ import { OBSERVABILITY_GET_METRIC_CHANGE_POINTS_TOOL_ID, createGetMetricChangePointsTool, } from './get_metric_change_points/tool'; +import { + OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, + createGetTraceChangePointsTool, +} from './get_trace_change_points/tool'; const PLATFORM_TOOL_IDS = [ platformCoreTools.search, @@ -75,6 +79,7 @@ const OBSERVABILITY_TOOL_IDS = [ OBSERVABILITY_GET_TRACE_METRICS_TOOL_ID, OBSERVABILITY_GET_LOG_CHANGE_POINTS_TOOL_ID, OBSERVABILITY_GET_METRIC_CHANGE_POINTS_TOOL_ID, + OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, ]; export const OBSERVABILITY_AGENT_TOOL_IDS = [...PLATFORM_TOOL_IDS, ...OBSERVABILITY_TOOL_IDS]; @@ -106,6 +111,7 @@ export async function registerTools({ createGetTraceMetricsTool({ core, plugins, logger }), createGetLogChangePointsTool({ core, plugins, logger }), createGetMetricChangePointsTool({ core, plugins, logger }), + createGetTraceChangePointsTool({ core, plugins, logger }), ]; for (const tool of observabilityTools) { diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts index fd75016ec724a..0da4e461a2706 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts @@ -22,6 +22,7 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./tools/get_trace_metrics.spec.ts')); loadTestFile(require.resolve('./tools/get_log_change_points.spec.ts')); loadTestFile(require.resolve('./tools/get_metric_change_points.spec.ts')); + loadTestFile(require.resolve('./tools/get_trace_change_points.spec.ts')); // ai insights loadTestFile(require.resolve('./ai_insights/error.spec.ts')); diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts new file mode 100644 index 0000000000000..9e38d3a95b57c --- /dev/null +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { ApmSynthtraceEsClient } from '@kbn/synthtrace'; +import type { ToolResultType } from '@kbn/agent-builder-common/tools/tool_result'; +import { OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID } from '@kbn/observability-agent-builder-plugin/server/tools/get_trace_change_points/tool'; +import type { ChangePoint } from '@kbn/observability-agent-builder-plugin/server/utils/get_change_points'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { createAgentBuilderApiClient } from '../utils/agent_builder_client'; +import { + TRACE_CHANGE_POINTS_ANALYSIS_WINDOW, + createTraceChangePointsData, +} from '../utils/synthtrace_scenarios/create_trace_change_points_data'; + +interface ToolResult { + type: ToolResultType.other; + data: { + changePoints: ChangePoint[]; + }; +} + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('synthtrace'); + + describe(`tool: ${OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID}`, function () { + let agentBuilderApiClient: ReturnType; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + + before(async () => { + const supertest = await roleScopedSupertest.getSupertestWithRoleScope('admin'); + agentBuilderApiClient = createAgentBuilderApiClient(supertest); + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await createTraceChangePointsData({ apmSynthtraceEsClient }); + }); + + after(async () => { + await apmSynthtraceEsClient.clean(); + }); + + describe('when retrieving trace change points', () => { + let traceChangePoints: ChangePoint[]; + + before(async () => { + const toolResults: ToolResult[] = await agentBuilderApiClient.executeTool({ + id: OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, + params: { + start: TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.start, + end: TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.end, + aggregation: { + field: 'transaction.duration.us', + type: 'p95', + }, + }, + }); + traceChangePoints = toolResults[0]?.data?.changePoints ?? []; + }); + + it('should detect spike', () => { + expect(traceChangePoints.length).to.be.greaterThan(0); + const spike = traceChangePoints.find((cp: ChangePoint) => cp.changes?.type === 'spike'); + expect(spike).to.be.ok(); + }); + + it('should include time series data for visualization', () => { + traceChangePoints.forEach((cp: ChangePoint) => { + expect(cp).to.have.property('timeSeries'); + expect(cp?.timeSeries?.length).to.be.greaterThan(0); + expect(cp?.timeSeries?.[0]).to.have.property('x'); + expect(cp?.timeSeries?.[0]).to.have.property('y'); + }); + }); + }); + }); +} diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts new file mode 100644 index 0000000000000..8417a32acc4bb --- /dev/null +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import datemath from '@elastic/datemath'; +import type { ApmSynthtraceEsClient } from '@kbn/synthtrace'; +import { apm, timerange } from '@kbn/synthtrace-client'; + +const SERVICE_NAME = 'test-service'; +export const TRACE_CHANGE_POINTS_INDEX = `traces-apm.app.${SERVICE_NAME}-default`; +export const TRACE_CHANGE_POINTS_ANALYSIS_WINDOW = { + start: 'now-60m', + end: 'now', +}; +export const TRACE_CHANGE_POINTS_ANALYSIS_SPIKE_WINDOW = { + start: 'now-30m', + end: 'now-28m', +}; + +/** + * Creates trace data with SPIKE pattern. + */ +export async function createTraceChangePointsData({ + apmSynthtraceEsClient, +}: { + apmSynthtraceEsClient: ApmSynthtraceEsClient; +}) { + const range = timerange( + TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.start, + TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.end + ); + + const spikeStart = datemath.parse(TRACE_CHANGE_POINTS_ANALYSIS_SPIKE_WINDOW.start)!.valueOf(); + const spikeEnd = datemath.parse(TRACE_CHANGE_POINTS_ANALYSIS_SPIKE_WINDOW.end)!.valueOf(); + + const instance = apm + .service({ name: SERVICE_NAME, environment: 'test', agentName: 'test-agent' }) + .instance('instance-test'); + + const transactionName = 'GET /api/orders'; + + const traceStream = range + .interval('1m') + .rate(1) + .generator((timestamp) => { + const isSpike = timestamp >= spikeStart && timestamp < spikeEnd; + + const txDurationUs = isSpike ? 5_000 : 500; + const spanDurationUs = isSpike ? 10_000 : 1_000; + + const traces = []; + for (let i = 0; i < 25; i++) { + traces.push( + instance + .transaction({ transactionName }) + .timestamp(timestamp) + .duration(txDurationUs) + .children( + instance + .span({ spanName: 'db.query', spanType: 'db' }) + .timestamp(timestamp) + .duration(spanDurationUs) + .success() + ) + .success() + ); + } + return traces; + }); + + await apmSynthtraceEsClient.index([traceStream]); +} From 6472b69df0e31f6783180d576e12d4a696466e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Thu, 8 Jan 2026 17:11:19 +0100 Subject: [PATCH 02/17] add trace chenge points analisis for avg latency --- .../data_provider/register_data_providers.ts | 25 +++ .../tools/get_trace_change_points/index.ts | 122 +++++++++++ .../data_registry/data_registry_types.ts | 39 ++++ .../tools/get_trace_change_points/handler.ts | 196 +----------------- .../tools/get_trace_change_points/tool.ts | 35 +--- .../server/tools/register_tools.ts | 2 +- 6 files changed, 207 insertions(+), 212 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts index 3027ce75d8efd..b30e84c772bac 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts @@ -19,6 +19,7 @@ import { getExitSpanChangePoints, getServiceChangePoints, } from '../../routes/assistant_functions/get_changepoints'; +import { getTraceChangePoints } from '../tools/get_trace_change_points'; import { buildApmToolResources } from '../utils/build_apm_tool_resources'; import type { APMPluginSetupDependencies, APMPluginStartDependencies } from '../../types'; @@ -183,4 +184,28 @@ export function registerDataProviders({ }); } ); + + observabilityAgentBuilder.registerDataProvider( + 'traceChangePoints', + async ({ request, start, end, kqlFilter, groupBy }) => { + const { apmEventClient, apmDataAccessServices } = await buildApmToolResources({ + core, + plugins, + request, + logger, + }); + + const startMs = parseDatemath(start); + const endMs = parseDatemath(end); + + return getTraceChangePoints({ + apmEventClient, + apmDataAccessServices, + start: startMs, + end: endMs, + kqlFilter, + groupBy, + }); + } + ); } diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts new file mode 100644 index 0000000000000..70b342738d98d --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; +import type { ChangePointType } from '@kbn/es-types/src'; +import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { ApmDocumentType } from '../../../../common/document_type'; +import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; +import { getPreferredDocumentSource } from '../get_trace_metrics'; + +interface ChangePointDetails { + change_point?: number; + r_value?: number; + trend?: string; + p_value?: number; +} + +interface Bucket { + key: string | number; + key_as_string?: string; + doc_count: number; +} + +interface ChangePointResult { + type: Record; + bucket?: Bucket; +} + +export interface BucketChangePoints extends Bucket { + changes: ChangePointResult; + time_series: { + buckets: Array< + Bucket & { + avg_latency: { + value: number | null; + }; + } + >; + }; +} + +type DocumentType = + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent; + +export async function getTraceChangePoints({ + apmEventClient, + apmDataAccessServices, + start, + end, + kqlFilter, + groupBy, +}: { + apmEventClient: APMEventClient; + apmDataAccessServices: ApmDataAccessServices; + start: number; + end: number; + kqlFilter?: string; + groupBy: string; +}): Promise { + const source = await getPreferredDocumentSource({ + apmDataAccessServices, + start, + end, + groupBy, + kqlFilter, + }); + + const { rollupInterval, hasDurationSummaryField } = source; + const documentType = source.documentType as DocumentType; + const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); + + const response = await apmEventClient.search('get_trace_change_points', { + apm: { + sources: [{ documentType, rollupInterval }], + }, + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [...rangeQuery(start, end), ...kqlQuery(kqlFilter)], + }, + }, + aggs: { + groups: { + terms: { + field: groupBy, + }, + aggs: { + time_series: { + date_histogram: { + field: '@timestamp', + fixed_interval: rollupInterval, + }, + aggs: { + avg_latency: { + avg: { + field: durationField, + }, + }, + }, + }, + changes: { + change_point: { + buckets_path: 'time_series>avg_latency', + }, + // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 + } as AggregationsAggregationContainer, + }, + }, + }, + }); + + return (response.aggregations?.groups?.buckets as BucketChangePoints[]) ?? []; +} diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts index 78581c3d31543..382d144d19311 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts @@ -141,6 +141,37 @@ interface InfraHostsResponse { nodes: InfraEntityMetricsItem[]; } +interface ChangePointDetails { + change_point?: number; + r_value?: number; + trend?: string; + p_value?: number; +} + +interface Bucket { + key: string | number; + key_as_string?: string; + doc_count: number; +} + +interface ChangePointResult { + type: Record; + bucket?: Bucket; +} + +interface BucketChangePoints extends Bucket { + changes: ChangePointResult; + time_series: { + buckets: Array< + Bucket & { + avg_latency: { + value: number | null; + }; + } + >; + }; +} + export interface ObservabilityAgentBuilderDataRegistryTypes { apmErrors: (params: { request: KibanaRequest; @@ -212,4 +243,12 @@ export interface ObservabilityAgentBuilderDataRegistryTypes { query: Record | undefined; hostNames?: string[]; }) => Promise; + + traceChangePoints: (params: { + request: KibanaRequest; + start: string; + end: string; + kqlFilter?: string; + groupBy: string; + }) => Promise; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts index 3531a6d30c6f4..25e904fa73a12 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -4,206 +4,30 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { IScopedClusterClient } from '@kbn/core/server'; -import { orderBy } from 'lodash'; -import { type Bucket, getChangePoints } from '../../utils/get_change_points'; -import { parseDatemath } from '../../utils/time'; -import { timeRangeFilter, kqlFilter } from '../../utils/dsl_filters'; -import { getTypedSearch } from '../../utils/get_typed_search'; - -type TraceType = 'min' | 'max' | 'sum' | 'count' | 'avg' | 'p95' | 'p99'; - -interface AggregationConfig { - field: string; - type: TraceType; -} - -function getTraceAggregation(aggregation: AggregationConfig | undefined): { - agg: AggregationsAggregationContainer; - buckets_path: string; -} { - if (!aggregation) { - return { - agg: { - percentiles: { - field: 'transaction.duration.us', - percents: [95.0], - keyed: true, - }, - }, - buckets_path: 'metric[95.0]', - }; - } - - if (aggregation.type === 'count') { - return { - agg: { - filter: { - match_all: {}, - }, - }, - buckets_path: '_count', - }; - } - - if (['min', 'max', 'sum', 'avg'].includes(aggregation.type)) { - return { - agg: { - [aggregation.type]: { - field: aggregation.field, - }, - } as Record, { field: string }>, - buckets_path: 'metric', - }; - } - - const percentile = `${aggregation.type.split('p')[1]}.0`; - - return { - agg: { - percentiles: { - field: aggregation.field, - percents: [Number(percentile)], - keyed: true, - }, - }, - buckets_path: `metric[${percentile}]`, - }; -} - -function getGroupingAggregation(groupingFields: string[]) { - if (groupingFields.length === 0) { - return { - filters: { - filters: [ - { - match_all: {}, - }, - ], - }, - }; - } - - if (groupingFields.length === 1) { - return { terms: { field: groupingFields[0] } }; - } - - return { - multi_terms: { - terms: groupingFields.map((groupingField) => ({ field: groupingField })), - size: 10, - }, - }; -} - -async function getTraceChangePoints({ - index, - start, - end, - kqlFilter: kuery, - groupBy, - aggregation, - esClient, -}: { - esClient: IScopedClusterClient; - start: string; - end: string; - index: string; - aggregation: AggregationConfig | undefined; - kqlFilter?: string; - groupBy: string[]; -}) { - const { agg: traceAgg, buckets_path: bucketsPathTrace } = getTraceAggregation(aggregation); - - const groupAgg = getGroupingAggregation(groupBy); - - const aggregations = { - groups: { - ...groupAgg, - aggs: { - time_series: { - auto_date_histogram: { - field: '@timestamp', - buckets: 100, - }, - aggs: { - metric: traceAgg, - value: { - bucket_script: { - buckets_path: { - metric: bucketsPathTrace, - }, - script: 'params.metric', - }, - }, - }, - }, - changes: { - change_point: { - buckets_path: 'time_series>value', - }, - // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 - } as AggregationsAggregationContainer, - }, - }, - }; - - const search = getTypedSearch(esClient.asCurrentUser); - - const response = await search({ - index, - size: 0, - track_total_hits: false, - query: { - bool: { - filter: [ - ...timeRangeFilter('@timestamp', { - start: parseDatemath(start), - end: parseDatemath(end), - }), - ...kqlFilter(kuery), - ], - }, - }, - aggs: aggregations as Record, - }); - - const buckets = (response.aggregations as { groups?: { buckets?: Bucket[] } })?.groups?.buckets; - if (!buckets) { - return []; - } - - return await getChangePoints({ - buckets, - }); -} +import type { KibanaRequest } from '@kbn/core/server'; +import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; export async function getToolHandler({ - esClient, + request, + dataRegistry, start, end, - index, - aggregation, kqlFilter: kqlFilterValue, groupBy, }: { - esClient: IScopedClusterClient; + request: KibanaRequest; + dataRegistry: ObservabilityAgentBuilderDataRegistry; start: string; end: string; - index: string; - aggregation: AggregationConfig | undefined; kqlFilter?: string; - groupBy: string[]; + groupBy: string; }) { - const traceChangePoints = await getTraceChangePoints({ - esClient, - index, + const items = await dataRegistry.getData('traceChangePoints', { + request, start, end, - aggregation, kqlFilter: kqlFilterValue, groupBy, }); - return orderBy(traceChangePoints.flat(), [(item) => item.changes.p_value]).slice(0, 25); + return items; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts index e9cdeab840dbe..74730bc35b10f 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -14,9 +14,8 @@ import type { ObservabilityAgentBuilderPluginStart, ObservabilityAgentBuilderPluginStartDependencies, } from '../../types'; +import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; import { timeRangeSchemaRequired } from '../../utils/tool_schemas'; -import { getApmIndices } from '../../utils/get_apm_indices'; - import { getToolHandler } from './handler'; export const OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID = @@ -24,27 +23,14 @@ export const OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID = const getTraceChangePointsSchema = z.object({ ...timeRangeSchemaRequired, - index: z.string().describe('The index or index pattern to find the traces').optional(), kqlFilter: z .string() .describe( 'A KQL query to filter the trace documents. Examples: trace.id:"abc123", service.name:"my-service"' ) .optional(), - aggregation: z - .object({ - field: z - .string() - .describe( - `Numeric field to aggregate and observe for changes (e.g., 'transaction.duration.us', 'span.duration.us').` - ), - type: z - .enum(['avg', 'sum', 'min', 'max', 'p95', 'p99', 'count']) - .describe('Aggregation to apply to the specified field.'), - }) - .optional(), groupBy: z - .array(z.string()) + .string() .describe( `Optional keyword fields to break down metrics by to identify which specific group experienced a change. Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. @@ -60,6 +46,7 @@ common fields to group by include: export function createGetTraceChangePointsTool({ core, + dataRegistry, plugins, logger, }: { @@ -67,26 +54,24 @@ export function createGetTraceChangePointsTool({ ObservabilityAgentBuilderPluginStartDependencies, ObservabilityAgentBuilderPluginStart >; + dataRegistry: ObservabilityAgentBuilderDataRegistry; plugins: ObservabilityAgentBuilderPluginSetupDependencies; logger: Logger; }) { const toolDefinition: BuiltinToolDefinition = { id: OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, type: ToolType.builtin, - description: `Detects statistically significant changes (spikes/dips) in traces across groups (e.g., by service, host, or custom fields).`, + description: `TBD`, schema: getTraceChangePointsSchema, tags: ['observability', 'traces'], - handler: async ({ start, end, index, kqlFilter, aggregation, groupBy = [] }, { esClient }) => { + handler: async ({ start, end, kqlFilter, groupBy = 'service.name' }, { request }) => { try { - const { transaction, span } = await getApmIndices({ core, plugins, logger }); - - const topTraceChangePoints = await getToolHandler({ - esClient, + const changePoints = await getToolHandler({ + request, + dataRegistry, start, end, - index: index || transaction + ',' + span, kqlFilter, - aggregation, groupBy, }); @@ -95,7 +80,7 @@ export function createGetTraceChangePointsTool({ { type: ToolResultType.other, data: { - changePoints: topTraceChangePoints, + changePoints, }, }, ], diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts index a7e3af11bc457..7189762da7bff 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts @@ -111,7 +111,7 @@ export async function registerTools({ createGetTraceMetricsTool({ core, plugins, logger }), createGetLogChangePointsTool({ core, plugins, logger }), createGetMetricChangePointsTool({ core, plugins, logger }), - createGetTraceChangePointsTool({ core, plugins, logger }), + createGetTraceChangePointsTool({ core, dataRegistry, plugins, logger }), ]; for (const tool of observabilityTools) { From f029a16def02a3353099bffcabe052c3d3e96acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 9 Jan 2026 08:25:23 +0100 Subject: [PATCH 03/17] Update x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Søren Louv-Jansen --- .../server/tools/get_trace_change_points/tool.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts index 74730bc35b10f..17343cc9ac923 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -33,11 +33,9 @@ const getTraceChangePointsSchema = z.object({ .string() .describe( `Optional keyword fields to break down metrics by to identify which specific group experienced a change. -Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. -common fields to group by include: +Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. Common fields to group by include: - Service level: 'service.name', 'service.environment', 'service.version' - Transaction level: 'transaction.name', 'transaction.type' -- Dependency level: 'span.destination.service.resource', 'span.type' (external, db, cache) - Infrastructure level: 'host.name', 'container.id', 'kubernetes.pod.name' ` ) From 199993c94b501097fc1b5323ffa5a8292a00a9e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 9 Jan 2026 11:05:59 +0100 Subject: [PATCH 04/17] add latency, throughput and failure_rate trace change points analisis --- .../tools/get_trace_change_points/index.ts | 71 +++++++++++++++++-- .../data_registry/data_registry_types.ts | 12 +++- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts index 70b342738d98d..84a76cf8a59c2 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -9,10 +9,12 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; import type { ChangePointType } from '@kbn/es-types/src'; import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { ApmDocumentType } from '../../../../common/document_type'; +import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; +import { ApmDocumentType } from '../../../../common/document_type'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; import { getPreferredDocumentSource } from '../get_trace_metrics'; +import { getOutcomeAggregation } from '../../../lib/helpers/transaction_error_rate'; interface ChangePointDetails { change_point?: number; @@ -33,13 +35,21 @@ interface ChangePointResult { } export interface BucketChangePoints extends Bucket { - changes: ChangePointResult; + changes_latency: ChangePointResult; + changes_throughput: ChangePointResult; + changes_failure_rate: ChangePointResult; time_series: { buckets: Array< Bucket & { avg_latency: { value: number | null; }; + throughput: { + value: number | null; + }; + failure_rate: { + value: number | null; + }; } >; }; @@ -76,6 +86,11 @@ export async function getTraceChangePoints({ const { rollupInterval, hasDurationSummaryField } = source; const documentType = source.documentType as DocumentType; const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); + const outcomeAggs = getOutcomeAggregation(documentType); + const bucketSizeInSeconds = intervalToSeconds(rollupInterval); + + const calculateFailedTransactionRate = + 'params.successful_or_failed != null && params.successful_or_failed > 0 ? (params.successful_or_failed - params.success) / params.successful_or_failed : 0'; const response = await apmEventClient.search('get_trace_change_points', { apm: { @@ -97,22 +112,70 @@ export async function getTraceChangePoints({ time_series: { date_histogram: { field: '@timestamp', - fixed_interval: rollupInterval, + fixed_interval: `${bucketSizeInSeconds}s`, }, aggs: { + ...outcomeAggs, avg_latency: { avg: { field: durationField, }, }, + failure_rate: + documentType === ApmDocumentType.ServiceTransactionMetric + ? { + bucket_script: { + buckets_path: { + successful_or_failed: 'successful_or_failed', + success: 'successful', + }, + script: { + source: calculateFailedTransactionRate, + }, + }, + } + : { + bucket_script: { + buckets_path: { + successful_or_failed: 'successful_or_failed>_count', + success: 'successful>_count', + }, + script: { + source: calculateFailedTransactionRate, + }, + }, + }, + throughput: { + bucket_script: { + buckets_path: { + count: '_count', + }, + script: { + source: 'params.count != null ? params.count / (params.bucketSize / 60.0) : 0', + params: { + bucketSize: bucketSizeInSeconds, + }, + }, + }, + }, }, }, - changes: { + changes_latency: { change_point: { buckets_path: 'time_series>avg_latency', }, // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 } as AggregationsAggregationContainer, + changes_throughput: { + change_point: { + buckets_path: 'time_series>throughput', + }, + } as AggregationsAggregationContainer, + changes_failure_rate: { + change_point: { + buckets_path: 'time_series>failure_rate', + }, + } as AggregationsAggregationContainer, }, }, }, diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts index 382d144d19311..52ab977f9b496 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts @@ -159,14 +159,22 @@ interface ChangePointResult { bucket?: Bucket; } -interface BucketChangePoints extends Bucket { - changes: ChangePointResult; +export interface BucketChangePoints extends Bucket { + changes_latency: ChangePointResult; + changes_throughput: ChangePointResult; + changes_failure_rate: ChangePointResult; time_series: { buckets: Array< Bucket & { avg_latency: { value: number | null; }; + throughput: { + value: number | null; + }; + failure_rate: { + value: number | null; + }; } >; }; From 57483beeb180655ef49bdc644f91f343e2ca1f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 9 Jan 2026 13:30:25 +0100 Subject: [PATCH 05/17] move getPreferredDocumentSource to utils --- .../tools/get_trace_change_points/index.ts | 2 +- .../utils/get_preferred_document_source.ts | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts index 84a76cf8a59c2..96a2e1c909b87 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -13,8 +13,8 @@ import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_ import { ApmDocumentType } from '../../../../common/document_type'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -import { getPreferredDocumentSource } from '../get_trace_metrics'; import { getOutcomeAggregation } from '../../../lib/helpers/transaction_error_rate'; +import { getPreferredDocumentSource } from '../../utils/get_preferred_document_source'; interface ChangePointDetails { change_point?: number; diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts new file mode 100644 index 0000000000000..473c1df20170e --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getPreferredBucketSizeAndDataSource } from '@kbn/apm-data-access-plugin/common'; +import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; +import { getBucketSize } from '../../../common/utils/get_bucket_size'; + +/** + * Gets the preferred document source based on groupBy, filter, and data availability. + * + * Uses getDocumentSources to determine which document types have data for the given + * filter and groupBy field. This automatically handles: + * - ServiceTransactionMetric: Most efficient, but only has service.name, service.environment, transaction.type + * - TransactionMetric: Has more dimensions (transaction.*, host.*, container.*, kubernetes.*, cloud.*, faas.*, etc.) + * - TransactionEvent: Raw transaction docs, fallback when metrics don't have required fields + */ +export async function getPreferredDocumentSource({ + apmDataAccessServices, + start, + end, + groupBy, + kqlFilter, +}: { + apmDataAccessServices: ApmDataAccessServices; + start: number; + end: number; + groupBy: string; + kqlFilter?: string; +}) { + const kueryParts: string[] = []; + if (kqlFilter) { + kueryParts.push(kqlFilter); + } + kueryParts.push(`${groupBy}: *`); + const kuery = kueryParts.join(' AND '); + + const documentSources = await apmDataAccessServices.getDocumentSources({ + start, + end, + kuery, + }); + + const { bucketSize } = getBucketSize({ + start, + end, + numBuckets: 100, + }); + + const { source } = getPreferredBucketSizeAndDataSource({ + sources: documentSources, + bucketSizeInSeconds: bucketSize, + }); + + return source; +} From 62957c4eaa1cae0dad2592ae9b93c02bde286826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 9 Jan 2026 15:27:54 +0100 Subject: [PATCH 06/17] add support for avg|p95|p99 latancy agg type --- .../data_provider/register_data_providers.ts | 3 +- .../tools/get_trace_change_points/index.ts | 58 ++++++++++++------- .../data_registry/data_registry_types.ts | 3 +- .../tools/get_trace_change_points/handler.ts | 3 + .../tools/get_trace_change_points/tool.ts | 10 +++- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts index b30e84c772bac..9c6de7cdf9840 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts @@ -187,7 +187,7 @@ export function registerDataProviders({ observabilityAgentBuilder.registerDataProvider( 'traceChangePoints', - async ({ request, start, end, kqlFilter, groupBy }) => { + async ({ request, start, end, kqlFilter, groupBy, latencyType }) => { const { apmEventClient, apmDataAccessServices } = await buildApmToolResources({ core, plugins, @@ -205,6 +205,7 @@ export function registerDataProviders({ end: endMs, kqlFilter, groupBy, + latencyType, }); } ); diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts index 96a2e1c909b87..785bfeae6aade 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -41,7 +41,7 @@ export interface BucketChangePoints extends Bucket { time_series: { buckets: Array< Bucket & { - avg_latency: { + latency: { value: number | null; }; throughput: { @@ -60,6 +60,16 @@ type DocumentType = | ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent; +function getChangePointsAggs(bucketsPath: string) { + const changePointAggs = { + change_point: { + buckets_path: bucketsPath, + }, + // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 + } as AggregationsAggregationContainer; + return changePointAggs; +} + export async function getTraceChangePoints({ apmEventClient, apmDataAccessServices, @@ -67,6 +77,7 @@ export async function getTraceChangePoints({ end, kqlFilter, groupBy, + latencyType = 'avg', }: { apmEventClient: APMEventClient; apmDataAccessServices: ApmDataAccessServices; @@ -74,6 +85,7 @@ export async function getTraceChangePoints({ end: number; kqlFilter?: string; groupBy: string; + latencyType?: 'avg' | 'p95' | 'p99'; }): Promise { const source = await getPreferredDocumentSource({ apmDataAccessServices, @@ -116,11 +128,26 @@ export async function getTraceChangePoints({ }, aggs: { ...outcomeAggs, - avg_latency: { - avg: { - field: durationField, - }, - }, + latency: + // Avoid unsupported aggregation on downsampled index, see example error: + // "reason": { + // "type": "unsupported_aggregation_on_downsampled_index", + // "reason": "Field [transaction.duration.summary] of type [aggregate_metric_double] is not supported for aggregation [percentiles]" + // } + durationField !== 'transaction.duration.summary' && + (latencyType === 'p95' || latencyType === 'p99') + ? { + percentiles: { + field: durationField, + percents: [Number(`${latencyType.split('p')[1]}.0`)], + keyed: true, + }, + } + : { + avg: { + field: durationField, + }, + }, failure_rate: documentType === ApmDocumentType.ServiceTransactionMetric ? { @@ -160,22 +187,9 @@ export async function getTraceChangePoints({ }, }, }, - changes_latency: { - change_point: { - buckets_path: 'time_series>avg_latency', - }, - // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 - } as AggregationsAggregationContainer, - changes_throughput: { - change_point: { - buckets_path: 'time_series>throughput', - }, - } as AggregationsAggregationContainer, - changes_failure_rate: { - change_point: { - buckets_path: 'time_series>failure_rate', - }, - } as AggregationsAggregationContainer, + changes_latency: getChangePointsAggs('time_series>latency'), + changes_throughput: getChangePointsAggs('time_series>throughput'), + changes_failure_rate: getChangePointsAggs('time_series>failure_rate'), }, }, }, diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts index 52ab977f9b496..89f1ddc76c1d9 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts @@ -166,7 +166,7 @@ export interface BucketChangePoints extends Bucket { time_series: { buckets: Array< Bucket & { - avg_latency: { + latency: { value: number | null; }; throughput: { @@ -258,5 +258,6 @@ export interface ObservabilityAgentBuilderDataRegistryTypes { end: string; kqlFilter?: string; groupBy: string; + latencyType?: 'avg' | 'p95' | 'p99'; }) => Promise; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts index 25e904fa73a12..9267718167341 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -14,6 +14,7 @@ export async function getToolHandler({ end, kqlFilter: kqlFilterValue, groupBy, + latencyType, }: { request: KibanaRequest; dataRegistry: ObservabilityAgentBuilderDataRegistry; @@ -21,6 +22,7 @@ export async function getToolHandler({ end: string; kqlFilter?: string; groupBy: string; + latencyType: 'avg' | 'p95' | 'p99' | undefined; }) { const items = await dataRegistry.getData('traceChangePoints', { request, @@ -28,6 +30,7 @@ export async function getToolHandler({ end, kqlFilter: kqlFilterValue, groupBy, + latencyType, }); return items; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts index 17343cc9ac923..41cbe4ed7f102 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -40,6 +40,10 @@ Use only low-cardinality fields. Using many fields or high-cardinality fields ca ` ) .optional(), + latencyType: z + .enum(['avg', 'p95', 'p99']) + .describe('Aggregation type for latency change points analysis. default is avg.') + .optional(), }); export function createGetTraceChangePointsTool({ @@ -62,7 +66,10 @@ export function createGetTraceChangePointsTool({ description: `TBD`, schema: getTraceChangePointsSchema, tags: ['observability', 'traces'], - handler: async ({ start, end, kqlFilter, groupBy = 'service.name' }, { request }) => { + handler: async ( + { start, end, kqlFilter, groupBy = 'service.name', latencyType = 'avg' }, + { request } + ) => { try { const changePoints = await getToolHandler({ request, @@ -71,6 +78,7 @@ export function createGetTraceChangePointsTool({ end, kqlFilter, groupBy, + latencyType, }); return { From bb94790dd124d2eb3179646553d527a7fa8eabd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Mon, 12 Jan 2026 11:07:53 +0100 Subject: [PATCH 07/17] update latency aggregation handling --- .../tools/get_trace_change_points/index.ts | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts index 785bfeae6aade..098aed804842e 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -11,6 +11,9 @@ import type { ChangePointType } from '@kbn/es-types/src'; import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; import { ApmDocumentType } from '../../../../common/document_type'; +import { TRANSACTION_DURATION_SUMMARY } from '../../../../common/es_fields/apm'; +import { getLatencyAggregationType } from '../../../../common/latency_aggregation_types'; +import { getLatencyAggregation } from '../../../lib/helpers/latency_aggregation_type'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; import { getOutcomeAggregation } from '../../../lib/helpers/transaction_error_rate'; @@ -100,6 +103,7 @@ export async function getTraceChangePoints({ const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); const outcomeAggs = getOutcomeAggregation(documentType); const bucketSizeInSeconds = intervalToSeconds(rollupInterval); + const latencyAggregationType = getLatencyAggregationType(latencyType); const calculateFailedTransactionRate = 'params.successful_or_failed != null && params.successful_or_failed > 0 ? (params.successful_or_failed - params.success) / params.successful_or_failed : 0'; @@ -129,25 +133,14 @@ export async function getTraceChangePoints({ aggs: { ...outcomeAggs, latency: - // Avoid unsupported aggregation on downsampled index, see example error: - // "reason": { - // "type": "unsupported_aggregation_on_downsampled_index", - // "reason": "Field [transaction.duration.summary] of type [aggregate_metric_double] is not supported for aggregation [percentiles]" - // } - durationField !== 'transaction.duration.summary' && - (latencyType === 'p95' || latencyType === 'p99') + // cant calculate percentile aggregation on transaction.duration.summary field + durationField === TRANSACTION_DURATION_SUMMARY ? { - percentiles: { - field: durationField, - percents: [Number(`${latencyType.split('p')[1]}.0`)], - keyed: true, - }, - } - : { avg: { field: durationField, }, - }, + } + : getLatencyAggregation(latencyAggregationType, durationField).latency, failure_rate: documentType === ApmDocumentType.ServiceTransactionMetric ? { From c6d7b6f3f6124cb8612cbd1dd8ffa2cc1b68ba25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Mon, 12 Jan 2026 14:12:30 +0100 Subject: [PATCH 08/17] update get_trace_change_points tool test --- .../tools/get_trace_change_points.spec.ts | 25 ++++++++++--------- .../create_trace_change_points_data.ts | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts index 9e38d3a95b57c..f73f90d452ba2 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_change_points.spec.ts @@ -13,6 +13,7 @@ import type { ChangePoint } from '@kbn/observability-agent-builder-plugin/server import type { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; import { createAgentBuilderApiClient } from '../utils/agent_builder_client'; import { + SERVICE_NAME, TRACE_CHANGE_POINTS_ANALYSIS_WINDOW, createTraceChangePointsData, } from '../utils/synthtrace_scenarios/create_trace_change_points_data'; @@ -52,27 +53,27 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { params: { start: TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.start, end: TRACE_CHANGE_POINTS_ANALYSIS_WINDOW.end, - aggregation: { - field: 'transaction.duration.us', - type: 'p95', - }, }, }); traceChangePoints = toolResults[0]?.data?.changePoints ?? []; }); - it('should detect spike', () => { - expect(traceChangePoints.length).to.be.greaterThan(0); - const spike = traceChangePoints.find((cp: ChangePoint) => cp.changes?.type === 'spike'); - expect(spike).to.be.ok(); + it('should return results grouped by service.name', () => { + const serviceNames = traceChangePoints.find((cp: ChangePoint) => cp.key === SERVICE_NAME); + expect(serviceNames).to.not.be(undefined); + }); + + it('should include changes_latency, changes_throughput, and changes_failure_rate change points results', () => { + traceChangePoints.forEach((cp: ChangePoint) => { + expect(cp).to.have.property('changes_latency'); + expect(cp).to.have.property('changes_throughput'); + expect(cp).to.have.property('changes_failure_rate'); + }); }); it('should include time series data for visualization', () => { traceChangePoints.forEach((cp: ChangePoint) => { - expect(cp).to.have.property('timeSeries'); - expect(cp?.timeSeries?.length).to.be.greaterThan(0); - expect(cp?.timeSeries?.[0]).to.have.property('x'); - expect(cp?.timeSeries?.[0]).to.have.property('y'); + expect(cp).to.have.property('time_series'); }); }); }); diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts index 8417a32acc4bb..0a939237e4e52 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/utils/synthtrace_scenarios/create_trace_change_points_data.ts @@ -9,7 +9,7 @@ import datemath from '@elastic/datemath'; import type { ApmSynthtraceEsClient } from '@kbn/synthtrace'; import { apm, timerange } from '@kbn/synthtrace-client'; -const SERVICE_NAME = 'test-service'; +export const SERVICE_NAME = 'test-service'; export const TRACE_CHANGE_POINTS_INDEX = `traces-apm.app.${SERVICE_NAME}-default`; export const TRACE_CHANGE_POINTS_ANALYSIS_WINDOW = { start: 'now-60m', From ad6eaaac98da56145201e9beb06d1daf9d206f76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Mon, 12 Jan 2026 14:37:13 +0100 Subject: [PATCH 09/17] update README and tool description --- .../tools/get_trace_change_points/README.md | 61 +------------------ .../tools/get_trace_change_points/tool.ts | 17 ++++-- 2 files changed, 14 insertions(+), 64 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md index 17e20f6ab0723..501d608e576b6 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md @@ -1,6 +1,6 @@ # get_trace_change_points -Detects statistically significant changes (e.g., "spike", "dip", "trend_change", "step_change", "distribution_change", "non_stationary", "stationary", or "indeterminable") in trace data over time. Returns the top 25 most significant change points ordered by p-value. +Detects statistically significant traces (latency, throughput, and failure rate) changes (e.g., "spike", "dip", "trend_change", "step_change", "distribution_change", "non_stationary", "stationary", or "indeterminable") in trace data over time. Returns the top 25 most significant change points ordered by p-value. ## Examples @@ -16,62 +16,3 @@ POST kbn://api/agent_builder/tools/_execute } } ``` - -### With explicit index pattern - -``` -POST kbn://api/agent_builder/tools/_execute -{ - "tool_id": "observability.get_trace_change_points", - "tool_params": { - "start": "now-1h", - "end": "now", - "index": "index-example-*" - } -} -``` - -### With custom aggregation (span latency p99) - -``` -POST kbn://api/agent_builder/tools/_execute -{ - "tool_id": "observability.get_trace_change_points", - "tool_params": { - "start": "now-24h", - "end": "now", - "aggregation": { - "field": "span.duration.us", - "type": "p99" - } - } -} -``` - -### With groupBy fields - -``` -POST kbn://api/agent_builder/tools/_execute -{ - "tool_id": "observability.get_trace_change_points", - "tool_params": { - "start": "now-24h", - "end": "now", - "groupBy": ["service.name", "service.environment"] - } -} -``` - -### With KQL filter - -``` -POST kbn://api/agent_builder/tools/_execute -{ - "tool_id": "observability.get_trace_change_points", - "tool_params": { - "start": "now-24h", - "end": "now", - "kqlFilter": "service.name:my-service AND transaction.type:request" - } -} -``` diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts index 41cbe4ed7f102..9ae791abec6ce 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -26,14 +26,13 @@ const getTraceChangePointsSchema = z.object({ kqlFilter: z .string() .describe( - 'A KQL query to filter the trace documents. Examples: trace.id:"abc123", service.name:"my-service"' + 'Optional KQL query to filter the trace documents. Examples: trace.id:"abc123", service.name:"my-service"' ) .optional(), groupBy: z .string() .describe( - `Optional keyword fields to break down metrics by to identify which specific group experienced a change. -Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. Common fields to group by include: + `Field to group results by. Use only low-cardinality fields. Using many fields or high-cardinality fields can cause a large number of groups and severely impact performance. Common fields to group by include: - Service level: 'service.name', 'service.environment', 'service.version' - Transaction level: 'transaction.name', 'transaction.type' - Infrastructure level: 'host.name', 'container.id', 'kubernetes.pod.name' @@ -63,7 +62,17 @@ export function createGetTraceChangePointsTool({ const toolDefinition: BuiltinToolDefinition = { id: OBSERVABILITY_GET_TRACE_CHANGE_POINTS_TOOL_ID, type: ToolType.builtin, - description: `TBD`, + description: `Analyzes traces to detect statistically significant change points in latency, throughput, and failure rate across group (e.g., service, transaction, host). +Trace metrics: +- Latency: avg/p95/p99 response time. +- Throughput: requests per minute. +- Failure rate: percentage of failed transactions. + +Supports optional KQL filtering + +When to use: +- Detecting significant changes in trace behavior (spike, dip, step change, trend change, distribution change, stationary/non‑stationary, indeterminable) and identifying when they occur. +`, schema: getTraceChangePointsSchema, tags: ['observability', 'traces'], handler: async ( From d9601808c47045c3b85f4f5ab3ba61e59f39b18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Tue, 13 Jan 2026 11:38:19 +0100 Subject: [PATCH 10/17] fix useDurationSummaryField logic --- .../tools/get_trace_change_points/index.ts | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts index 098aed804842e..06df433216ea3 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts @@ -11,8 +11,10 @@ import type { ChangePointType } from '@kbn/es-types/src'; import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; import { ApmDocumentType } from '../../../../common/document_type'; -import { TRANSACTION_DURATION_SUMMARY } from '../../../../common/es_fields/apm'; -import { getLatencyAggregationType } from '../../../../common/latency_aggregation_types'; +import { + getLatencyAggregationType, + LatencyAggregationType, +} from '../../../../common/latency_aggregation_types'; import { getLatencyAggregation } from '../../../lib/helpers/latency_aggregation_type'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; @@ -100,10 +102,15 @@ export async function getTraceChangePoints({ const { rollupInterval, hasDurationSummaryField } = source; const documentType = source.documentType as DocumentType; - const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); + const latencyAggregationType = getLatencyAggregationType(latencyType); + // cant calculate percentile aggregation on transaction.duration.summary field + const useDurationSummaryField = + hasDurationSummaryField && + latencyAggregationType !== LatencyAggregationType.p95 && + latencyAggregationType !== LatencyAggregationType.p99; + const durationField = getDurationFieldForTransactions(documentType, useDurationSummaryField); const outcomeAggs = getOutcomeAggregation(documentType); const bucketSizeInSeconds = intervalToSeconds(rollupInterval); - const latencyAggregationType = getLatencyAggregationType(latencyType); const calculateFailedTransactionRate = 'params.successful_or_failed != null && params.successful_or_failed > 0 ? (params.successful_or_failed - params.success) / params.successful_or_failed : 0'; @@ -132,15 +139,7 @@ export async function getTraceChangePoints({ }, aggs: { ...outcomeAggs, - latency: - // cant calculate percentile aggregation on transaction.duration.summary field - durationField === TRANSACTION_DURATION_SUMMARY - ? { - avg: { - field: durationField, - }, - } - : getLatencyAggregation(latencyAggregationType, durationField).latency, + ...getLatencyAggregation(latencyAggregationType, durationField), failure_rate: documentType === ApmDocumentType.ServiceTransactionMetric ? { From 295d862a99ce292a769f44cc9ec356cf16647ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Wed, 14 Jan 2026 12:29:41 +0100 Subject: [PATCH 11/17] refactor after: Move helpers required for trace metrics to the apm-data-access plugin --- .../data_provider/register_data_providers.ts | 26 --- .../tools/get_trace_change_points/index.ts | 191 ----------------- .../plugins/apm_data_access/common/index.ts | 3 + .../common/latency_aggregation_types.ts | 18 ++ .../server/lib/helpers/index.ts | 2 + .../helpers/latency_aggregation_type/index.ts | 26 +++ .../plugins/apm_data_access/server/utils.ts | 1 + .../data_registry/data_registry_types.ts | 48 ----- .../tools/get_trace_change_points/handler.ts | 196 ++++++++++++++++-- .../tools/get_trace_change_points/tool.ts | 7 +- .../server/tools/get_trace_metrics/handler.ts | 31 +-- .../server/tools/register_tools.ts | 2 +- .../server/utils/get_change_points.ts | 2 +- .../utils/get_preferred_document_source.ts | 6 +- 14 files changed, 248 insertions(+), 311 deletions(-) delete mode 100644 x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts create mode 100644 x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts create mode 100644 x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts rename x-pack/solutions/observability/plugins/{apm/server/agent_builder => observability_agent_builder/server}/utils/get_preferred_document_source.ts (91%) diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts index 9c6de7cdf9840..3027ce75d8efd 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts @@ -19,7 +19,6 @@ import { getExitSpanChangePoints, getServiceChangePoints, } from '../../routes/assistant_functions/get_changepoints'; -import { getTraceChangePoints } from '../tools/get_trace_change_points'; import { buildApmToolResources } from '../utils/build_apm_tool_resources'; import type { APMPluginSetupDependencies, APMPluginStartDependencies } from '../../types'; @@ -184,29 +183,4 @@ export function registerDataProviders({ }); } ); - - observabilityAgentBuilder.registerDataProvider( - 'traceChangePoints', - async ({ request, start, end, kqlFilter, groupBy, latencyType }) => { - const { apmEventClient, apmDataAccessServices } = await buildApmToolResources({ - core, - plugins, - request, - logger, - }); - - const startMs = parseDatemath(start); - const endMs = parseDatemath(end); - - return getTraceChangePoints({ - apmEventClient, - apmDataAccessServices, - start: startMs, - end: endMs, - kqlFilter, - groupBy, - latencyType, - }); - } - ); } diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts deleted file mode 100644 index 06df433216ea3..0000000000000 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_change_points/index.ts +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; -import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; -import type { ChangePointType } from '@kbn/es-types/src'; -import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; -import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; -import { ApmDocumentType } from '../../../../common/document_type'; -import { - getLatencyAggregationType, - LatencyAggregationType, -} from '../../../../common/latency_aggregation_types'; -import { getLatencyAggregation } from '../../../lib/helpers/latency_aggregation_type'; -import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; -import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -import { getOutcomeAggregation } from '../../../lib/helpers/transaction_error_rate'; -import { getPreferredDocumentSource } from '../../utils/get_preferred_document_source'; - -interface ChangePointDetails { - change_point?: number; - r_value?: number; - trend?: string; - p_value?: number; -} - -interface Bucket { - key: string | number; - key_as_string?: string; - doc_count: number; -} - -interface ChangePointResult { - type: Record; - bucket?: Bucket; -} - -export interface BucketChangePoints extends Bucket { - changes_latency: ChangePointResult; - changes_throughput: ChangePointResult; - changes_failure_rate: ChangePointResult; - time_series: { - buckets: Array< - Bucket & { - latency: { - value: number | null; - }; - throughput: { - value: number | null; - }; - failure_rate: { - value: number | null; - }; - } - >; - }; -} - -type DocumentType = - | ApmDocumentType.ServiceTransactionMetric - | ApmDocumentType.TransactionMetric - | ApmDocumentType.TransactionEvent; - -function getChangePointsAggs(bucketsPath: string) { - const changePointAggs = { - change_point: { - buckets_path: bucketsPath, - }, - // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 - } as AggregationsAggregationContainer; - return changePointAggs; -} - -export async function getTraceChangePoints({ - apmEventClient, - apmDataAccessServices, - start, - end, - kqlFilter, - groupBy, - latencyType = 'avg', -}: { - apmEventClient: APMEventClient; - apmDataAccessServices: ApmDataAccessServices; - start: number; - end: number; - kqlFilter?: string; - groupBy: string; - latencyType?: 'avg' | 'p95' | 'p99'; -}): Promise { - const source = await getPreferredDocumentSource({ - apmDataAccessServices, - start, - end, - groupBy, - kqlFilter, - }); - - const { rollupInterval, hasDurationSummaryField } = source; - const documentType = source.documentType as DocumentType; - const latencyAggregationType = getLatencyAggregationType(latencyType); - // cant calculate percentile aggregation on transaction.duration.summary field - const useDurationSummaryField = - hasDurationSummaryField && - latencyAggregationType !== LatencyAggregationType.p95 && - latencyAggregationType !== LatencyAggregationType.p99; - const durationField = getDurationFieldForTransactions(documentType, useDurationSummaryField); - const outcomeAggs = getOutcomeAggregation(documentType); - const bucketSizeInSeconds = intervalToSeconds(rollupInterval); - - const calculateFailedTransactionRate = - 'params.successful_or_failed != null && params.successful_or_failed > 0 ? (params.successful_or_failed - params.success) / params.successful_or_failed : 0'; - - const response = await apmEventClient.search('get_trace_change_points', { - apm: { - sources: [{ documentType, rollupInterval }], - }, - size: 0, - track_total_hits: false, - query: { - bool: { - filter: [...rangeQuery(start, end), ...kqlQuery(kqlFilter)], - }, - }, - aggs: { - groups: { - terms: { - field: groupBy, - }, - aggs: { - time_series: { - date_histogram: { - field: '@timestamp', - fixed_interval: `${bucketSizeInSeconds}s`, - }, - aggs: { - ...outcomeAggs, - ...getLatencyAggregation(latencyAggregationType, durationField), - failure_rate: - documentType === ApmDocumentType.ServiceTransactionMetric - ? { - bucket_script: { - buckets_path: { - successful_or_failed: 'successful_or_failed', - success: 'successful', - }, - script: { - source: calculateFailedTransactionRate, - }, - }, - } - : { - bucket_script: { - buckets_path: { - successful_or_failed: 'successful_or_failed>_count', - success: 'successful>_count', - }, - script: { - source: calculateFailedTransactionRate, - }, - }, - }, - throughput: { - bucket_script: { - buckets_path: { - count: '_count', - }, - script: { - source: 'params.count != null ? params.count / (params.bucketSize / 60.0) : 0', - params: { - bucketSize: bucketSizeInSeconds, - }, - }, - }, - }, - }, - }, - changes_latency: getChangePointsAggs('time_series>latency'), - changes_throughput: getChangePointsAggs('time_series>throughput'), - changes_failure_rate: getChangePointsAggs('time_series>failure_rate'), - }, - }, - }, - }); - - return (response.aggregations?.groups?.buckets as BucketChangePoints[]) ?? []; -} diff --git a/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts index df61d8d9c3702..5ff5ea400ffff 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts @@ -17,6 +17,9 @@ export { export type { TimeRangeMetadata } from './time_range_metadata'; +export { LatencyAggregationType } from './latency_aggregation_types'; +export { getLatencyAggregationType } from './latency_aggregation_types'; + export { getPreferredBucketSizeAndDataSource } from './utils/get_preferred_bucket_size_and_data_source'; export { getBucketSize } from './utils/get_bucket_size'; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts b/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts new file mode 100644 index 0000000000000..3f0e3d6f5ff8b --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum LatencyAggregationType { + avg = 'avg', + p99 = 'p99', + p95 = 'p95', +} + +export const getLatencyAggregationType = ( + latencyAggregationType: string | null | undefined +): LatencyAggregationType => { + return (latencyAggregationType ?? LatencyAggregationType.avg) as LatencyAggregationType; +}; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts index b0297af7cec04..647939c4f1cf9 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts @@ -34,3 +34,5 @@ export { calculateFailedTransactionRate, type OutcomeAggregation, } from './transaction_error_rate'; + +export { getLatencyAggregation } from './latency_aggregation_type'; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts new file mode 100644 index 0000000000000..d475f804204ec --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LatencyAggregationType } from '../../../../common/latency_aggregation_types'; + +export function getLatencyAggregation( + latencyAggregationType: LatencyAggregationType, + field: string +) { + return { + latency: { + ...(latencyAggregationType === LatencyAggregationType.avg + ? { avg: { field } } + : { + percentiles: { + field, + percents: [latencyAggregationType === LatencyAggregationType.p95 ? 95 : 99], + }, + }), + }, + }; +} diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts index 32ccc3cb90835..a204f6a1157cf 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts @@ -18,6 +18,7 @@ export { calculateThroughputWithRange, getOutcomeAggregation, calculateFailedTransactionRate, + getLatencyAggregation, type OutcomeAggregation, } from './lib/helpers'; diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts index 89f1ddc76c1d9..78581c3d31543 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts @@ -141,45 +141,6 @@ interface InfraHostsResponse { nodes: InfraEntityMetricsItem[]; } -interface ChangePointDetails { - change_point?: number; - r_value?: number; - trend?: string; - p_value?: number; -} - -interface Bucket { - key: string | number; - key_as_string?: string; - doc_count: number; -} - -interface ChangePointResult { - type: Record; - bucket?: Bucket; -} - -export interface BucketChangePoints extends Bucket { - changes_latency: ChangePointResult; - changes_throughput: ChangePointResult; - changes_failure_rate: ChangePointResult; - time_series: { - buckets: Array< - Bucket & { - latency: { - value: number | null; - }; - throughput: { - value: number | null; - }; - failure_rate: { - value: number | null; - }; - } - >; - }; -} - export interface ObservabilityAgentBuilderDataRegistryTypes { apmErrors: (params: { request: KibanaRequest; @@ -251,13 +212,4 @@ export interface ObservabilityAgentBuilderDataRegistryTypes { query: Record | undefined; hostNames?: string[]; }) => Promise; - - traceChangePoints: (params: { - request: KibanaRequest; - start: string; - end: string; - kqlFilter?: string; - groupBy: string; - latencyType?: 'avg' | 'p95' | 'p99'; - }) => Promise; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts index 9267718167341..d33ec7d368b8c 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -4,33 +4,205 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { KibanaRequest } from '@kbn/core/server'; -import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; +import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import { + ApmDocumentType, + LatencyAggregationType, + getLatencyAggregationType, +} from '@kbn/apm-data-access-plugin/common'; +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import type { ChangePointType } from '@kbn/es-types/src'; +import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; +import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; +import { + getOutcomeAggregation, + getDurationFieldForTransactions, + getLatencyAggregation, +} from '@kbn/apm-data-access-plugin/server/utils'; +import type { + ObservabilityAgentBuilderPluginSetupDependencies, + ObservabilityAgentBuilderPluginStart, + ObservabilityAgentBuilderPluginStartDependencies, +} from '../../types'; +import { parseDatemath } from '../../utils/time'; +import { buildApmResources } from '../../utils/build_apm_resources'; +import { getPreferredDocumentSource } from '../../utils/get_preferred_document_source'; +import type { ChangePointDetails } from '../../utils/get_change_points'; + +interface Bucket { + key: string | number; + key_as_string?: string; + doc_count: number; +} + +interface ChangePointResult { + type: Record; + bucket?: Bucket; +} + +export interface BucketChangePoints extends Bucket { + changes_latency: ChangePointResult; + changes_throughput: ChangePointResult; + changes_failure_rate: ChangePointResult; + time_series: { + buckets: Array< + Bucket & { + latency: { + value: number | null; + }; + throughput: { + value: number | null; + }; + failure_rate: { + value: number | null; + }; + } + >; + }; +} + +type DocumentType = + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent; + +function getChangePointsAggs(bucketsPath: string) { + const changePointAggs = { + change_point: { + buckets_path: bucketsPath, + }, + // elasticsearch@9.0.0 change_point aggregation is missing in the types: https://github.com/elastic/elasticsearch-specification/issues/3671 + } as AggregationsAggregationContainer; + return changePointAggs; +} export async function getToolHandler({ + core, + plugins, request, - dataRegistry, + logger, start, end, - kqlFilter: kqlFilterValue, + kqlFilter, groupBy, latencyType, }: { + core: CoreSetup< + ObservabilityAgentBuilderPluginStartDependencies, + ObservabilityAgentBuilderPluginStart + >; + plugins: ObservabilityAgentBuilderPluginSetupDependencies; request: KibanaRequest; - dataRegistry: ObservabilityAgentBuilderDataRegistry; + logger: Logger; start: string; end: string; kqlFilter?: string; groupBy: string; latencyType: 'avg' | 'p95' | 'p99' | undefined; -}) { - const items = await dataRegistry.getData('traceChangePoints', { +}): Promise { + const { apmEventClient, apmDataAccessServices } = await buildApmResources({ + core, + plugins, request, - start, - end, - kqlFilter: kqlFilterValue, + logger, + }); + + const startMs = parseDatemath(start); + const endMs = parseDatemath(end); + const source = await getPreferredDocumentSource({ + apmDataAccessServices, + start: startMs, + end: endMs, groupBy, - latencyType, + kqlFilter, + }); + + const { rollupInterval, hasDurationSummaryField } = source; + const documentType = source.documentType as DocumentType; + const latencyAggregationType = getLatencyAggregationType(latencyType); + // cant calculate percentile aggregation on transaction.duration.summary field + const useDurationSummaryField = + hasDurationSummaryField && + latencyAggregationType !== LatencyAggregationType.p95 && + latencyAggregationType !== LatencyAggregationType.p99; + const durationField = getDurationFieldForTransactions(documentType, useDurationSummaryField); + const outcomeAggs = getOutcomeAggregation(documentType); + const bucketSizeInSeconds = intervalToSeconds(rollupInterval); + + const calculateFailedTransactionRate = + 'params.successful_or_failed != null && params.successful_or_failed > 0 ? (params.successful_or_failed - params.success) / params.successful_or_failed : 0'; + + const response = await apmEventClient.search('get_trace_change_points', { + apm: { + sources: [{ documentType, rollupInterval }], + }, + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [...rangeQuery(startMs, endMs), ...kqlQuery(kqlFilter)], + }, + }, + aggs: { + groups: { + terms: { + field: groupBy, + }, + aggs: { + time_series: { + date_histogram: { + field: '@timestamp', + fixed_interval: `${bucketSizeInSeconds}s`, + }, + aggs: { + ...outcomeAggs, + ...getLatencyAggregation(latencyAggregationType, durationField), + failure_rate: + documentType === ApmDocumentType.ServiceTransactionMetric + ? { + bucket_script: { + buckets_path: { + successful_or_failed: 'successful_or_failed', + success: 'successful', + }, + script: { + source: calculateFailedTransactionRate, + }, + }, + } + : { + bucket_script: { + buckets_path: { + successful_or_failed: 'successful_or_failed>_count', + success: 'successful>_count', + }, + script: { + source: calculateFailedTransactionRate, + }, + }, + }, + throughput: { + bucket_script: { + buckets_path: { + count: '_count', + }, + script: { + source: 'params.count != null ? params.count / (params.bucketSize / 60.0) : 0', + params: { + bucketSize: bucketSizeInSeconds, + }, + }, + }, + }, + }, + }, + changes_latency: getChangePointsAggs('time_series>latency'), + changes_throughput: getChangePointsAggs('time_series>throughput'), + changes_failure_rate: getChangePointsAggs('time_series>failure_rate'), + }, + }, + }, }); - return items; + + return (response.aggregations?.groups?.buckets as BucketChangePoints[]) ?? []; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts index 9ae791abec6ce..941e5a23e853b 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/tool.ts @@ -14,7 +14,6 @@ import type { ObservabilityAgentBuilderPluginStart, ObservabilityAgentBuilderPluginStartDependencies, } from '../../types'; -import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; import { timeRangeSchemaRequired } from '../../utils/tool_schemas'; import { getToolHandler } from './handler'; @@ -47,7 +46,6 @@ const getTraceChangePointsSchema = z.object({ export function createGetTraceChangePointsTool({ core, - dataRegistry, plugins, logger, }: { @@ -55,7 +53,6 @@ export function createGetTraceChangePointsTool({ ObservabilityAgentBuilderPluginStartDependencies, ObservabilityAgentBuilderPluginStart >; - dataRegistry: ObservabilityAgentBuilderDataRegistry; plugins: ObservabilityAgentBuilderPluginSetupDependencies; logger: Logger; }) { @@ -81,8 +78,10 @@ When to use: ) => { try { const changePoints = await getToolHandler({ + core, + plugins, request, - dataRegistry, + logger, start, end, kqlFilter, diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts index c3680613d243a..ec02ddd0c4656 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts @@ -7,10 +7,6 @@ import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; import type { ApmDocumentType } from '@kbn/apm-data-access-plugin/common'; -import { - getPreferredBucketSizeAndDataSource, - getBucketSize, -} from '@kbn/apm-data-access-plugin/common'; import { calculateFailedTransactionRate, getOutcomeAggregation, @@ -25,6 +21,7 @@ import type { import { parseDatemath } from '../../utils/time'; import { buildApmResources } from '../../utils/build_apm_resources'; import { timeRangeFilter, kqlFilter as buildKqlFilter } from '../../utils/dsl_filters'; +import { getPreferredDocumentSource } from '../../utils/get_preferred_document_source'; export interface TraceMetricsItem { group: string; @@ -68,30 +65,12 @@ export async function getToolHandler({ const startMs = parseDatemath(start); const endMs = parseDatemath(end); - - // Get preferred document source based on groupBy, filter, and data availability - const kueryParts: string[] = []; - if (kqlFilter) { - kueryParts.push(kqlFilter); - } - kueryParts.push(`${groupBy}: *`); - const kuery = kueryParts.join(' AND '); - - const documentSources = await apmDataAccessServices.getDocumentSources({ + const source = await getPreferredDocumentSource({ + apmDataAccessServices, start: startMs, end: endMs, - kuery, - }); - - const { bucketSize } = getBucketSize({ - start: startMs, - end: endMs, - numBuckets: 100, - }); - - const { source } = getPreferredBucketSizeAndDataSource({ - sources: documentSources, - bucketSizeInSeconds: bucketSize, + groupBy, + kqlFilter, }); const { rollupInterval, hasDurationSummaryField } = source; diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts index 7189762da7bff..a7e3af11bc457 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts @@ -111,7 +111,7 @@ export async function registerTools({ createGetTraceMetricsTool({ core, plugins, logger }), createGetLogChangePointsTool({ core, plugins, logger }), createGetMetricChangePointsTool({ core, plugins, logger }), - createGetTraceChangePointsTool({ core, dataRegistry, plugins, logger }), + createGetTraceChangePointsTool({ core, plugins, logger }), ]; for (const tool of observabilityTools) { diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_change_points.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_change_points.ts index 39c9a866c915a..22b1450b2bd6e 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_change_points.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_change_points.ts @@ -7,7 +7,7 @@ import type { ChangePointType } from '@kbn/es-types/src'; -interface ChangePointDetails { +export interface ChangePointDetails { change_point?: number; r_value?: number; trend?: string; diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_preferred_document_source.ts similarity index 91% rename from x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts rename to x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_preferred_document_source.ts index 473c1df20170e..5a934d2f82241 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_preferred_document_source.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/get_preferred_document_source.ts @@ -5,9 +5,11 @@ * 2.0. */ -import { getPreferredBucketSizeAndDataSource } from '@kbn/apm-data-access-plugin/common'; import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; -import { getBucketSize } from '../../../common/utils/get_bucket_size'; +import { + getPreferredBucketSizeAndDataSource, + getBucketSize, +} from '@kbn/apm-data-access-plugin/common'; /** * Gets the preferred document source based on groupBy, filter, and data availability. From 751d5ddec7ea2ec11922dc564a520fe658abd4e4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 14 Jan 2026 11:49:59 +0000 Subject: [PATCH 12/17] Changes from node scripts/lint_ts_projects --fix --- .../plugins/observability_agent_builder/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json b/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json index 58b050c444e63..f76b1131ab88a 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json @@ -43,7 +43,8 @@ "@kbn/licensing-plugin", "@kbn/observability-nav-icons", "@kbn/server-route-repository-client", - "@kbn/agent-builder-browser" + "@kbn/agent-builder-browser", + "@kbn/observability-plugin" ], "exclude": ["target/**/*"] } From 7f4520fa445418c9cc7a8f46dafeebbc353c3133 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 14 Jan 2026 12:01:27 +0000 Subject: [PATCH 13/17] Changes from node scripts/regenerate_moon_projects.js --update --- .../observability/plugins/observability_agent_builder/moon.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml b/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml index 5b40b982ecdbc..f5c37acc6527c 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml @@ -57,6 +57,7 @@ dependsOn: - '@kbn/observability-nav-icons' - '@kbn/server-route-repository-client' - '@kbn/agent-builder-browser' + - '@kbn/observability-plugin' tags: - plugin - prod From b8f22454f24798287fb20ac51354b30a63c7dc5e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 15 Jan 2026 08:16:34 +0000 Subject: [PATCH 14/17] Changes from node scripts/eslint_all_files --no-cache --fix --- .../apis/observability_agent_builder/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts index 7fab43f455ef7..40307b7699ed1 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/index.ts @@ -23,7 +23,7 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./tools/get_metric_change_points.spec.ts')); loadTestFile(require.resolve('./tools/get_trace_change_points.spec.ts')); loadTestFile(require.resolve('./tools/get_index_info.spec.ts')); - + // ai insights loadTestFile(require.resolve('./ai_insights/error.spec.ts')); loadTestFile(require.resolve('./ai_insights/alert.spec.ts')); From 25995231478bbcef5925d1a63f4bee4c092aa3c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Thu, 15 Jan 2026 10:21:11 +0100 Subject: [PATCH 15/17] fix circular dependency --- .../plugins/observability_agent_builder/moon.yml | 1 - .../server/tools/get_trace_change_points/handler.ts | 10 ++++++++-- .../plugins/observability_agent_builder/tsconfig.json | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml b/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml index f5c37acc6527c..5b40b982ecdbc 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/moon.yml @@ -57,7 +57,6 @@ dependsOn: - '@kbn/observability-nav-icons' - '@kbn/server-route-repository-client' - '@kbn/agent-builder-browser' - - '@kbn/observability-plugin' tags: - plugin - prod diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts index d33ec7d368b8c..6be4840bff4d0 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -10,7 +10,6 @@ import { LatencyAggregationType, getLatencyAggregationType, } from '@kbn/apm-data-access-plugin/common'; -import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import type { ChangePointType } from '@kbn/es-types/src'; import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; @@ -24,6 +23,7 @@ import type { ObservabilityAgentBuilderPluginStart, ObservabilityAgentBuilderPluginStartDependencies, } from '../../types'; +import { timeRangeFilter, kqlFilter as buildKqlFilter } from '../../utils/dsl_filters'; import { parseDatemath } from '../../utils/time'; import { buildApmResources } from '../../utils/build_apm_resources'; import { getPreferredDocumentSource } from '../../utils/get_preferred_document_source'; @@ -140,7 +140,13 @@ export async function getToolHandler({ track_total_hits: false, query: { bool: { - filter: [...rangeQuery(startMs, endMs), ...kqlQuery(kqlFilter)], + filter: [ + ...timeRangeFilter('@timestamp', { + start: startMs, + end: endMs, + }), + ...buildKqlFilter(kqlFilter), + ], }, }, aggs: { diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json b/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json index f76b1131ab88a..58b050c444e63 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/tsconfig.json @@ -43,8 +43,7 @@ "@kbn/licensing-plugin", "@kbn/observability-nav-icons", "@kbn/server-route-repository-client", - "@kbn/agent-builder-browser", - "@kbn/observability-plugin" + "@kbn/agent-builder-browser" ], "exclude": ["target/**/*"] } From ffd81274b0401fe6694a1966b501b3247498d522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 16 Jan 2026 09:56:39 +0100 Subject: [PATCH 16/17] Update x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Søren Louv-Jansen --- .../server/tools/get_trace_change_points/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md index 501d608e576b6..d03b47d0c61fe 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md @@ -1,6 +1,6 @@ # get_trace_change_points -Detects statistically significant traces (latency, throughput, and failure rate) changes (e.g., "spike", "dip", "trend_change", "step_change", "distribution_change", "non_stationary", "stationary", or "indeterminable") in trace data over time. Returns the top 25 most significant change points ordered by p-value. +Detects statistically significant changes (e.g., "spike", "dip", "trend_change", "step_change", "distribution_change", "non_stationary", "stationary", or "indeterminable") in trace metrics (latency, throughput, and failure rate). Returns the top 25 most significant change points ordered by p-value. ## Examples From ad0be7564e9e6d6496da0deb423f21d7a0d681ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Lidue=C3=B1a?= Date: Fri, 16 Jan 2026 10:35:42 +0100 Subject: [PATCH 17/17] remove duplicated code --- .../plugins/apm_data_access/common/index.ts | 3 -- .../common/latency_aggregation_types.ts | 18 --------- .../server/lib/helpers/index.ts | 2 - .../helpers/latency_aggregation_type/index.ts | 26 ------------ .../plugins/apm_data_access/server/utils.ts | 1 - .../tools/get_trace_change_points/README.md | 24 +++++++++++ .../tools/get_trace_change_points/handler.ts | 40 +++++++++++-------- 7 files changed, 48 insertions(+), 66 deletions(-) delete mode 100644 x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts delete mode 100644 x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts diff --git a/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts index 5ff5ea400ffff..df61d8d9c3702 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/common/index.ts @@ -17,9 +17,6 @@ export { export type { TimeRangeMetadata } from './time_range_metadata'; -export { LatencyAggregationType } from './latency_aggregation_types'; -export { getLatencyAggregationType } from './latency_aggregation_types'; - export { getPreferredBucketSizeAndDataSource } from './utils/get_preferred_bucket_size_and_data_source'; export { getBucketSize } from './utils/get_bucket_size'; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts b/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts deleted file mode 100644 index 3f0e3d6f5ff8b..0000000000000 --- a/x-pack/solutions/observability/plugins/apm_data_access/common/latency_aggregation_types.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export enum LatencyAggregationType { - avg = 'avg', - p99 = 'p99', - p95 = 'p95', -} - -export const getLatencyAggregationType = ( - latencyAggregationType: string | null | undefined -): LatencyAggregationType => { - return (latencyAggregationType ?? LatencyAggregationType.avg) as LatencyAggregationType; -}; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts index 647939c4f1cf9..b0297af7cec04 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts @@ -34,5 +34,3 @@ export { calculateFailedTransactionRate, type OutcomeAggregation, } from './transaction_error_rate'; - -export { getLatencyAggregation } from './latency_aggregation_type'; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts deleted file mode 100644 index d475f804204ec..0000000000000 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/latency_aggregation_type/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { LatencyAggregationType } from '../../../../common/latency_aggregation_types'; - -export function getLatencyAggregation( - latencyAggregationType: LatencyAggregationType, - field: string -) { - return { - latency: { - ...(latencyAggregationType === LatencyAggregationType.avg - ? { avg: { field } } - : { - percentiles: { - field, - percents: [latencyAggregationType === LatencyAggregationType.p95 ? 95 : 99], - }, - }), - }, - }; -} diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts index a204f6a1157cf..32ccc3cb90835 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts @@ -18,7 +18,6 @@ export { calculateThroughputWithRange, getOutcomeAggregation, calculateFailedTransactionRate, - getLatencyAggregation, type OutcomeAggregation, } from './lib/helpers'; diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md index d03b47d0c61fe..085f1fdcc55e8 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/README.md @@ -16,3 +16,27 @@ POST kbn://api/agent_builder/tools/_execute } } ``` + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-1h", + "end": "now", + "latencyType": "p95" + } +} +``` + +``` +POST kbn://api/agent_builder/tools/_execute +{ + "tool_id": "observability.get_trace_change_points", + "tool_params": { + "start": "now-1h", + "end": "now", + "groupBy": "transaction.name" + } +} +``` diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts index 6be4840bff4d0..dab3f6f6dd78e 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_change_points/handler.ts @@ -5,18 +5,13 @@ * 2.0. */ import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; -import { - ApmDocumentType, - LatencyAggregationType, - getLatencyAggregationType, -} from '@kbn/apm-data-access-plugin/common'; +import { ApmDocumentType } from '@kbn/apm-data-access-plugin/common'; import type { ChangePointType } from '@kbn/es-types/src'; import type { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; import { intervalToSeconds } from '@kbn/apm-data-access-plugin/common/utils/get_preferred_bucket_size_and_data_source'; import { getOutcomeAggregation, getDurationFieldForTransactions, - getLatencyAggregation, } from '@kbn/apm-data-access-plugin/server/utils'; import type { ObservabilityAgentBuilderPluginSetupDependencies, @@ -40,7 +35,7 @@ interface ChangePointResult { bucket?: Bucket; } -export interface BucketChangePoints extends Bucket { +interface BucketChangePoints extends Bucket { changes_latency: ChangePointResult; changes_throughput: ChangePointResult; changes_failure_rate: ChangePointResult; @@ -61,6 +56,8 @@ export interface BucketChangePoints extends Bucket { }; } +type LatencyAggregationType = 'avg' | 'p99' | 'p95'; + type DocumentType = | ApmDocumentType.ServiceTransactionMetric | ApmDocumentType.TransactionMetric @@ -76,6 +73,21 @@ function getChangePointsAggs(bucketsPath: string) { return changePointAggs; } +function getLatencyAggregation(latencyAggregationType: LatencyAggregationType, field: string) { + return { + latency: { + ...(latencyAggregationType === 'avg' + ? { avg: { field } } + : { + percentiles: { + field, + percents: [latencyAggregationType === 'p95' ? 95 : 99], + }, + }), + }, + }; +} + export async function getToolHandler({ core, plugins, @@ -85,7 +97,7 @@ export async function getToolHandler({ end, kqlFilter, groupBy, - latencyType, + latencyType = 'avg', }: { core: CoreSetup< ObservabilityAgentBuilderPluginStartDependencies, @@ -98,7 +110,7 @@ export async function getToolHandler({ end: string; kqlFilter?: string; groupBy: string; - latencyType: 'avg' | 'p95' | 'p99' | undefined; + latencyType: LatencyAggregationType | undefined; }): Promise { const { apmEventClient, apmDataAccessServices } = await buildApmResources({ core, @@ -119,14 +131,10 @@ export async function getToolHandler({ const { rollupInterval, hasDurationSummaryField } = source; const documentType = source.documentType as DocumentType; - const latencyAggregationType = getLatencyAggregationType(latencyType); // cant calculate percentile aggregation on transaction.duration.summary field const useDurationSummaryField = - hasDurationSummaryField && - latencyAggregationType !== LatencyAggregationType.p95 && - latencyAggregationType !== LatencyAggregationType.p99; + hasDurationSummaryField && latencyType !== 'p95' && latencyType !== 'p99'; const durationField = getDurationFieldForTransactions(documentType, useDurationSummaryField); - const outcomeAggs = getOutcomeAggregation(documentType); const bucketSizeInSeconds = intervalToSeconds(rollupInterval); const calculateFailedTransactionRate = @@ -161,8 +169,8 @@ export async function getToolHandler({ fixed_interval: `${bucketSizeInSeconds}s`, }, aggs: { - ...outcomeAggs, - ...getLatencyAggregation(latencyAggregationType, durationField), + ...getOutcomeAggregation(documentType), + ...getLatencyAggregation(latencyType, durationField), failure_rate: documentType === ApmDocumentType.ServiceTransactionMetric ? {