diff --git a/x-pack/solutions/observability/plugins/apm/moon.yml b/x-pack/solutions/observability/plugins/apm/moon.yml index 5f0379e308406..52a5b7fc7b2c0 100644 --- a/x-pack/solutions/observability/plugins/apm/moon.yml +++ b/x-pack/solutions/observability/plugins/apm/moon.yml @@ -150,7 +150,6 @@ dependsOn: - '@kbn/react-query' - '@kbn/core-chrome-layout-constants' - '@kbn/control-group-renderer' - - '@kbn/zod' - '@kbn/agent-builder-plugin' - '@kbn/observability-agent-builder-plugin' tags: diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts index ed69f55d6eae6..3027ce75d8efd 100644 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts +++ b/x-pack/solutions/observability/plugins/apm/server/agent_builder/data_provider/register_data_providers.ts @@ -6,9 +6,9 @@ */ import type { CoreSetup, Logger } from '@kbn/core/server'; +import { getRollupIntervalForTimeRange } from '@kbn/apm-data-access-plugin/server/utils'; import { getErrorSampleDetails } from '../../routes/errors/get_error_groups/get_error_sample_details'; import { parseDatemath } from '../utils/time'; -import { getRollupIntervalForTimeRange } from '../utils/get_rollup_interval_for_time_range'; import { getApmServiceSummary } from '../../routes/assistant_functions/get_apm_service_summary'; import { getApmDownstreamDependencies } from '../../routes/assistant_functions/get_apm_downstream_dependencies'; import { getServicesItems } from '../../routes/services/get_services/get_services_items'; @@ -19,7 +19,6 @@ import { getExitSpanChangePoints, getServiceChangePoints, } from '../../routes/assistant_functions/get_changepoints'; -import { getTraceMetrics } from '../tools/get_trace_metrics'; import { buildApmToolResources } from '../utils/build_apm_tool_resources'; import type { APMPluginSetupDependencies, APMPluginStartDependencies } from '../../types'; @@ -184,28 +183,4 @@ export function registerDataProviders({ }); } ); - - observabilityAgentBuilder.registerDataProvider( - 'traceMetrics', - async ({ request, start, end, kqlFilter, groupBy }) => { - const { apmEventClient, apmDataAccessServices } = await buildApmToolResources({ - core, - plugins, - request, - logger, - }); - - const startMs = parseDatemath(start); - const endMs = parseDatemath(end); - - return getTraceMetrics({ - apmEventClient, - apmDataAccessServices, - start: startMs, - end: endMs, - kqlFilter, - groupBy, - }); - } - ); } diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_metrics/index.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_metrics/index.ts deleted file mode 100644 index 0cf9b0071dcc3..0000000000000 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/tools/get_trace_metrics/index.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; -import type { ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; -import { getPreferredBucketSizeAndDataSource } from '@kbn/apm-data-access-plugin/common'; -import type { ApmDocumentType } from '../../../../common/document_type'; -import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; -import { - calculateFailedTransactionRate, - getOutcomeAggregation, -} from '../../../lib/helpers/transaction_error_rate'; -import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput'; -import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -import { getBucketSize } from '../../../../common/utils/get_bucket_size'; - -const MAX_NUMBER_OF_GROUPS = 100; - -export interface TraceMetricsItem { - group: string; - latency: number | null; - throughput: number; - failureRate: number; -} - -export type GetTraceMetricsResponse = TraceMetricsItem[]; - -/** - * Gets the preferred document source based on groupBy, filter, and data availability. - * - * Uses getDocumentSources to determine which document types have data for the given - * filter and groupBy field. This automatically handles: - * - ServiceTransactionMetric: Most efficient, but only has service.name, service.environment, transaction.type - * - TransactionMetric: Has more dimensions (transaction.*, host.*, container.*, kubernetes.*, cloud.*, faas.*, etc.) - * - TransactionEvent: Raw transaction docs, fallback when metrics don't have required fields - */ -async function getPreferredDocumentSource({ - apmDataAccessServices, - start, - end, - groupBy, - kqlFilter, -}: { - apmDataAccessServices: ApmDataAccessServices; - start: number; - end: number; - groupBy: string; - kqlFilter?: string; -}) { - const kueryParts: string[] = []; - if (kqlFilter) { - kueryParts.push(kqlFilter); - } - kueryParts.push(`${groupBy}: *`); - const kuery = kueryParts.join(' AND '); - - const documentSources = await apmDataAccessServices.getDocumentSources({ - start, - end, - kuery, - }); - - const { bucketSize } = getBucketSize({ - start, - end, - numBuckets: 100, - }); - - const { source } = getPreferredBucketSizeAndDataSource({ - sources: documentSources, - bucketSizeInSeconds: bucketSize, - }); - - return source; -} - -export async function getTraceMetrics({ - apmEventClient, - apmDataAccessServices, - start, - end, - kqlFilter, - groupBy, -}: { - apmEventClient: APMEventClient; - apmDataAccessServices: ApmDataAccessServices; - start: number; - end: number; - kqlFilter?: string; - groupBy: string; -}): Promise { - const source = await getPreferredDocumentSource({ - apmDataAccessServices, - start, - end, - groupBy, - kqlFilter, - }); - - const { rollupInterval, hasDurationSummaryField } = source; - const documentType = source.documentType as - | ApmDocumentType.ServiceTransactionMetric - | ApmDocumentType.TransactionMetric - | ApmDocumentType.TransactionEvent; - - const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); - const outcomeAggs = getOutcomeAggregation(documentType); - - const response = await apmEventClient.search('get_trace_metrics', { - apm: { - sources: [{ documentType, rollupInterval }], - }, - size: 0, - track_total_hits: false, - query: { - bool: { - filter: [...rangeQuery(start, end), ...kqlQuery(kqlFilter)], - }, - }, - aggs: { - groups: { - terms: { - field: groupBy, - size: MAX_NUMBER_OF_GROUPS, - }, - aggs: { - avg_latency: { - avg: { - field: durationField, - }, - }, - ...outcomeAggs, - }, - }, - }, - }); - - const buckets = response.aggregations?.groups?.buckets ?? []; - - const items: TraceMetricsItem[] = buckets.map((bucket) => { - const docCount = bucket.doc_count; - const latencyValue = bucket.avg_latency?.value; - - const latencyMs = - latencyValue !== null && latencyValue !== undefined ? latencyValue / 1000 : null; - - const failureRate = calculateFailedTransactionRate(bucket); - - const throughput = calculateThroughputWithRange({ - start, - end, - value: docCount, - }); - - return { - group: bucket.key as string, - latency: latencyMs, - throughput, - failureRate, - }; - }); - - return items; -} diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/tool_schemas.ts b/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/tool_schemas.ts deleted file mode 100644 index 2782712706e3b..0000000000000 --- a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/tool_schemas.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; - -export const timeRangeSchema = z.object({ - start: z - .string() - .describe('The start of the time range, in Elasticsearch date math, like `now-24h`.'), - end: z.string().describe('The end of the time range, in Elasticsearch date math, like `now`.'), -}); diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts index 713a6aa5c86de..c8674f21576a6 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts @@ -7,10 +7,10 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { merge } from 'lodash'; +import { calculateThroughputWithRange } from '@kbn/apm-data-access-plugin/server/utils'; import type { ValuesType } from 'utility-types'; import { joinByKey } from '../../../../common/utils/join_by_key'; import { withApmSpan } from '../../../utils/with_apm_span'; -import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import type { APMEventClient } from '../../helpers/create_es_client/create_apm_event_client'; import type { RandomSampler } from '../../helpers/get_random_sampler'; import { getDestinationMap } from './get_destination_map'; diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transaction_error_rate.ts b/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transaction_error_rate.ts index 3121e4007044f..706ab9a5874cc 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transaction_error_rate.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transaction_error_rate.ts @@ -5,92 +5,11 @@ * 2.0. */ -import type { - AggregationsSumAggregation, - AggregationsValueCountAggregation, - QueryDslQueryContainer, -} from '@elastic/elasticsearch/lib/api/types'; -import type { - AggregationOptionsByType, - AggregationResultOf, - AggregationResultOfMap, -} from '@kbn/es-types'; -import { ApmDocumentType } from '../../../common/document_type'; -import { EVENT_OUTCOME, EVENT_SUCCESS_COUNT } from '../../../common/es_fields/apm'; -import { EventOutcome } from '../../../common/event_outcome'; - -export const getOutcomeAggregation = ( - documentType: ApmDocumentType -): { - successful_or_failed: - | { value_count: AggregationsValueCountAggregation } - | { filter: QueryDslQueryContainer }; - successful: { sum: AggregationsSumAggregation } | { filter: QueryDslQueryContainer }; -} => { - if (documentType === ApmDocumentType.ServiceTransactionMetric) { - return { - successful_or_failed: { - value_count: { - field: EVENT_SUCCESS_COUNT, - }, - }, - successful: { - sum: { - field: EVENT_SUCCESS_COUNT, - }, - }, - }; - } - - return { - successful_or_failed: { - filter: { - bool: { - filter: [ - { - terms: { - [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], - }, - }, - ], - }, - }, - }, - successful: { - filter: { - bool: { - filter: [ - { - terms: { - [EVENT_OUTCOME]: [EventOutcome.success], - }, - }, - ], - }, - }, - }, - }; -}; - -type OutcomeAggregation = ReturnType; - -export function calculateFailedTransactionRate( - outcomeResponse: AggregationResultOfMap -) { - const successfulTransactions = - 'value' in outcomeResponse.successful - ? outcomeResponse.successful.value ?? 0 - : outcomeResponse.successful.doc_count; - - const successfulOrFailedTransactions = - 'value' in outcomeResponse.successful_or_failed - ? outcomeResponse.successful_or_failed.value - : outcomeResponse.successful_or_failed.doc_count; - - const failedTransactions = successfulOrFailedTransactions - successfulTransactions; - - return failedTransactions / successfulOrFailedTransactions; -} +import type { AggregationOptionsByType, AggregationResultOf } from '@kbn/es-types'; +import { + calculateFailedTransactionRate, + type OutcomeAggregation, +} from '@kbn/apm-data-access-plugin/server/utils'; export function getFailedTransactionRateTimeSeries( buckets: AggregationResultOf< diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transactions/index.ts b/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transactions/index.ts index ed2c9d8dca749..d721f55223fe1 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transactions/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/helpers/transactions/index.ts @@ -5,54 +5,23 @@ * 2.0. */ -import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { getHasTransactionsEvents } from '@kbn/apm-data-access-plugin/server/utils'; import { SearchAggregatedTransactionSetting } from '../../../../common/aggregated_transactions'; import { - TRANSACTION_DURATION, - TRANSACTION_DURATION_HISTOGRAM, TRANSACTION_ROOT, PARENT_ID, TRANSACTION_DURATION_SUMMARY, } from '../../../../common/es_fields/apm'; import type { APMConfig } from '../../..'; import type { APMEventClient } from '../create_es_client/create_apm_event_client'; -import { ApmDocumentType } from '../../../../common/document_type'; -export { getBackwardCompatibleDocumentTypeFilter } from '@kbn/apm-data-access-plugin/server/utils'; - -export async function getHasTransactionsEvents({ - start, - end, - apmEventClient, - kuery, -}: { - start?: number; - end?: number; - apmEventClient: APMEventClient; - kuery?: string; -}) { - const response = await apmEventClient.search('get_has_aggregated_transactions', { - apm: { - events: [ProcessorEvent.metric], - }, - track_total_hits: 1, - terminate_after: 1, - size: 0, - query: { - bool: { - filter: [ - { exists: { field: TRANSACTION_DURATION_HISTOGRAM } }, - ...(start && end ? rangeQuery(start, end) : []), - ...kqlQuery(kuery), - ], - }, - }, - }); - - return response.hits.total.value > 0; -} +export { + getBackwardCompatibleDocumentTypeFilter, + isSummaryFieldSupportedByDocType, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; export async function getSearchTransactionsEvents({ config, @@ -84,45 +53,6 @@ export async function getSearchTransactionsEvents({ } } -export function isSummaryFieldSupportedByDocType( - typeOrSearchAgggregatedTransactions: - | ApmDocumentType.ServiceTransactionMetric - | ApmDocumentType.TransactionMetric - | ApmDocumentType.TransactionEvent - | boolean -) { - let type: ApmDocumentType; - - if (typeOrSearchAgggregatedTransactions === true) { - type = ApmDocumentType.TransactionMetric; - } else if (typeOrSearchAgggregatedTransactions === false) { - type = ApmDocumentType.TransactionEvent; - } else { - type = typeOrSearchAgggregatedTransactions; - } - - return ( - type === ApmDocumentType.ServiceTransactionMetric || type === ApmDocumentType.TransactionMetric - ); -} -export function getDurationFieldForTransactions( - typeOrSearchAgggregatedTransactions: - | ApmDocumentType.ServiceTransactionMetric - | ApmDocumentType.TransactionMetric - | ApmDocumentType.TransactionEvent - | boolean, - useDurationSummaryField?: boolean -) { - if (isSummaryFieldSupportedByDocType(typeOrSearchAgggregatedTransactions)) { - if (useDurationSummaryField) { - return TRANSACTION_DURATION_SUMMARY; - } - return TRANSACTION_DURATION_HISTOGRAM; - } - - return TRANSACTION_DURATION; -} - export function getProcessorEventForTransactions( searchAggregatedTransactions: boolean ): ProcessorEvent.metric | ProcessorEvent.transaction { diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts b/x-pack/solutions/observability/plugins/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts index 167aef0fb1266..c3c8fb395ac0f 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts @@ -6,6 +6,11 @@ */ import type { BoolQuery } from '@kbn/es-query'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; +import { + calculateFailedTransactionRate, + getOutcomeAggregation, +} from '@kbn/apm-data-access-plugin/server/utils'; +import { getFailedTransactionRateTimeSeries } from '../helpers/transaction_error_rate'; import type { ApmServiceTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE } from '../../../common/es_fields/apm'; import type { RollupInterval } from '../../../common/rollup'; @@ -13,11 +18,6 @@ import { environmentQuery } from '../../../common/utils/environment_query'; import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; import type { Coordinate } from '../../../typings/timeseries'; import type { APMEventClient } from '../helpers/create_es_client/create_apm_event_client'; -import { - calculateFailedTransactionRate, - getFailedTransactionRateTimeSeries, - getOutcomeAggregation, -} from '../helpers/transaction_error_rate'; export async function getFailedTransactionRate({ environment, diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/average_or_percentile_agg.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/average_or_percentile_agg.ts index 40878fe079772..d0b828fcfae49 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/average_or_percentile_agg.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/average_or_percentile_agg.ts @@ -4,8 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { getDurationFieldForTransactions } from '@kbn/apm-data-access-plugin/server/utils'; import { AggregationType } from '../../../../../common/rules/apm_rule_types'; -import type { getDurationFieldForTransactions } from '../../../../lib/helpers/transactions'; type TransactionDurationField = ReturnType; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/get_transaction_duration_chart_preview.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/get_transaction_duration_chart_preview.ts index dae1591bf5167..feef0def06c2b 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/get_transaction_duration_chart_preview.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/get_transaction_duration_chart_preview.ts @@ -8,6 +8,7 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { getParsedFilterQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ApmRuleType } from '@kbn/rule-data-utils'; +import { getDurationFieldForTransactions } from '@kbn/apm-data-access-plugin/server/utils'; import { AggregationType } from '../../../../../common/rules/apm_rule_types'; import { SERVICE_NAME, @@ -19,7 +20,6 @@ import type { AlertParams, PreviewChartResponse } from '../../route'; import { getSearchTransactionsEvents, getBackwardCompatibleDocumentTypeFilter, - getDurationFieldForTransactions, getProcessorEventForTransactions, } from '../../../../lib/helpers/transactions'; import { averageOrPercentileAgg, getMultiTermsSortOrder } from './average_or_percentile_agg'; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 2386836edb590..691ff6e5ea5ca 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -37,6 +37,7 @@ import type { ObservabilityApmAlert } from '@kbn/alerts-as-data-utils'; import { addSpaceIdToPath } from '@kbn/spaces-plugin/common'; import { transactionDurationParamsSchema } from '@kbn/response-ops-rule-params/transaction_duration'; import { unflattenObject } from '@kbn/object-utils'; +import { getDurationFieldForTransactions } from '@kbn/apm-data-access-plugin/server/utils'; import { getGroupByTerms } from '../utils/get_groupby_terms'; import { SearchAggregatedTransactionSetting } from '../../../../../common/aggregated_transactions'; import { getEnvironmentEsField } from '../../../../../common/environment_filter_values'; @@ -62,10 +63,7 @@ import { getAlertUrlTransaction, getDurationFormatter, } from '../../../../../common/utils/formatters'; -import { - getBackwardCompatibleDocumentTypeFilter, - getDurationFieldForTransactions, -} from '../../../../lib/helpers/transactions'; +import { getBackwardCompatibleDocumentTypeFilter } from '../../../../lib/helpers/transactions'; import { apmActionVariables } from '../../action_variables'; import { alertingEsClient } from '../../alerting_es_client'; import type { RegisterRuleDependencies } from '../../register_apm_rule_types'; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_timeseries/get_transaction_failure_rate.ts b/x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_timeseries/get_transaction_failure_rate.ts index 41193c4b096df..a17b07083765b 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_timeseries/get_transaction_failure_rate.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_timeseries/get_transaction_failure_rate.ts @@ -7,11 +7,11 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { termQuery } from '@kbn/observability-plugin/server'; +import { getOutcomeAggregation } from '@kbn/apm-data-access-plugin/server/utils'; import { ApmDocumentType } from '../../../../common/document_type'; import { TRANSACTION_NAME, TRANSACTION_TYPE } from '../../../../common/es_fields/apm'; import { RollupInterval } from '../../../../common/rollup'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; -import { getOutcomeAggregation } from '../../../lib/helpers/transaction_error_rate'; import { fetchSeries } from './fetch_timeseries'; export async function getTransactionFailureRate({ diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependency_operations.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependency_operations.ts index 2004448735e24..33b4c8e0c10ec 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependency_operations.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependency_operations.ts @@ -7,6 +7,7 @@ import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { isFiniteNumber } from '@kbn/observability-plugin/common/utils/is_finite_number'; +import { calculateThroughputWithRange } from '@kbn/apm-data-access-plugin/server/utils'; import { EVENT_OUTCOME, SPAN_DESTINATION_SERVICE_RESOURCE, @@ -18,7 +19,6 @@ import type { Environment } from '../../../common/environment_rt'; import { EventOutcome } from '../../../common/event_outcome'; import { environmentQuery } from '../../../common/utils/environment_query'; import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import { getBucketSizeForAggregatedTransactions } from '../../lib/helpers/get_bucket_size_for_aggregated_transactions'; import { getDocumentTypeFilterForServiceDestinationStatistics, diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/mobile/get_mobile_main_statistics_by_field.ts b/x-pack/solutions/observability/plugins/apm/server/routes/mobile/get_mobile_main_statistics_by_field.ts index 06ad7123dfd46..5eccb847d9de1 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/mobile/get_mobile_main_statistics_by_field.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/mobile/get_mobile_main_statistics_by_field.ts @@ -7,6 +7,7 @@ import { termQuery, kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { merge } from 'lodash'; +import { calculateThroughputWithRange } from '@kbn/apm-data-access-plugin/server/utils'; import { SERVICE_NAME, SESSION_ID, @@ -17,7 +18,6 @@ import { environmentQuery } from '../../../common/utils/environment_query'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getLatencyValue } from '../../lib/helpers/latency_aggregation_type'; import { LatencyAggregationType } from '../../../common/latency_aggregation_types'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import { ApmDocumentType } from '../../../common/document_type'; import { RollupInterval } from '../../../common/rollup'; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/observability_overview/get_transactions_per_minute.ts b/x-pack/solutions/observability/plugins/apm/server/routes/observability_overview/get_transactions_per_minute.ts index fa4a7d287623f..b75aea6a566e5 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/observability_overview/get_transactions_per_minute.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/observability_overview/get_transactions_per_minute.ts @@ -6,13 +6,13 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; -import { isDefaultTransactionType } from '../../../common/transaction_types'; -import { TRANSACTION_TYPE } from '../../../common/es_fields/apm'; import { getBackwardCompatibleDocumentTypeFilter, - getProcessorEventForTransactions, -} from '../../lib/helpers/transactions'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; + calculateThroughputWithRange, +} from '@kbn/apm-data-access-plugin/server/utils'; +import { isDefaultTransactionType } from '../../../common/transaction_types'; +import { TRANSACTION_TYPE } from '../../../common/es_fields/apm'; +import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export async function getTransactionsPerMinute({ diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_dependency_node_info.ts b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_dependency_node_info.ts index 48752572cabea..ccd91e3c56be7 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_dependency_node_info.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map_dependency_node_info.ts @@ -7,6 +7,12 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { + calculateThroughputWithRange, + calculateFailedTransactionRate, + getOutcomeAggregation, +} from '@kbn/apm-data-access-plugin/server/utils'; +import { getFailedTransactionRateTimeSeries } from '../../lib/helpers/transaction_error_rate'; import { ApmDocumentType } from '../../../common/document_type'; import { SPAN_DESTINATION_SERVICE_RESOURCE, @@ -17,14 +23,8 @@ import type { NodeStats } from '../../../common/service_map'; import { environmentQuery } from '../../../common/utils/environment_query'; import { getBucketSize } from '../../../common/utils/get_bucket_size'; import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getDocumentTypeFilterForServiceDestinationStatistics } from '../../lib/helpers/spans/get_is_using_service_destination_metrics'; -import { - calculateFailedTransactionRate, - getFailedTransactionRateTimeSeries, - getOutcomeAggregation, -} from '../../lib/helpers/transaction_error_rate'; import { withApmSpan } from '../../utils/with_apm_span'; interface Options { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_instances/get_service_instances_transaction_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_instances/get_service_instances_transaction_statistics.ts index 94c92cf7395de..58a77acf852e6 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_instances/get_service_instances_transaction_statistics.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_instances/get_service_instances_transaction_statistics.ts @@ -5,6 +5,10 @@ * 2.0. */ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { + getDurationFieldForTransactions, + calculateThroughputWithRange, +} from '@kbn/apm-data-access-plugin/server/utils'; import { EVENT_OUTCOME, SERVICE_NAME, @@ -18,10 +22,8 @@ import type { Coordinate } from '../../../../typings/timeseries'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, - getDurationFieldForTransactions, getProcessorEventForTransactions, } from '../../../lib/helpers/transactions'; -import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput'; import { getBucketSizeForAggregatedTransactions } from '../../../lib/helpers/get_bucket_size_for_aggregated_transactions'; import { getLatencyAggregation, diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts index 5b7ad1cfac87a..ad0cbc5193326 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts @@ -7,6 +7,11 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; +import { + calculateFailedTransactionRate, + getOutcomeAggregation, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; import type { ApmTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE } from '../../../common/es_fields/apm'; import type { LatencyAggregationType } from '../../../common/latency_aggregation_types'; @@ -16,11 +21,6 @@ import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_pr import type { Coordinate } from '../../../typings/timeseries'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getLatencyAggregation, getLatencyValue } from '../../lib/helpers/latency_aggregation_type'; -import { getDurationFieldForTransactions } from '../../lib/helpers/transactions'; -import { - calculateFailedTransactionRate, - getOutcomeAggregation, -} from '../../lib/helpers/transaction_error_rate'; import type { RollupInterval } from '../../../common/rollup'; interface ServiceTransactionGroupDetailedStat { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_groups.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_groups.ts index 97cea2eb68ce1..e56d87c70c0da 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_groups.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_transaction_groups.ts @@ -6,6 +6,12 @@ */ import { kqlQuery, rangeQuery, wildcardQuery } from '@kbn/observability-plugin/server'; +import { + calculateThroughputWithRange, + calculateFailedTransactionRate, + getOutcomeAggregation, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; import type { ApmTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, @@ -16,14 +22,8 @@ import { import type { LatencyAggregationType } from '../../../common/latency_aggregation_types'; import type { RollupInterval } from '../../../common/rollup'; import { environmentQuery } from '../../../common/utils/environment_query'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getLatencyAggregation, getLatencyValue } from '../../lib/helpers/latency_aggregation_type'; -import { getDurationFieldForTransactions } from '../../lib/helpers/transactions'; -import { - calculateFailedTransactionRate, - getOutcomeAggregation, -} from '../../lib/helpers/transaction_error_rate'; const txGroupsDroppedBucketName = '_other'; export const MAX_NUMBER_OF_TX_GROUPS = 1_000; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts index 89aa2f2da908f..56db7c721bd26 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts @@ -7,6 +7,12 @@ import { kqlQuery, rangeQuery, wildcardQuery } from '@kbn/observability-plugin/server'; import { getAgentName } from '@kbn/elastic-agent-utils'; +import { + calculateThroughputWithRange, + calculateFailedTransactionRate, + getOutcomeAggregation, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; import type { ApmDocumentType } from '../../../../common/document_type'; import { AGENT_NAME, @@ -22,14 +28,8 @@ import type { ServiceGroup } from '../../../../common/service_groups'; import { isDefaultTransactionType } from '../../../../common/transaction_types'; import { environmentQuery } from '../../../../common/utils/environment_query'; import type { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import type { RandomSampler } from '../../../lib/helpers/get_random_sampler'; -import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -import { - calculateFailedTransactionRate, - getOutcomeAggregation, -} from '../../../lib/helpers/transaction_error_rate'; import { maybe } from '../../../../common/utils/maybe'; import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow'; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts index b4d456216a9b4..e96e93dc9f600 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts @@ -7,20 +7,20 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; +import { + calculateFailedTransactionRate, + getOutcomeAggregation, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; +import { calculateThroughputWithInterval } from '../../../lib/helpers/calculate_throughput'; import type { ApmServiceTransactionDocumentType } from '../../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_TYPE } from '../../../../common/es_fields/apm'; import type { RollupInterval } from '../../../../common/rollup'; import { isDefaultTransactionType } from '../../../../common/transaction_types'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms'; -import { calculateThroughputWithInterval } from '../../../lib/helpers/calculate_throughput'; import type { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import type { RandomSampler } from '../../../lib/helpers/get_random_sampler'; -import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -import { - calculateFailedTransactionRate, - getOutcomeAggregation, -} from '../../../lib/helpers/transaction_error_rate'; import { withApmSpan } from '../../../utils/with_apm_span'; import { maybe } from '../../../../common/utils/maybe'; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/storage_explorer/get_summary_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/storage_explorer/get_summary_statistics.ts index 916807a2231b6..b7bba7c1bf720 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/storage_explorer/get_summary_statistics.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/storage_explorer/get_summary_statistics.ts @@ -7,6 +7,10 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { termQuery, kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { + getDurationFieldForTransactions, + calculateThroughputWithRange, +} from '@kbn/apm-data-access-plugin/server/utils'; import { getTotalIndicesStats, getEstimatedSizeForDocumentsInIndex, @@ -23,10 +27,8 @@ import { environmentQuery } from '../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, getProcessorEventForTransactions, - getDurationFieldForTransactions, isRootTransaction, } from '../../lib/helpers/transactions'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; interface SharedOptions { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts b/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts index 6b43717c77282..de2d5c06cf94f 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts @@ -7,14 +7,16 @@ import { sortBy } from 'lodash'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; +import { + calculateThroughputWithRange, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; import type { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; import { withApmSpan } from '../../utils/with_apm_span'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { environmentQuery } from '../../../common/utils/environment_query'; import { calculateImpactBuilder } from './calculate_impact_builder'; -import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput'; import { - getDurationFieldForTransactions, getBackwardCompatibleDocumentTypeFilter, getProcessorEventForTransactions, isRootTransaction, diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/transactions/get_latency_charts/index.ts b/x-pack/solutions/observability/plugins/apm/server/routes/transactions/get_latency_charts/index.ts index 8ba492a7457a3..88a366d0a4682 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/transactions/get_latency_charts/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/transactions/get_latency_charts/index.ts @@ -6,6 +6,7 @@ */ import type { BoolQuery } from '@kbn/es-query'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; +import { getDurationFieldForTransactions } from '@kbn/apm-data-access-plugin/server/utils'; import type { ApmServiceTransactionDocumentType } from '../../../../common/document_type'; import { FAAS_ID, @@ -24,7 +25,6 @@ import { getLatencyAggregation, getLatencyValue, } from '../../../lib/helpers/latency_aggregation_type'; -import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; function searchLatency({ environment, diff --git a/x-pack/solutions/observability/plugins/apm/tsconfig.json b/x-pack/solutions/observability/plugins/apm/tsconfig.json index 48567d4ada61a..2c509165d060d 100644 --- a/x-pack/solutions/observability/plugins/apm/tsconfig.json +++ b/x-pack/solutions/observability/plugins/apm/tsconfig.json @@ -148,7 +148,6 @@ "@kbn/react-query", "@kbn/core-chrome-layout-constants", "@kbn/control-group-renderer", - "@kbn/zod", "@kbn/agent-builder-plugin", "@kbn/observability-agent-builder-plugin", ], diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/calculate_throughput.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/calculate_throughput.ts new file mode 100644 index 0000000000000..59390050be730 --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/calculate_throughput.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function calculateThroughputWithRange({ + start, + end, + value, +}: { + start: number; + end: number; + value: number; +}) { + const durationAsMinutes = (end - start) / 1000 / 60; + return value / durationAsMinutes; +} diff --git a/x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_rollup_interval_for_time_range.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/get_rollup_interval_for_time_range.ts similarity index 100% rename from x-pack/solutions/observability/plugins/apm/server/agent_builder/utils/get_rollup_interval_for_time_range.ts rename to x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/get_rollup_interval_for_time_range.ts diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts index 55d2a9545c772..b0297af7cec04 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/index.ts @@ -6,7 +6,14 @@ */ export { getDocumentTypeFilterForServiceDestinationStatistics } from './spans/get_is_using_service_destination_metrics'; -export { getBackwardCompatibleDocumentTypeFilter } from './transactions'; +export { + getDurationFieldForTransactions, + getHasTransactionsEvents, + getBackwardCompatibleDocumentTypeFilter, + isSummaryFieldSupportedByDocType, + isDurationSummaryNotSupportedFilter, +} from './transactions'; +export { getRollupIntervalForTimeRange } from './get_rollup_interval_for_time_range'; export { APMEventClient, type APMEventESSearchRequest, @@ -19,3 +26,11 @@ export { callAsyncWithDebug } from './create_es_client/call_async_with_debug'; export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort'; export { getDataTierFilterCombined } from './tier_filter'; + +export { calculateThroughputWithRange } from './calculate_throughput'; + +export { + getOutcomeAggregation, + calculateFailedTransactionRate, + type OutcomeAggregation, +} from './transaction_error_rate'; diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transaction_error_rate.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transaction_error_rate.ts new file mode 100644 index 0000000000000..bed70ef2d3e27 --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transaction_error_rate.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + AggregationsSumAggregation, + AggregationsValueCountAggregation, + QueryDslQueryContainer, +} from '@elastic/elasticsearch/lib/api/types'; +import type { AggregationResultOfMap } from '@kbn/es-types'; +import { EVENT_OUTCOME, EVENT_SUCCESS_COUNT } from '@kbn/apm-types/es_fields'; +import { ApmDocumentType } from '../../../common/document_type'; + +enum EventOutcome { + success = 'success', + failure = 'failure', + unknown = 'unknown', +} + +export const getOutcomeAggregation = ( + documentType: ApmDocumentType +): { + successful_or_failed: + | { value_count: AggregationsValueCountAggregation } + | { filter: QueryDslQueryContainer }; + successful: { sum: AggregationsSumAggregation } | { filter: QueryDslQueryContainer }; +} => { + if (documentType === ApmDocumentType.ServiceTransactionMetric) { + return { + successful_or_failed: { + value_count: { + field: EVENT_SUCCESS_COUNT, + }, + }, + successful: { + sum: { + field: EVENT_SUCCESS_COUNT, + }, + }, + }; + } + + return { + successful_or_failed: { + filter: { + bool: { + filter: [ + { + terms: { + [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], + }, + }, + ], + }, + }, + }, + successful: { + filter: { + bool: { + filter: [ + { + terms: { + [EVENT_OUTCOME]: [EventOutcome.success], + }, + }, + ], + }, + }, + }, + }; +}; + +export type OutcomeAggregation = ReturnType; + +export function calculateFailedTransactionRate( + outcomeResponse: AggregationResultOfMap +) { + const successfulTransactions = + 'value' in outcomeResponse.successful + ? outcomeResponse.successful.value ?? 0 + : outcomeResponse.successful.doc_count; + + const successfulOrFailedTransactions = + 'value' in outcomeResponse.successful_or_failed + ? outcomeResponse.successful_or_failed.value + : outcomeResponse.successful_or_failed.doc_count; + + const failedTransactions = successfulOrFailedTransactions - successfulTransactions; + + return failedTransactions / successfulOrFailedTransactions; +} diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transactions/index.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transactions/index.ts index d46200d3a7300..ceb2e18e0d0bd 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transactions/index.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/lib/helpers/transactions/index.ts @@ -7,15 +7,21 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { + TRANSACTION_DURATION, TRANSACTION_DURATION_HISTOGRAM, METRICSET_INTERVAL, METRICSET_NAME, TRANSACTION_DURATION_SUMMARY, } from '@kbn/apm-types/es_fields'; +import { ProcessorEvent } from '@kbn/apm-types-shared'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; +import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; import { termQuery } from '@kbn/observability-utils-common/es/queries/term_query'; import { termsQuery } from '@kbn/observability-utils-common/es/queries/terms_query'; import { existsQuery } from '@kbn/observability-utils-common/es/queries/exists_query'; import { RollupInterval } from '../../../../common/rollup'; +import { ApmDocumentType } from '../../../../common/document_type'; +import type { APMEventClient } from '../create_es_client/create_apm_event_client'; // The function returns Document type filter for 1m Transaction Metrics export function getBackwardCompatibleDocumentTypeFilter(searchAggregatedTransactions: boolean) { @@ -45,3 +51,75 @@ export function isDurationSummaryNotSupportedFilter(): QueryDslQueryContainer { }, }; } + +export function isSummaryFieldSupportedByDocType( + typeOrSearchAgggregatedTransactions: + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent + | boolean +) { + let type: ApmDocumentType; + + if (typeOrSearchAgggregatedTransactions === true) { + type = ApmDocumentType.TransactionMetric; + } else if (typeOrSearchAgggregatedTransactions === false) { + type = ApmDocumentType.TransactionEvent; + } else { + type = typeOrSearchAgggregatedTransactions; + } + + return ( + type === ApmDocumentType.ServiceTransactionMetric || type === ApmDocumentType.TransactionMetric + ); +} + +export function getDurationFieldForTransactions( + typeOrSearchAgggregatedTransactions: + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent + | boolean, + useDurationSummaryField?: boolean +) { + if (isSummaryFieldSupportedByDocType(typeOrSearchAgggregatedTransactions)) { + if (useDurationSummaryField) { + return TRANSACTION_DURATION_SUMMARY; + } + return TRANSACTION_DURATION_HISTOGRAM; + } + + return TRANSACTION_DURATION; +} + +export async function getHasTransactionsEvents({ + start, + end, + apmEventClient, + kuery, +}: { + start?: number; + end?: number; + apmEventClient: APMEventClient; + kuery?: string; +}) { + const response = await apmEventClient.search('get_has_aggregated_transactions', { + apm: { + events: [ProcessorEvent.metric], + }, + track_total_hits: 1, + terminate_after: 1, + size: 0, + query: { + bool: { + filter: [ + { exists: { field: TRANSACTION_DURATION_HISTOGRAM } }, + ...(start && end ? rangeQuery(start, end) : []), + ...kqlQuery(kuery), + ], + }, + }, + }); + + return response.hits.total.value > 0; +} diff --git a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts index c56658ad0b54e..32ccc3cb90835 100644 --- a/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts +++ b/x-pack/solutions/observability/plugins/apm_data_access/server/utils.ts @@ -6,10 +6,19 @@ */ export { getDocumentTypeFilterForServiceDestinationStatistics, + getDurationFieldForTransactions, + getHasTransactionsEvents, getBackwardCompatibleDocumentTypeFilter, + isSummaryFieldSupportedByDocType, + isDurationSummaryNotSupportedFilter, + getRollupIntervalForTimeRange, callAsyncWithDebug, cancelEsRequestOnAbort, getDataTierFilterCombined, + calculateThroughputWithRange, + getOutcomeAggregation, + calculateFailedTransactionRate, + type OutcomeAggregation, } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts index 7a039906c2474..78581c3d31543 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/data_registry/data_registry_types.ts @@ -141,15 +141,6 @@ interface InfraHostsResponse { nodes: InfraEntityMetricsItem[]; } -export interface TraceMetricsItem { - group: string; - latency: number | null; - throughput: number; - failureRate: number; -} - -type TraceMetricsResponse = TraceMetricsItem[]; - export interface ObservabilityAgentBuilderDataRegistryTypes { apmErrors: (params: { request: KibanaRequest; @@ -221,12 +212,4 @@ export interface ObservabilityAgentBuilderDataRegistryTypes { query: Record | undefined; hostNames?: string[]; }) => Promise; - - traceMetrics: (params: { - request: KibanaRequest; - start: string; - end: string; - kqlFilter?: string; - groupBy: string; - }) => Promise; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts index bf98eb390c677..c3680613d243a 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/handler.ts @@ -5,20 +5,53 @@ * 2.0. */ -import type { KibanaRequest } from '@kbn/core/server'; -import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; -import type { TraceMetricsItem } from '../../data_registry/data_registry_types'; +import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import type { ApmDocumentType } from '@kbn/apm-data-access-plugin/common'; +import { + getPreferredBucketSizeAndDataSource, + getBucketSize, +} from '@kbn/apm-data-access-plugin/common'; +import { + calculateFailedTransactionRate, + getOutcomeAggregation, + calculateThroughputWithRange, + getDurationFieldForTransactions, +} from '@kbn/apm-data-access-plugin/server/utils'; +import type { + ObservabilityAgentBuilderPluginSetupDependencies, + ObservabilityAgentBuilderPluginStart, + ObservabilityAgentBuilderPluginStartDependencies, +} from '../../types'; +import { parseDatemath } from '../../utils/time'; +import { buildApmResources } from '../../utils/build_apm_resources'; +import { timeRangeFilter, kqlFilter as buildKqlFilter } from '../../utils/dsl_filters'; + +export interface TraceMetricsItem { + group: string; + latency: number | null; + throughput: number; + failureRate: number; +} + +const MAX_NUMBER_OF_GROUPS = 100; export async function getToolHandler({ + core, + plugins, request, - dataRegistry, + logger, start, end, kqlFilter, groupBy, }: { + core: CoreSetup< + ObservabilityAgentBuilderPluginStartDependencies, + ObservabilityAgentBuilderPluginStart + >; + plugins: ObservabilityAgentBuilderPluginSetupDependencies; request: KibanaRequest; - dataRegistry: ObservabilityAgentBuilderDataRegistry; + logger: Logger; start: string; end: string; groupBy: string; @@ -26,15 +59,108 @@ export async function getToolHandler({ }): Promise<{ items: TraceMetricsItem[]; }> { - const items = await dataRegistry.getData('traceMetrics', { + const { apmEventClient, apmDataAccessServices } = await buildApmResources({ + core, + plugins, request, - start, - end, - kqlFilter, - groupBy, + logger, + }); + + const startMs = parseDatemath(start); + const endMs = parseDatemath(end); + + // Get preferred document source based on groupBy, filter, and data availability + const kueryParts: string[] = []; + if (kqlFilter) { + kueryParts.push(kqlFilter); + } + kueryParts.push(`${groupBy}: *`); + const kuery = kueryParts.join(' AND '); + + const documentSources = await apmDataAccessServices.getDocumentSources({ + start: startMs, + end: endMs, + kuery, + }); + + const { bucketSize } = getBucketSize({ + start: startMs, + end: endMs, + numBuckets: 100, + }); + + const { source } = getPreferredBucketSizeAndDataSource({ + sources: documentSources, + bucketSizeInSeconds: bucketSize, + }); + + const { rollupInterval, hasDurationSummaryField } = source; + const documentType = source.documentType as + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent; + + const durationField = getDurationFieldForTransactions(documentType, hasDurationSummaryField); + const outcomeAggs = getOutcomeAggregation(documentType); + + const response = await apmEventClient.search('get_trace_metrics', { + apm: { + sources: [{ documentType, rollupInterval }], + }, + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [ + ...timeRangeFilter('@timestamp', { start: startMs, end: endMs }), + ...buildKqlFilter(kqlFilter), + ], + }, + }, + aggs: { + groups: { + terms: { + field: groupBy, + size: MAX_NUMBER_OF_GROUPS, + }, + aggs: { + avg_latency: { + avg: { + field: durationField, + }, + }, + ...outcomeAggs, + }, + }, + }, + }); + + const buckets = response.aggregations?.groups?.buckets ?? []; + + const items: TraceMetricsItem[] = buckets.map((bucket) => { + const docCount = bucket.doc_count; + const latencyValue = bucket.avg_latency?.value; + + const latencyMs = + latencyValue !== null && latencyValue !== undefined ? latencyValue / 1000 : null; + + const failureRate = calculateFailedTransactionRate(bucket); + + const throughput = calculateThroughputWithRange({ + start: startMs, + end: endMs, + value: docCount, + }); + + return { + group: bucket.key as string, + latency: latencyMs, + throughput, + failureRate, + }; }); return { - items: items ?? [], + items, }; } diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/tool.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/tool.ts index e135e14b1e2f5..48feaa62bfbd8 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/tool.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/get_trace_metrics/tool.ts @@ -11,10 +11,10 @@ import type { BuiltinToolDefinition, StaticToolRegistration } from '@kbn/agent-b import { ToolType } from '@kbn/agent-builder-common'; import { ToolResultType } from '@kbn/agent-builder-common/tools/tool_result'; import type { + ObservabilityAgentBuilderPluginSetupDependencies, ObservabilityAgentBuilderPluginStart, ObservabilityAgentBuilderPluginStartDependencies, } from '../../types'; -import type { ObservabilityAgentBuilderDataRegistry } from '../../data_registry/data_registry'; import { timeRangeSchemaRequired } from '../../utils/tool_schemas'; import { getAgentBuilderResourceAvailability } from '../../utils/get_agent_builder_resource_availability'; import { getToolHandler } from './handler'; @@ -40,14 +40,14 @@ const getTraceMetricsSchema = z.object({ export function createGetTraceMetricsTool({ core, - dataRegistry, + plugins, logger, }: { core: CoreSetup< ObservabilityAgentBuilderPluginStartDependencies, ObservabilityAgentBuilderPluginStart >; - dataRegistry: ObservabilityAgentBuilderDataRegistry; + plugins: ObservabilityAgentBuilderPluginSetupDependencies; logger: Logger; }): StaticToolRegistration { const toolDefinition: BuiltinToolDefinition = { @@ -94,8 +94,10 @@ Returns an array of items with: group (the groupBy field value), latency (ms), t try { const { items } = await getToolHandler({ + core, + plugins, request, - dataRegistry, + logger, start, end, kqlFilter, diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts index 8f61d6bdbbf23..66205629a19b7 100644 --- a/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/tools/register_tools.ts @@ -103,7 +103,7 @@ export async function registerTools({ createDownstreamDependenciesTool({ core, dataRegistry, logger }), createGetCorrelatedLogsTool({ core, logger }), createGetHostsTool({ core, logger, dataRegistry }), - createGetTraceMetricsTool({ core, dataRegistry, logger }), + createGetTraceMetricsTool({ core, plugins, logger }), createGetLogChangePointsTool({ core, plugins, logger }), createGetMetricChangePointsTool({ core, plugins, logger }), ]; diff --git a/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/build_apm_resources.ts b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/build_apm_resources.ts new file mode 100644 index 0000000000000..58dd747d346b5 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_agent_builder/server/utils/build_apm_resources.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import { SavedObjectsClient } from '@kbn/core/server'; +import { APMEventClient, type ApmDataAccessServices } from '@kbn/apm-data-access-plugin/server'; +import type { + ObservabilityAgentBuilderPluginSetupDependencies, + ObservabilityAgentBuilderPluginStart, + ObservabilityAgentBuilderPluginStartDependencies, +} from '../types'; + +export interface ApmResources { + apmEventClient: APMEventClient; + apmDataAccessServices: ApmDataAccessServices; +} + +export async function buildApmResources({ + core, + plugins, + request, + logger, +}: { + core: CoreSetup< + ObservabilityAgentBuilderPluginStartDependencies, + ObservabilityAgentBuilderPluginStart + >; + plugins: ObservabilityAgentBuilderPluginSetupDependencies; + request: KibanaRequest; + logger: Logger; +}): Promise { + const [coreStart] = await core.getStartServices(); + const savedObjectsClient = new SavedObjectsClient( + coreStart.savedObjects.createInternalRepository() + ); + const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + const indices = await plugins.apmDataAccess.getApmIndices(savedObjectsClient); + + const apmEventClient = new APMEventClient({ + esClient, + debug: false, + request, + indices, + options: { + includeFrozen: false, + excludedDataTiers: [], + }, + }); + + const apmDataAccessServices = plugins.apmDataAccess.getServices({ apmEventClient }); + + return { + apmEventClient, + apmDataAccessServices, + }; +} diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_metrics.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_metrics.spec.ts index e11d223c878e2..1c3cec7f9faa7 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_metrics.spec.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/observability_agent_builder/tools/get_trace_metrics.spec.ts @@ -14,16 +14,10 @@ import { } from '@kbn/synthtrace'; import type { OtherResult } from '@kbn/agent-builder-common'; import { OBSERVABILITY_GET_TRACE_METRICS_TOOL_ID } from '@kbn/observability-agent-builder-plugin/server/tools'; +import type { TraceMetricsItem } from '@kbn/observability-agent-builder-plugin/server/tools/get_trace_metrics/handler'; import type { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; import { createAgentBuilderApiClient } from '../utils/agent_builder_client'; -interface TraceMetricsItem { - group: string; - latency: number | null; - throughput: number; - failureRate: number; -} - interface GetTraceMetricsToolResult extends OtherResult { data: { items: TraceMetricsItem[];