From a6b610629ed482bbe4feb5296ab9649d4187ab55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:24:37 -0300 Subject: [PATCH 1/4] [APM] Breakdown Top dependencies API (#211441) closes https://github.com/elastic/kibana/issues/210552 Before: - Top dependencies API returned baseline and comparison timeseries data. Screenshot 2025-02-14 at 14 27 28 After: - Removing timeseries and comparison data. - The API is ~2s faster than before. - Response size is also smaller after removing the timeseries data. Screenshot 2025-02-14 at 14 26 34 Created a new API: `POST /internal/apm/dependencies/top_dependencies/statistics` to fetch the statistics for the visible dependencies. --------- Co-authored-by: Carlos Crespo Co-authored-by: Milosz Marcinkowski <38698566+miloszmarcinkowski@users.noreply.github.com> (cherry picked from commit a6fd5b7e101b7e0d13b15220a247d4a29e5c0405) # Conflicts: # x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx # x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts # x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts # x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts --- .../plugins/apm/common/connections.ts | 8 +- .../dependencies_inventory_table/index.tsx | 224 ++++++++---- .../get_span_metric_columns.tsx | 14 +- .../shared/dependencies_table/index.tsx | 85 ++--- .../plugins/apm/public/hooks/use_fetcher.tsx | 25 +- .../get_connection_stats/get_stats.ts | 139 +++----- .../connections/get_connection_stats/index.ts | 36 +- ...dependencies_timeseries_statistics.test.ts | 109 ++++++ .../get_dependencies_timeseries_statistics.ts | 256 ++++++++++++++ .../dependencies/get_top_dependencies.ts | 3 + .../apm/server/routes/dependencies/route.ts | 49 ++- .../get_service_dependencies_breakdown.ts | 2 +- .../apm/dependencies/generate_data.ts | 62 ++-- .../apm/dependencies/top_dependencies.spec.ts | 71 +++- .../dependencies/upstream_services.spec.ts | 2 +- .../dependencies/generate_data.ts | 78 +++++ .../dependencies/index.spec.ts | 323 ++++-------------- 17 files changed, 961 insertions(+), 525 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts create mode 100644 x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts diff --git a/x-pack/solutions/observability/plugins/apm/common/connections.ts b/x-pack/solutions/observability/plugins/apm/common/connections.ts index 1253d10a94842..9bf303a1bc74f 100644 --- a/x-pack/solutions/observability/plugins/apm/common/connections.ts +++ b/x-pack/solutions/observability/plugins/apm/common/connections.ts @@ -36,19 +36,19 @@ export type Node = ServiceNode | DependencyNode; export interface ConnectionStats { latency: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; throughput: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; errorRate: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; totalTime: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; } diff --git a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx index 720703c2f52bd..0f8e33757ea92 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx @@ -4,25 +4,45 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { METRIC_TYPE } from '@kbn/analytics'; -import { i18n } from '@kbn/i18n'; -import React from 'react'; import { useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; +import { METRIC_TYPE } from '@kbn/analytics'; +import { usePerformanceContext } from '@kbn/ebt-tools'; +import { i18n } from '@kbn/i18n'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; -import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; -import { getNodeName, NodeType } from '../../../../../common/connections'; +import { orderBy } from 'lodash'; +import React, { useEffect, useMemo } from 'react'; +import { v4 as uuidv4 } from 'uuid'; +import { NodeType, getNodeName } from '../../../../../common/connections'; import { useApmParams } from '../../../../hooks/use_apm_params'; -import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, useFetcher } from '../../../../hooks/use_fetcher'; import { useTimeRange } from '../../../../hooks/use_time_range'; +import type { DependenciesItem } from '../../../shared/dependencies_table'; +import { + DependenciesTable, + INITIAL_SORTING_FIELD, + INITIAL_SORTING_DIRECTION, +} from '../../../shared/dependencies_table'; import { DependencyLink } from '../../../shared/links/dependency_link'; -import { DependenciesTable } from '../../../shared/dependencies_table'; +import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; import { RandomSamplerBadge } from '../random_sampler_badge'; +const INITIAL_PAGE_SIZE = 25; + export function DependenciesInventoryTable() { const { - query: { rangeFrom, rangeTo, environment, kuery, comparisonEnabled, offset }, + query: { + rangeFrom, + rangeTo, + environment, + kuery, + comparisonEnabled, + offset, + page = 0, + pageSize = INITIAL_PAGE_SIZE, + sortDirection = INITIAL_SORTING_DIRECTION, + sortField = INITIAL_SORTING_FIELD, + }, } = useApmParams('/dependencies/inventory'); const { start, end } = useTimeRange({ rangeFrom, rangeTo }); @@ -37,61 +57,151 @@ export function DependenciesInventoryTable() { } return callApmApi('GET /internal/apm/dependencies/top_dependencies', { - params: { - query: { - start, - end, - environment, - numBuckets: 8, - offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, - kuery, - }, - }, + params: { query: { start, end, environment, numBuckets: 8, kuery } }, + }).then((response) => { + return { + ...response, + requestId: uuidv4(), + }; }); }, - [start, end, environment, offset, kuery, comparisonEnabled] + [start, end, environment, kuery] ); - const dependencies = - data?.dependencies.map((dependency) => { - const { location } = dependency; - const name = getNodeName(location); + const visibleDependenciesNames = useMemo( + () => + data?.dependencies + ? orderBy( + data.dependencies.map((item) => ({ + name: getNodeName(item.location), + impact: item.currentStats.impact, + latency: item.currentStats.latency.value, + throughput: item.currentStats.throughput.value, + failureRate: item.currentStats.errorRate.value, + })), + sortField, + sortDirection + ) + .slice(page * pageSize, (page + 1) * pageSize) + .map(({ name }) => name) + .sort() + : undefined, + [data?.dependencies, page, pageSize, sortDirection, sortField] + ); - if (location.type !== NodeType.dependency) { - throw new Error('Expected a dependency node'); + const { data: timeseriesData, status: timeseriesStatus } = useFetcher( + (callApmApi) => { + if (data?.requestId && visibleDependenciesNames?.length) { + return callApmApi('POST /internal/apm/dependencies/top_dependencies/statistics', { + params: { + query: { + start, + end, + environment, + numBuckets: 8, + offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, + kuery, + }, + body: { + dependencyNames: JSON.stringify(visibleDependenciesNames), + }, + }, + }); } - const link = ( - { - trackEvent({ - app: 'apm', - metricType: METRIC_TYPE.CLICK, - metric: 'dependencies_inventory_to_dependency_detail', - }); - }} - /> - ); + }, + // Disables exhaustive deps because the statistics api must only be called when the rendered items changed or when comparison is toggled or changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + [data?.requestId, visibleDependenciesNames, comparisonEnabled, offset], + // Do not invalidate this API call when the refresh button is clicked + { skipTimeRangeRefreshUpdate: true } + ); - return { - name, - currentStats: dependency.currentStats, - previousStats: dependency.previousStats, - link, - }; - }) ?? []; + const dependencies: DependenciesItem[] = useMemo( + () => + data?.dependencies.map((dependency) => { + const { location } = dependency; + const name = getNodeName(location); + + if (location.type !== NodeType.dependency) { + throw new Error('Expected a dependency node'); + } + const link = ( + { + trackEvent({ + app: 'apm', + metricType: METRIC_TYPE.CLICK, + metric: 'dependencies_inventory_to_dependency_detail', + }); + }} + /> + ); + + return { + name, + currentStats: { + impact: dependency.currentStats.impact, + totalTime: { value: dependency.currentStats.totalTime.value }, + latency: { + value: dependency.currentStats.latency.value, + timeseries: timeseriesData?.currentTimeseries[name]?.latency, + }, + throughput: { + value: dependency.currentStats.throughput.value, + timeseries: timeseriesData?.currentTimeseries[name]?.throughput, + }, + errorRate: { + value: dependency.currentStats.errorRate.value, + timeseries: timeseriesData?.currentTimeseries[name]?.errorRate, + }, + }, + previousStats: { + impact: dependency.previousStats?.impact ?? 0, + totalTime: { value: dependency.previousStats?.totalTime.value ?? null }, + latency: { + value: dependency.previousStats?.latency.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.latency, + }, + throughput: { + value: dependency.previousStats?.throughput.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.throughput, + }, + errorRate: { + value: dependency.previousStats?.errorRate.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.errorRate, + }, + }, + link, + }; + }) ?? [], + [ + comparisonEnabled, + data?.dependencies, + environment, + kuery, + offset, + rangeFrom, + rangeTo, + timeseriesData?.comparisonTimeseries, + timeseriesData?.currentTimeseries, + trackEvent, + ] + ); const showRandomSamplerBadge = data?.sampled && status === FETCH_STATUS.SUCCESS; + const fetchingStatus = + isPending(status) || isPending(timeseriesStatus) ? FETCH_STATUS.LOADING : FETCH_STATUS.SUCCESS; return ( <> @@ -109,9 +219,9 @@ export function DependenciesInventoryTable() { nameColumnTitle={i18n.translate('xpack.apm.dependenciesInventory.dependencyTableColumn', { defaultMessage: 'Dependency', })} - status={status} + status={fetchingStatus} compact={false} - initialPageSize={25} + initialPageSize={INITIAL_PAGE_SIZE} /> ); diff --git a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx index f6a33c9afa3cb..fb7d409333480 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx @@ -28,17 +28,17 @@ export interface SpanMetricGroup { impact: number | null; currentStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; } | undefined; previousStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; - impact: number; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; + impact?: number; } | undefined; } diff --git a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx index 6266091a3c21b..bfde5668ff2d9 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx @@ -7,7 +7,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useMemo } from 'react'; import type { ConnectionStatsItemWithComparisonData } from '../../../../common/connections'; import { useBreakpoints } from '../../../hooks/use_breakpoints'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; @@ -24,6 +24,8 @@ export type DependenciesItem = Omit void; } -type FormattedSpanMetricGroup = SpanMetricGroup & { +export type FormattedSpanMetricGroup = SpanMetricGroup & { name: string; link: React.ReactElement; }; -export function DependenciesTable(props: Props) { - const { - dependencies, - fixedHeight, - link, - title, - nameColumnTitle, - status, - compact = true, - showPerPageOptions = true, - initialPageSize, - showSparkPlots, - } = props; - +export function DependenciesTable({ + dependencies, + fixedHeight, + link, + title, + nameColumnTitle, + status, + compact = true, + showPerPageOptions = true, + initialPageSize, + showSparkPlots, + onChangeRenderedItems, +}: Props) { const { isLarge } = useBreakpoints(); const shouldShowSparkPlots = showSparkPlots ?? !isLarge; - const items: FormattedSpanMetricGroup[] = dependencies.map((dependency) => ({ - name: dependency.name, - link: dependency.link, - latency: dependency.currentStats.latency.value, - throughput: dependency.currentStats.throughput.value, - failureRate: dependency.currentStats.errorRate.value, - impact: dependency.currentStats.impact, - currentStats: { - latency: dependency.currentStats.latency.timeseries, - throughput: dependency.currentStats.throughput.timeseries, - failureRate: dependency.currentStats.errorRate.timeseries, - }, - previousStats: dependency.previousStats - ? { - latency: dependency.previousStats.latency.timeseries, - throughput: dependency.previousStats.throughput.timeseries, - failureRate: dependency.previousStats.errorRate.timeseries, - impact: dependency.previousStats.impact, - } - : undefined, - })); + const items: FormattedSpanMetricGroup[] = useMemo( + () => + dependencies.map((dependency) => ({ + name: dependency.name, + link: dependency.link, + latency: dependency.currentStats.latency.value, + throughput: dependency.currentStats.throughput.value, + failureRate: dependency.currentStats.errorRate.value, + impact: dependency.currentStats.impact, + currentStats: { + latency: dependency.currentStats.latency.timeseries, + throughput: dependency.currentStats.throughput.timeseries, + failureRate: dependency.currentStats.errorRate.timeseries, + }, + previousStats: dependency.previousStats + ? { + latency: dependency.previousStats.latency.timeseries, + throughput: dependency.previousStats.throughput.timeseries, + failureRate: dependency.previousStats.errorRate.timeseries, + impact: dependency.previousStats.impact, + } + : undefined, + })), + [dependencies] + ); const columns: Array> = [ { @@ -133,11 +139,12 @@ export function DependenciesTable(props: Props) { columns={columns} items={items} noItemsMessage={noItemsMessage} - initialSortField="impact" - initialSortDirection="desc" + initialSortField={INITIAL_SORTING_FIELD} + initialSortDirection={INITIAL_SORTING_DIRECTION} pagination={true} showPerPageOptions={showPerPageOptions} initialPageSize={initialPageSize} + onChangeRenderedItems={onChangeRenderedItems} /> diff --git a/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx b/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx index 419b540c24d3c..498b623f760ec 100644 --- a/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx @@ -87,6 +87,7 @@ export function useFetcher( options: { preservePreviousData?: boolean; showToastOnError?: boolean; + skipTimeRangeRefreshUpdate?: boolean; } = {} ): FetcherResult> & { refetch: () => void } { const { notifications } = useKibana(); @@ -99,6 +100,21 @@ export function useFetcher( const { timeRangeId } = useTimeRangeId(); const { addInspectorRequest } = useInspectorContext(); + const deps = useMemo(() => { + const _deps = [counter, preservePreviousData, showToastOnError, ...fnDeps]; + if (options.skipTimeRangeRefreshUpdate !== true) { + _deps.push(timeRangeId); + } + return _deps; + }, [ + counter, + fnDeps, + options.skipTimeRangeRefreshUpdate, + preservePreviousData, + showToastOnError, + timeRangeId, + ]); + useEffect(() => { let controller: AbortController = new AbortController(); @@ -176,14 +192,7 @@ export function useFetcher( controller.abort(); }; /* eslint-disable react-hooks/exhaustive-deps */ - }, [ - counter, - preservePreviousData, - timeRangeId, - showToastOnError, - ...fnDeps, - /* eslint-enable react-hooks/exhaustive-deps */ - ]); + }, deps); return useMemo(() => { return { diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts index a583e4de02c92..ab45ee8fcba0d 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { sum } from 'lodash'; import objectHash from 'object-hash'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { rangeQuery } from '@kbn/observability-plugin/server'; @@ -41,6 +40,7 @@ export const getStats = async ({ filter, numBuckets, offset, + withTimeseries, }: { apmEventClient: APMEventClient; start: number; @@ -48,6 +48,7 @@ export const getStats = async ({ filter: QueryDslQueryContainer[]; numBuckets: number; offset?: string; + withTimeseries: boolean; }) => { const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ start, @@ -61,6 +62,7 @@ export const getStats = async ({ endWithOffset, filter, numBuckets, + withTimeseries, }); return ( @@ -85,27 +87,15 @@ export const getStats = async ({ type: NodeType.dependency as const, }, value: { - count: sum(bucket.timeseries.buckets.map((dateBucket) => dateBucket.count.value ?? 0)), - latency_sum: sum( - bucket.timeseries.buckets.map((dateBucket) => dateBucket.latency_sum.value ?? 0) - ), - error_count: sum( - bucket.timeseries.buckets.flatMap( - (dateBucket) => - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0 - ) - ), + count: bucket.doc_count ?? 0, + latency_sum: bucket.total_latency_sum.value ?? 0, + error_count: bucket.error_count.doc_count ?? 0, }, - timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + timeseries: bucket.timeseries?.buckets.map((dateBucket) => ({ x: dateBucket.key + offsetInMs, - count: dateBucket.count.value ?? 0, - latency_sum: dateBucket.latency_sum.value ?? 0, - error_count: - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0, + count: dateBucket.doc_count ?? 0, + latency_sum: dateBucket.total_latency_sum.value ?? 0, + error_count: dateBucket.error_count.doc_count ?? 0, })), }; }) ?? [] @@ -118,6 +108,7 @@ async function getConnectionStats({ endWithOffset, filter, numBuckets, + withTimeseries, }: { apmEventClient: APMEventClient; startWithOffset: number; @@ -125,7 +116,27 @@ async function getConnectionStats({ filter: QueryDslQueryContainer[]; numBuckets: number; after?: { serviceName: string | number; dependencyName: string | number }; + withTimeseries: boolean; + dependencyNames?: string[]; }) { + const statsAggs = { + total_latency_sum: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, + }, + }, + total_latency_count: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + }, + }, + error_count: { + filter: { + bool: { filter: [{ terms: { [EVENT_OUTCOME]: [EventOutcome.failure] } }] }, + }, + }, + }; + return apmEventClient.search('get_connection_stats', { apm: { sources: [ @@ -169,79 +180,27 @@ async function getConnectionStats({ }, ] as const), }, - aggs: { - sample: { - top_metrics: { - size: 1, - metrics: asMutableArray([ - { - field: SERVICE_ENVIRONMENT, - }, - { - field: AGENT_NAME, - }, - { - field: SPAN_TYPE, - }, - { - field: SPAN_SUBTYPE, - }, - ] as const), - sort: { - '@timestamp': 'desc', - }, - }, - }, - total_latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - total_latency_count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, - }, - }, - aggs: { - latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - [EVENT_OUTCOME]: { - terms: { - field: EVENT_OUTCOME, - }, - aggs: { - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, + ...statsAggs, + ...(withTimeseries + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start: startWithOffset, + end: endWithOffset, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, }, }, + aggs: statsAggs, }, - }, - }, - }, + } + : undefined), }, }, }, diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts index 82e8dd5d1d950..713a6aa5c86de 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts @@ -5,16 +5,16 @@ * 2.0. */ -import type { ValuesType } from 'utility-types'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { merge } from 'lodash'; -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { ValuesType } from 'utility-types'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getStats } from './get_stats'; -import { getDestinationMap } from './get_destination_map'; -import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import { withApmSpan } from '../../../utils/with_apm_span'; +import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import type { APMEventClient } from '../../helpers/create_es_client/create_apm_event_client'; import type { RandomSampler } from '../../helpers/get_random_sampler'; +import { getDestinationMap } from './get_destination_map'; +import { getStats } from './get_stats'; export function getConnectionStats({ apmEventClient, @@ -25,6 +25,7 @@ export function getConnectionStats({ collapseBy, offset, randomSampler, + withTimeseries = true, }: { apmEventClient: APMEventClient; start: number; @@ -34,6 +35,7 @@ export function getConnectionStats({ collapseBy: 'upstream' | 'downstream'; offset?: string; randomSampler: RandomSampler; + withTimeseries?: boolean; }) { return withApmSpan('get_connection_stats_and_map', async () => { const [allMetrics, { nodesBydependencyName: destinationMap, sampled }] = await Promise.all([ @@ -44,6 +46,7 @@ export function getConnectionStats({ filter, numBuckets, offset, + withTimeseries, }), getDestinationMap({ apmEventClient, @@ -84,12 +87,15 @@ export function getConnectionStats({ latency_sum: prev.value.latency_sum + current.value.latency_sum, error_count: prev.value.error_count + current.value.error_count, }, - timeseries: joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ - x: a.x, - count: a.count + b.count, - latency_sum: a.latency_sum + b.latency_sum, - error_count: a.error_count + b.error_count, - })), + timeseries: + prev.timeseries && current.timeseries + ? joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ + x: a.x, + count: a.count + b.count, + latency_sum: a.latency_sum + b.latency_sum, + error_count: a.error_count + b.error_count, + })) + : undefined, }; }, { @@ -108,14 +114,14 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? mergedStats.value.latency_sum / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? point.latency_sum / point.count : null, })), }, totalTime: { value: mergedStats.value.latency_sum, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.latency_sum, })), @@ -129,7 +135,7 @@ export function getConnectionStats({ value: mergedStats.value.count, }) : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 @@ -146,7 +152,7 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? (mergedStats.value.error_count ?? 0) / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? (point.error_count ?? 0) / point.count : null, })), diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts new file mode 100644 index 0000000000000..37582a8b2f56d --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { DependenciesTimeseriesBuckes } from './get_dependencies_timeseries_statistics'; +import { parseDependenciesStats } from './get_dependencies_timeseries_statistics'; + +describe('parseDependenciesStats', () => { + const offsetInMs = 1000; + + test('should parse dependency stats correctly with all values', () => { + const dependencies = [ + { + key: 'service-A', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 10, + total_count: { value: 10 }, + failures: { total_count: { value: 2 }, doc_count: 2 }, + latency_sum: { value: 5000 }, + latency_count: { value: 10 }, + throughput: { value: 50 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-A': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: 50 }], + }, + }); + }); + + test('should handle missing optional values correctly', () => { + const dependencies = [ + { + key: 'service-B', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 5, + failures: { doc_count: 1 }, + latency_sum: { value: 2000 }, + latency_count: { value: 5 }, + throughput: {}, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-B': { + latency: [{ x: 1700000001000, y: 400 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: undefined }], + }, + }); + }); + + test('should handle missing failures field', () => { + const dependencies = [ + { + key: 'service-C', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 8, + failures: { doc_count: 0 }, + total_count: { value: 8 }, + latency_sum: { value: 4000 }, + latency_count: { value: 8 }, + throughput: { value: 30 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-C': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0 }], + throughput: [{ x: 1700000001000, y: 30 }], + }, + }); + }); + + test('should return an empty object when dependencies are empty', () => { + const result = parseDependenciesStats({ dependencies: [], offsetInMs }); + expect(result).toEqual({}); + }); +}); diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts new file mode 100644 index 0000000000000..af9a33935128b --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { EVENT_OUTCOME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { EventOutcome } from '../../../common/event_outcome'; +import { getBucketSize } from '../../../common/utils/get_bucket_size'; +import { environmentQuery } from '../../../common/utils/environment_query'; +import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; +import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; +import { + getDocCountFieldForServiceDestinationStatistics, + getDocumentTypeFilterForServiceDestinationStatistics, + getLatencyFieldForServiceDestinationStatistics, + getProcessorEventForServiceDestinationStatistics, +} from '../../lib/helpers/spans/get_is_using_service_destination_metrics'; + +interface Options { + dependencyNames: string[]; + searchServiceDestinationMetrics: boolean; + apmEventClient: APMEventClient; + start: number; + end: number; + environment: string; + kuery: string; + offset?: string; + numBuckets: number; +} + +interface Statistics { + latency: Array<{ x: number; y: number }>; + errorRate: Array<{ x: number; y: number }>; + throughput: Array<{ x: number; y: number | null }>; +} + +async function fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + environment, + kuery, + numBuckets, +}: Options) { + const response = await apmEventClient.search('get_latency_for_dependency', { + apm: { + events: [getProcessorEventForServiceDestinationStatistics(searchServiceDestinationMetrics)], + }, + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...rangeQuery(start, end), + ...getDocumentTypeFilterForServiceDestinationStatistics(searchServiceDestinationMetrics), + { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, + ], + }, + }, + aggs: { + dependencies: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start, + end, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: start, + max: end, + }, + }, + aggs: { + // latency + latency_sum: { + sum: { + field: getLatencyFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + ...(searchServiceDestinationMetrics + ? { + latency_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + // error + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, + }, + }, + aggs: { + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + }, + }, + // throughput + throughput: { + rate: { + ...(searchServiceDestinationMetrics + ? { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + } + : {}), + unit: 'minute', + }, + }, + }, + }, + }, + }, + }, + }); + + return response.aggregations?.dependencies.buckets || []; +} + +export type DependenciesTimeseriesBuckes = Awaited< + ReturnType +>; + +export function parseDependenciesStats({ + dependencies, + offsetInMs, +}: { + dependencies: DependenciesTimeseriesBuckes; + offsetInMs: number; +}) { + return ( + dependencies.reduce>((acc, bucket) => { + const stats: Statistics = { + latency: [], + errorRate: [], + throughput: [], + }; + + for (const statsBucket of bucket.timeseries.buckets) { + const totalCount = statsBucket.total_count?.value ?? statsBucket.doc_count; + const failureCount = + statsBucket.failures.total_count?.value ?? statsBucket.failures.doc_count; + const x = statsBucket.key + offsetInMs; + + stats.latency.push({ + x, + y: + (statsBucket.latency_sum.value ?? 0) / + (statsBucket.latency_count?.value ?? statsBucket.doc_count), + }); + stats.errorRate.push({ x, y: failureCount / totalCount }); + stats.throughput.push({ x, y: statsBucket.throughput.value }); + } + + acc[bucket.key] = stats; + return acc; + }, {}) ?? {} + ); +} + +export interface DependenciesTimeseriesStatisticsResponse { + currentTimeseries: Record; + comparisonTimeseries: Record | null; +} + +export async function getDependenciesTimeseriesStatistics({ + apmEventClient, + dependencyNames, + start, + end, + environment, + kuery, + searchServiceDestinationMetrics, + offset, + numBuckets, +}: Options): Promise { + const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ + start, + end, + offset, + }); + + const [currentTimeseries, comparisonTimeseries] = await Promise.all([ + fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + kuery, + environment, + numBuckets, + }), + offset + ? fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start: startWithOffset, + end: endWithOffset, + kuery, + environment, + numBuckets, + }) + : null, + ]); + + return { + currentTimeseries: parseDependenciesStats({ dependencies: currentTimeseries, offsetInMs: 0 }), + comparisonTimeseries: comparisonTimeseries?.length + ? parseDependenciesStats({ dependencies: comparisonTimeseries, offsetInMs }) + : null, + }; +} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts index 6d23b9c871755..9168ef6729118 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts @@ -27,6 +27,7 @@ interface Options { offset?: string; kuery: string; randomSampler: RandomSampler; + withTimeseries: boolean; } interface TopDependenciesForTimeRange { @@ -44,6 +45,7 @@ async function getTopDependenciesForTimeRange({ offset, kuery, randomSampler, + withTimeseries, }: Options): Promise { const { statsItems, sampled } = await getConnectionStats({ apmEventClient, @@ -54,6 +56,7 @@ async function getTopDependenciesForTimeRange({ offset, collapseBy: 'downstream', randomSampler, + withTimeseries, }); return { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts index 36780980cc0bd..225091b5da0ad 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts @@ -5,13 +5,15 @@ * 2.0. */ -import { toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; +import { jsonRt, toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { offsetRt } from '../../../common/comparison_rt'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { getRandomSampler } from '../../lib/helpers/get_random_sampler'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import type { DependenciesTimeseriesStatisticsResponse } from './get_dependencies_timeseries_statistics'; +import { getDependenciesTimeseriesStatistics } from './get_dependencies_timeseries_statistics'; import type { DependencyLatencyDistributionResponse } from './get_dependency_latency_distribution'; import { getDependencyLatencyDistribution } from './get_dependency_latency_distribution'; import { getErrorRateChartsForDependency } from './get_error_rate_charts_for_dependency'; @@ -32,14 +34,9 @@ import { getUpstreamServicesForDependency } from './get_upstream_services_for_de const topDependenciesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/dependencies/top_dependencies', - params: t.intersection([ - t.type({ - query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), - }), - t.partial({ - query: offsetRt, - }), - ]), + params: t.type({ + query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), + }), security: { authz: { requiredPrivileges: ['apm'] } }, handler: async (resources): Promise => { const { request, core } = resources; @@ -49,7 +46,7 @@ const topDependenciesRoute = createApmServerRoute({ getApmEventClient(resources), getRandomSampler({ coreStart, request, probability: 1 }), ]); - const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { environment, numBuckets, kuery, start, end } = resources.params.query; return getTopDependencies({ apmEventClient, @@ -58,8 +55,37 @@ const topDependenciesRoute = createApmServerRoute({ numBuckets, environment, kuery, - offset, randomSampler, + withTimeseries: false, + }); + }, +}); + +const topDependenciesStatisticsRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: t.type({ + query: t.intersection([ + t.intersection([environmentRt, kueryRt, rangeRt, offsetRt]), + t.type({ numBuckets: toNumberRt }), + ]), + body: t.type({ dependencyNames: jsonRt.pipe(t.array(t.string)) }), + }), + security: { authz: { requiredPrivileges: ['apm'] } }, + handler: async (resources): Promise => { + const apmEventClient = await getApmEventClient(resources); + const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { dependencyNames } = resources.params.body; + + return getDependenciesTimeseriesStatistics({ + apmEventClient, + start, + end, + environment, + kuery, + offset, + dependencyNames, + searchServiceDestinationMetrics: true, + numBuckets, }); }, }); @@ -403,4 +429,5 @@ export const dependencisRouteRepository = { ...dependencyOperationsRoute, ...dependencyLatencyDistributionChartsRoute, ...topDependencySpansRoute, + ...topDependenciesStatisticsRoute, }; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts index d69b55a46a1ca..d629852fd9788 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts @@ -54,7 +54,7 @@ export async function getServiceDependenciesBreakdown({ return { title: getNodeName(location), - data: stats.totalTime.timeseries, + data: stats.totalTime.timeseries || [], }; }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts index 58b708f0ab253..e306f52653c51 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts @@ -9,6 +9,7 @@ import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; export const dataConfig = { rate: 20, + errorRate: 5, transaction: { name: 'GET /api/product/list', duration: 1000, @@ -33,26 +34,45 @@ export async function generateData({ const instance = apm .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) .instance('instance-a'); - const { rate, transaction, span } = dataConfig; + const { rate, transaction, span, errorRate } = dataConfig; - await apmSynthtraceEsClient.index( - timerange(start, end) - .interval('1m') - .rate(rate) - .generator((timestamp) => - instance - .transaction({ transactionName: transaction.name }) - .timestamp(timestamp) - .duration(transaction.duration) - .success() - .children( - instance - .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) - .duration(transaction.duration) - .success() - .destination(span.destination) - .timestamp(timestamp) - ) - ) - ); + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts index 21e990a1dbb52..76a07c0755de4 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts @@ -13,6 +13,8 @@ import { dataConfig, generateData } from './generate_data'; import { roundNumber } from '../utils/common'; type TopDependencies = APIReturnType<'GET /internal/apm/dependencies/top_dependencies'>; +type TopDependenciesStatistics = + APIReturnType<'POST /internal/apm/dependencies/top_dependencies/statistics'>; export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { const apmApiClient = getService('apmApi'); @@ -20,6 +22,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const bucketSize = Math.round((end - start) / (60 * 1000)); async function callApi() { return await apmApiClient.readUser({ @@ -31,7 +34,24 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon environment: 'ENVIRONMENT_ALL', kuery: '', numBuckets: 20, - offset: '', + }, + }, + }); + } + + async function callStatisticsApi() { + return await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: { + query: { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + environment: 'ENVIRONMENT_ALL', + kuery: '', + numBuckets: 20, + }, + body: { + dependencyNames: JSON.stringify([dataConfig.span.destination]), }, }, }); @@ -48,13 +68,15 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon describe('when data is generated', () => { let topDependencies: TopDependencies; + let topDependenciesStats: TopDependenciesStatistics; let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); await generateData({ apmSynthtraceEsClient, start, end }); - const response = await callApi(); + const [response, statisticsResponse] = await Promise.all([callApi(), callStatisticsApi()]); topDependencies = response.body; + topDependenciesStats = statisticsResponse.body; }); after(() => apmSynthtraceEsClient.clean()); @@ -90,6 +112,13 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon expect(dependencies.currentStats).to.have.property('impact'); }); + it("doesn't have timeseries stats", () => { + expect(dependencies.currentStats.latency).to.not.have.property('timeseries'); + expect(dependencies.currentStats.totalTime).to.not.have.property('timeseries'); + expect(dependencies.currentStats.throughput).to.not.have.property('timeseries'); + expect(dependencies.currentStats.errorRate).to.not.have.property('timeseries'); + }); + it('returns the correct latency', () => { const { currentStats: { latency }, @@ -97,38 +126,52 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const { transaction } = dataConfig; - expect(latency.value).to.be(transaction.duration * 1000); - expect(latency.timeseries.every(({ y }) => y === transaction.duration * 1000)).to.be( - true - ); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].latency.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); it('returns the correct throughput', () => { const { currentStats: { throughput }, } = dependencies; - const { rate } = dataConfig; + const { rate, errorRate } = dataConfig; - expect(roundNumber(throughput.value)).to.be(roundNumber(rate)); + const totalRate = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(totalRate)); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].throughput.every( + ({ y }) => roundNumber(y) === roundNumber(totalRate) + ) + ).to.be(true); }); it('returns the correct total time', () => { const { currentStats: { totalTime }, } = dependencies; - const { rate, transaction } = dataConfig; + const { rate, transaction, errorRate } = dataConfig; - expect( - totalTime.timeseries.every(({ y }) => y === rate * transaction.duration * 1000) - ).to.be(true); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); }); it('returns the correct error rate', () => { const { currentStats: { errorRate }, } = dependencies; - expect(errorRate.value).to.be(0); - expect(errorRate.timeseries.every(({ y }) => y === 0)).to.be(true); + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].errorRate.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts index 42d0a66c31a89..99e6da8f321d7 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts @@ -63,7 +63,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon ]); const currentStatsLatencyValues = body.services[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + expect(currentStatsLatencyValues?.every(({ y }) => y === 1000000)).to.be(true); }); }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts new file mode 100644 index 0000000000000..e306f52653c51 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +export const dataConfig = { + rate: 20, + errorRate: 5, + transaction: { + name: 'GET /api/product/list', + duration: 1000, + }, + span: { + name: 'GET apm-*/_search', + type: 'db', + subType: 'elasticsearch', + destination: 'elasticsearch', + }, +}; + +export async function generateData({ + apmSynthtraceEsClient, + start, + end, +}: { + apmSynthtraceEsClient: ApmSynthtraceEsClient; + start: number; + end: number; +}) { + const instance = apm + .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) + .instance('instance-a'); + const { rate, transaction, span, errorRate } = dataConfig; + + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts index a579e196e7aa4..d5164a0ea90c8 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts @@ -5,33 +5,22 @@ * 2.0. */ import expect from '@kbn/expect'; -import { last, pick } from 'lodash'; import { DependencyNode } from '@kbn/apm-plugin/common/connections'; -import type { ValuesType } from 'utility-types'; import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { type Node, NodeType } from '@kbn/apm-plugin/common/connections'; -import { - ENVIRONMENT_ALL, - ENVIRONMENT_NOT_DEFINED, -} from '@kbn/apm-plugin/common/environment_filter_values'; +import { NodeType } from '@kbn/apm-plugin/common/connections'; import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; import { roundNumber } from '../../utils/common'; -import { generateDependencyData } from '../generate_data'; -import { apmDependenciesMapping, createServiceDependencyDocs } from './es_utils'; +import { generateData, dataConfig } from './generate_data'; export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { const apmApiClient = getService('apmApi'); const synthtrace = getService('synthtrace'); - const es = getService('es'); const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; - const dependencyName = 'elasticsearch'; - const serviceName = 'synth-go'; + const bucketSize = Math.round((end - start) / (60 * 1000)); - function getName(node: Node) { - return node.type === NodeType.service ? node.serviceName : node.dependencyName; - } + const serviceName = 'synth-go'; async function callApi() { return await apmApiClient.readUser({ @@ -60,271 +49,91 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon }); describe('when specific data is loaded', () => { - let response: { - status: number; - body: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - }; - - const indices = { - metric: 'apm-dependencies-metric', - transaction: 'apm-dependencies-transaction', - span: 'apm-dependencies-span', - }; - - const startTime = new Date(start).getTime(); - const endTime = new Date(end).getTime(); - - after(async () => { - const allIndices = Object.values(indices).join(','); - const indexExists = await es.indices.exists({ index: allIndices }); - if (indexExists) { - await es.indices.delete({ - index: allIndices, - }); - } - }); + let dependencies: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { - await es.indices.create({ - index: indices.metric, - body: { - mappings: apmDependenciesMapping, - }, - }); - - await es.indices.create({ - index: indices.transaction, - body: { - mappings: apmDependenciesMapping, - }, - }); - - await es.indices.create({ - index: indices.span, - body: { - mappings: apmDependenciesMapping, - }, - }); - - const docs = [ - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'success', - responseTime: { - count: 2, - sum: 10, - }, - time: startTime, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'failure', - responseTime: { - count: 1, - sum: 10, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'postgres', - outcome: 'success', - responseTime: { - count: 1, - sum: 3, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node-via-proxy', - outcome: 'success', - responseTime: { - count: 1, - sum: 1, - }, - time: endTime - 1, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ]; - - const bulkActions = docs.reduce( - (prev, doc) => { - return [...prev, { index: { _index: indices[doc.processor.event] } }, doc]; - }, - [] as Array< - | { - index: { - _index: string; - }; - } - | ValuesType - > - ); - - await es.bulk({ - body: bulkActions, - refresh: 'wait_for', - }); - - response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/dependencies`, - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - numBuckets: 20, - environment: ENVIRONMENT_ALL.value, - }, - }, - }); + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await generateData({ apmSynthtraceEsClient, start, end }); + const response = await callApi(); + dependencies = response.body; }); - it('returns a 200', () => { - expect(response.status).to.be(200); - }); + after(() => apmSynthtraceEsClient.clean()); - it('returns two dependencies', () => { - expect(response.body.serviceDependencies.length).to.be(2); + it('returns one dependency', () => { + expect(dependencies.serviceDependencies.length).to.be(1); }); - it('returns opbeans-node as a dependency', () => { - const opbeansNode = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'opbeans-node' - ); + it('returns correct dependency information', () => { + const location = dependencies.serviceDependencies[0].location as DependencyNode; + const { span } = dataConfig; - expect(opbeansNode !== undefined).to.be(true); - - const values = { - latency: roundNumber(opbeansNode?.currentStats.latency.value), - throughput: roundNumber(opbeansNode?.currentStats.throughput.value), - errorRate: roundNumber(opbeansNode?.currentStats.errorRate.value), - impact: opbeansNode?.currentStats.impact, - ...pick(opbeansNode?.location, 'serviceName', 'type', 'agentName', 'environment'), - }; - - const count = 4; - const sum = 21; - const errors = 1; - - expect(values).to.eql({ - agentName: 'nodejs', - environment: ENVIRONMENT_NOT_DEFINED.value, - serviceName: 'opbeans-node', - type: 'service', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 100, - }); - - const firstValue = roundNumber(opbeansNode?.currentStats.latency.timeseries[0].y); - const lastValue = roundNumber(last(opbeansNode?.currentStats.latency.timeseries)?.y); - - expect(firstValue).to.be(roundNumber(20 / 3)); - expect(lastValue).to.be(1); + expect(location.type).to.be(NodeType.dependency); + expect(location.dependencyName).to.be(span.destination); + expect(location.spanType).to.be(span.type); + expect(location.spanSubtype).to.be(span.subType); + expect(location).to.have.property('id'); }); - it('returns postgres as an external dependency', () => { - const postgres = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'postgres' - ); + it("doesn't have previous stats", () => { + expect(dependencies.serviceDependencies[0].previousStats).to.be(null); + }); - expect(postgres !== undefined).to.be(true); + it('has an "impact" property', () => { + expect(dependencies.serviceDependencies[0].currentStats).to.have.property('impact'); + }); - const values = { - latency: roundNumber(postgres?.currentStats.latency.value), - throughput: roundNumber(postgres?.currentStats.throughput.value), - errorRate: roundNumber(postgres?.currentStats.errorRate.value), - impact: postgres?.currentStats.impact, - ...pick(postgres?.location, 'spanType', 'spanSubtype', 'dependencyName', 'type'), - }; + it('returns the correct latency', () => { + const { + currentStats: { latency }, + } = dependencies.serviceDependencies[0]; - const count = 1; - const sum = 3; - const errors = 0; + const { transaction } = dataConfig; - expect(values).to.eql({ - spanType: 'external', - spanSubtype: 'http', - dependencyName: 'postgres', - type: 'dependency', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 0, - }); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect(latency.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); - }); - describe('when data is loaded', () => { - let apmSynthtraceEsClient: ApmSynthtraceEsClient; + it('returns the correct throughput', () => { + const { + currentStats: { throughput }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate } = dataConfig; - before(async () => { - apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); - await generateDependencyData({ apmSynthtraceEsClient, start, end }); + const expectedThroughput = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(expectedThroughput)); + expect( + throughput.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedThroughput / bucketSize) + ) + ).to.be(true); }); - after(() => apmSynthtraceEsClient.clean()); - it('returns a list of dependencies for a service', async () => { - const { status, body } = await callApi(); + it('returns the correct total time', () => { + const { + currentStats: { totalTime }, + } = dependencies.serviceDependencies[0]; + const { rate, transaction, errorRate } = dataConfig; - expect(status).to.be(200); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); expect( - body.serviceDependencies.map( - ({ location }) => (location as DependencyNode).dependencyName + totalTime.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedValuePerBucket) ) - ).to.eql([dependencyName]); + ).to.be(true); + }); - const currentStatsLatencyValues = - body.serviceDependencies[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + it('returns the correct error rate', () => { + const { + currentStats: { errorRate }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect(errorRate.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); }); }); From bd93a60fa9b9d5591d191e2064ce27e12647aca3 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:10:01 +0000 Subject: [PATCH 2/4] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../dependencies_inventory_table/index.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx index 0f8e33757ea92..a58de6d72329f 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx @@ -7,11 +7,10 @@ import { useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; import { METRIC_TYPE } from '@kbn/analytics'; -import { usePerformanceContext } from '@kbn/ebt-tools'; import { i18n } from '@kbn/i18n'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; import { orderBy } from 'lodash'; -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import { v4 as uuidv4 } from 'uuid'; import { NodeType, getNodeName } from '../../../../../common/connections'; import { useApmParams } from '../../../../hooks/use_apm_params'; @@ -116,7 +115,6 @@ export function DependenciesInventoryTable() { { skipTimeRangeRefreshUpdate: true } ); - const dependencies: DependenciesItem[] = useMemo( () => data?.dependencies.map((dependency) => { From 4c1ce453b5e38b8e62c44f00b50ecd1c587dc04b Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Mon, 10 Mar 2025 14:26:41 -0300 Subject: [PATCH 3/4] fixing merge --- .../get_connection_stats/get_stats.ts | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts index ab45ee8fcba0d..13a3d6b913087 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts @@ -180,27 +180,51 @@ async function getConnectionStats({ }, ] as const), }, - ...statsAggs, - ...(withTimeseries - ? { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, - }, + aggs: { + sample: { + top_metrics: { + size: 1, + metrics: asMutableArray([ + { + field: SERVICE_ENVIRONMENT, + }, + { + field: AGENT_NAME, + }, + { + field: SPAN_TYPE, + }, + { + field: SPAN_SUBTYPE, }, - aggs: statsAggs, + ] as const), + sort: { + '@timestamp': 'desc', }, - } - : undefined), + }, + }, + ...statsAggs, + ...(withTimeseries + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start: startWithOffset, + end: endWithOffset, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, + }, + }, + aggs: statsAggs, + }, + } + : undefined), + }, }, }, }, From 0024a6e796a5fe98117f72f554ffc2c5da65eaef Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Mon, 10 Mar 2025 14:31:55 -0300 Subject: [PATCH 4/4] fixing merge --- .../get_dependencies_timeseries_statistics.ts | 178 +++++++++--------- 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts index af9a33935128b..a47e0c7c7290b 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts @@ -51,102 +51,106 @@ async function fetchDependenciesTimeseriesStatistics({ apm: { events: [getProcessorEventForServiceDestinationStatistics(searchServiceDestinationMetrics)], }, - track_total_hits: false, - size: 0, - query: { - bool: { - filter: [ - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...rangeQuery(start, end), - ...getDocumentTypeFilterForServiceDestinationStatistics(searchServiceDestinationMetrics), - { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, - ], - }, - }, - aggs: { - dependencies: { - terms: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, + body: { + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...rangeQuery(start, end), + ...getDocumentTypeFilterForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, + ], }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start, - end, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: start, - max: end, - }, - }, - aggs: { - // latency - latency_sum: { - sum: { - field: getLatencyFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + }, + aggs: { + dependencies: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start, + end, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: start, + max: end, }, }, - ...(searchServiceDestinationMetrics - ? { - latency_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + aggs: { + // latency + latency_sum: { + sum: { + field: getLatencyFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + ...(searchServiceDestinationMetrics + ? { + latency_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, - }, - } - : {}), - // error - ...(searchServiceDestinationMetrics - ? { - total_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + } + : {}), + // error + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, + } + : {}), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, }, - } - : {}), - failures: { - filter: { - term: { - [EVENT_OUTCOME]: EventOutcome.failure, }, - }, - aggs: { - ...(searchServiceDestinationMetrics - ? { - total_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + aggs: { + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, - }, - } - : {}), + } + : {}), + }, }, - }, - // throughput - throughput: { - rate: { - ...(searchServiceDestinationMetrics - ? { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), - } - : {}), - unit: 'minute', + // throughput + throughput: { + rate: { + ...(searchServiceDestinationMetrics + ? { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + } + : {}), + unit: 'minute', + }, }, }, },