From f990584611a27c764014c2cef1ac7e7d05f1b8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:24:37 -0300 Subject: [PATCH 1/6] [APM] Breakdown Top dependencies API (#211441) closes https://github.com/elastic/kibana/issues/210552 Before: - Top dependencies API returned baseline and comparison timeseries data. Screenshot 2025-02-14 at 14 27 28 After: - Removing timeseries and comparison data. - The API is ~2s faster than before. - Response size is also smaller after removing the timeseries data. Screenshot 2025-02-14 at 14 26 34 Created a new API: `POST /internal/apm/dependencies/top_dependencies/statistics` to fetch the statistics for the visible dependencies. --------- Co-authored-by: Carlos Crespo Co-authored-by: Milosz Marcinkowski <38698566+miloszmarcinkowski@users.noreply.github.com> (cherry picked from commit a6fd5b7e101b7e0d13b15220a247d4a29e5c0405) # Conflicts: # x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx # x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/index.tsx # x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts # x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/index.ts # x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts # x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts # x-pack/plugins/observability_solution/apm/server/routes/dependencies/route.ts # x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts # x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts # x-pack/test/apm_api_integration/tests/service_overview/dependencies/generate_data.ts --- .../apm/common/connections.ts | 8 +- .../dependencies_inventory_table/index.tsx | 223 +++++++++++---- .../get_span_metric_columns.tsx | 14 +- .../shared/dependencies_table/index.tsx | 87 +++--- .../apm/public/hooks/use_fetcher.tsx | 25 +- .../get_connection_stats/get_stats.ts | 139 ++++------ .../connections/get_connection_stats/index.ts | 36 ++- ...dependencies_timeseries_statistics.test.ts | 109 ++++++++ .../get_dependencies_timeseries_statistics.ts | 256 ++++++++++++++++++ .../dependencies/get_top_dependencies.ts | 3 + .../apm/server/routes/dependencies/route.ts | 57 +++- .../get_service_dependencies_breakdown.ts | 2 +- .../dependencies/index.spec.ts | 140 ++++++++++ .../tests/dependencies/generate_data.ts | 62 +++-- .../dependencies/top_dependencies.spec.ts | 72 ++++- .../dependencies/upstream_services.spec.ts | 2 +- .../dependencies/generate_data.ts | 78 ++++++ 17 files changed, 1040 insertions(+), 273 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts create mode 100644 x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts create mode 100644 x-pack/test/apm_api_integration/tests/service_overview/dependencies/generate_data.ts diff --git a/x-pack/plugins/observability_solution/apm/common/connections.ts b/x-pack/plugins/observability_solution/apm/common/connections.ts index 10a10e96e5190..1f755872ecc6b 100644 --- a/x-pack/plugins/observability_solution/apm/common/connections.ts +++ b/x-pack/plugins/observability_solution/apm/common/connections.ts @@ -36,19 +36,19 @@ export type Node = ServiceNode | DependencyNode; export interface ConnectionStats { latency: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; throughput: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; errorRate: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; totalTime: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; } diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx index 720703c2f52bd..1b973eecf3564 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx @@ -4,25 +4,45 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { METRIC_TYPE } from '@kbn/analytics'; -import { i18n } from '@kbn/i18n'; -import React from 'react'; import { useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; +import { METRIC_TYPE } from '@kbn/analytics'; +import { usePerformanceContext } from '@kbn/ebt-tools'; +import { i18n } from '@kbn/i18n'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; -import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; -import { getNodeName, NodeType } from '../../../../../common/connections'; +import { orderBy } from 'lodash'; +import React, { useEffect, useMemo } from 'react'; +import { v4 as uuidv4 } from 'uuid'; +import { NodeType, getNodeName } from '../../../../../common/connections'; import { useApmParams } from '../../../../hooks/use_apm_params'; -import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, useFetcher } from '../../../../hooks/use_fetcher'; import { useTimeRange } from '../../../../hooks/use_time_range'; +import type { DependenciesItem } from '../../../shared/dependencies_table'; +import { + DependenciesTable, + INITIAL_SORTING_FIELD, + INITIAL_SORTING_DIRECTION, +} from '../../../shared/dependencies_table'; import { DependencyLink } from '../../../shared/links/dependency_link'; -import { DependenciesTable } from '../../../shared/dependencies_table'; +import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; import { RandomSamplerBadge } from '../random_sampler_badge'; +const INITIAL_PAGE_SIZE = 25; + export function DependenciesInventoryTable() { const { - query: { rangeFrom, rangeTo, environment, kuery, comparisonEnabled, offset }, + query: { + rangeFrom, + rangeTo, + environment, + kuery, + comparisonEnabled, + offset, + page = 0, + pageSize = INITIAL_PAGE_SIZE, + sortDirection = INITIAL_SORTING_DIRECTION, + sortField = INITIAL_SORTING_FIELD, + }, } = useApmParams('/dependencies/inventory'); const { start, end } = useTimeRange({ rangeFrom, rangeTo }); @@ -37,61 +57,150 @@ export function DependenciesInventoryTable() { } return callApmApi('GET /internal/apm/dependencies/top_dependencies', { - params: { - query: { - start, - end, - environment, - numBuckets: 8, - offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, - kuery, - }, - }, + params: { query: { start, end, environment, numBuckets: 8, kuery } }, + }).then((response) => { + return { + ...response, + requestId: uuidv4(), + }; }); }, - [start, end, environment, offset, kuery, comparisonEnabled] + [start, end, environment, kuery] ); - const dependencies = - data?.dependencies.map((dependency) => { - const { location } = dependency; - const name = getNodeName(location); + const visibleDependenciesNames = useMemo( + () => + data?.dependencies + ? orderBy( + data.dependencies.map((item) => ({ + name: getNodeName(item.location), + impact: item.currentStats.impact, + latency: item.currentStats.latency.value, + throughput: item.currentStats.throughput.value, + failureRate: item.currentStats.errorRate.value, + })), + sortField, + sortDirection + ) + .slice(page * pageSize, (page + 1) * pageSize) + .map(({ name }) => name) + .sort() + : undefined, + [data?.dependencies, page, pageSize, sortDirection, sortField] + ); - if (location.type !== NodeType.dependency) { - throw new Error('Expected a dependency node'); + const { data: timeseriesData, status: timeseriesStatus } = useFetcher( + (callApmApi) => { + if (data?.requestId && visibleDependenciesNames?.length) { + return callApmApi('POST /internal/apm/dependencies/top_dependencies/statistics', { + params: { + query: { + start, + end, + environment, + numBuckets: 8, + offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, + kuery, + }, + body: { + dependencyNames: JSON.stringify(visibleDependenciesNames), + }, + }, + }); } - const link = ( - { - trackEvent({ - app: 'apm', - metricType: METRIC_TYPE.CLICK, - metric: 'dependencies_inventory_to_dependency_detail', - }); - }} - /> - ); + }, + // Disables exhaustive deps because the statistics api must only be called when the rendered items changed or when comparison is toggled or changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + [data?.requestId, visibleDependenciesNames, comparisonEnabled, offset], + // Do not invalidate this API call when the refresh button is clicked + { skipTimeRangeRefreshUpdate: true } + ); - return { - name, - currentStats: dependency.currentStats, - previousStats: dependency.previousStats, - link, - }; - }) ?? []; + const dependencies: DependenciesItem[] = useMemo( + () => + data?.dependencies.map((dependency) => { + const { location } = dependency; + const name = getNodeName(location); + if (location.type !== NodeType.dependency) { + throw new Error('Expected a dependency node'); + } + const link = ( + { + trackEvent({ + app: 'apm', + metricType: METRIC_TYPE.CLICK, + metric: 'dependencies_inventory_to_dependency_detail', + }); + }} + /> + ); + + return { + name, + currentStats: { + impact: dependency.currentStats.impact, + totalTime: { value: dependency.currentStats.totalTime.value }, + latency: { + value: dependency.currentStats.latency.value, + timeseries: timeseriesData?.currentTimeseries[name]?.latency, + }, + throughput: { + value: dependency.currentStats.throughput.value, + timeseries: timeseriesData?.currentTimeseries[name]?.throughput, + }, + errorRate: { + value: dependency.currentStats.errorRate.value, + timeseries: timeseriesData?.currentTimeseries[name]?.errorRate, + }, + }, + previousStats: { + impact: dependency.previousStats?.impact ?? 0, + totalTime: { value: dependency.previousStats?.totalTime.value ?? null }, + latency: { + value: dependency.previousStats?.latency.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.latency, + }, + throughput: { + value: dependency.previousStats?.throughput.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.throughput, + }, + errorRate: { + value: dependency.previousStats?.errorRate.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.errorRate, + }, + }, + link, + }; + }) ?? [], + [ + comparisonEnabled, + data?.dependencies, + environment, + kuery, + offset, + rangeFrom, + rangeTo, + timeseriesData?.comparisonTimeseries, + timeseriesData?.currentTimeseries, + trackEvent, + ] + ); const showRandomSamplerBadge = data?.sampled && status === FETCH_STATUS.SUCCESS; + const fetchingStatus = + isPending(status) || isPending(timeseriesStatus) ? FETCH_STATUS.LOADING : FETCH_STATUS.SUCCESS; return ( <> @@ -109,9 +218,9 @@ export function DependenciesInventoryTable() { nameColumnTitle={i18n.translate('xpack.apm.dependenciesInventory.dependencyTableColumn', { defaultMessage: 'Dependency', })} - status={status} + status={fetchingStatus} compact={false} - initialPageSize={25} + initialPageSize={INITIAL_PAGE_SIZE} /> ); diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx index 567b635f1eb42..0cfa1e85d66ba 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx @@ -27,17 +27,17 @@ export interface SpanMetricGroup { impact: number | null; currentStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; } | undefined; previousStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; - impact: number; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; + impact?: number; } | undefined; } diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/index.tsx index 48b8e6ebc7d09..7ab48c168dcd5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/dependencies_table/index.tsx @@ -7,8 +7,8 @@ import { EuiFlexGroup, EuiFlexItem, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; -import { ConnectionStatsItemWithComparisonData } from '../../../../common/connections'; +import React, { useMemo } from 'react'; +import type { ConnectionStatsItemWithComparisonData } from '../../../../common/connections'; import { useBreakpoints } from '../../../hooks/use_breakpoints'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyMessage } from '../empty_message'; @@ -22,6 +22,8 @@ export type DependenciesItem = Omit void; } -type FormattedSpanMetricGroup = SpanMetricGroup & { +export type FormattedSpanMetricGroup = SpanMetricGroup & { name: string; link: React.ReactElement; }; -export function DependenciesTable(props: Props) { - const { - dependencies, - fixedHeight, - link, - title, - nameColumnTitle, - status, - compact = true, - showPerPageOptions = true, - initialPageSize, - showSparkPlots, - } = props; - +export function DependenciesTable({ + dependencies, + fixedHeight, + link, + title, + nameColumnTitle, + status, + compact = true, + showPerPageOptions = true, + initialPageSize, + showSparkPlots, + onChangeRenderedItems, +}: Props) { const { isLarge } = useBreakpoints(); const shouldShowSparkPlots = showSparkPlots ?? !isLarge; - const items: FormattedSpanMetricGroup[] = dependencies.map((dependency) => ({ - name: dependency.name, - link: dependency.link, - latency: dependency.currentStats.latency.value, - throughput: dependency.currentStats.throughput.value, - failureRate: dependency.currentStats.errorRate.value, - impact: dependency.currentStats.impact, - currentStats: { - latency: dependency.currentStats.latency.timeseries, - throughput: dependency.currentStats.throughput.timeseries, - failureRate: dependency.currentStats.errorRate.timeseries, - }, - previousStats: dependency.previousStats - ? { - latency: dependency.previousStats.latency.timeseries, - throughput: dependency.previousStats.throughput.timeseries, - failureRate: dependency.previousStats.errorRate.timeseries, - impact: dependency.previousStats.impact, - } - : undefined, - })); + const items: FormattedSpanMetricGroup[] = useMemo( + () => + dependencies.map((dependency) => ({ + name: dependency.name, + link: dependency.link, + latency: dependency.currentStats.latency.value, + throughput: dependency.currentStats.throughput.value, + failureRate: dependency.currentStats.errorRate.value, + impact: dependency.currentStats.impact, + currentStats: { + latency: dependency.currentStats.latency.timeseries, + throughput: dependency.currentStats.throughput.timeseries, + failureRate: dependency.currentStats.errorRate.timeseries, + }, + previousStats: dependency.previousStats + ? { + latency: dependency.previousStats.latency.timeseries, + throughput: dependency.previousStats.throughput.timeseries, + failureRate: dependency.previousStats.errorRate.timeseries, + impact: dependency.previousStats.impact, + } + : undefined, + })), + [dependencies] + ); const columns: Array> = [ { @@ -131,11 +137,12 @@ export function DependenciesTable(props: Props) { columns={columns} items={items} noItemsMessage={noItemsMessage} - initialSortField="impact" - initialSortDirection="desc" + initialSortField={INITIAL_SORTING_FIELD} + initialSortDirection={INITIAL_SORTING_DIRECTION} pagination={true} showPerPageOptions={showPerPageOptions} initialPageSize={initialPageSize} + onChangeRenderedItems={onChangeRenderedItems} /> diff --git a/x-pack/plugins/observability_solution/apm/public/hooks/use_fetcher.tsx b/x-pack/plugins/observability_solution/apm/public/hooks/use_fetcher.tsx index 7522fa87a20e0..ea390c92abed3 100644 --- a/x-pack/plugins/observability_solution/apm/public/hooks/use_fetcher.tsx +++ b/x-pack/plugins/observability_solution/apm/public/hooks/use_fetcher.tsx @@ -86,6 +86,7 @@ export function useFetcher( options: { preservePreviousData?: boolean; showToastOnError?: boolean; + skipTimeRangeRefreshUpdate?: boolean; } = {} ): FetcherResult> & { refetch: () => void } { const { notifications } = useKibana(); @@ -98,6 +99,21 @@ export function useFetcher( const { timeRangeId } = useTimeRangeId(); const { addInspectorRequest } = useInspectorContext(); + const deps = useMemo(() => { + const _deps = [counter, preservePreviousData, showToastOnError, ...fnDeps]; + if (options.skipTimeRangeRefreshUpdate !== true) { + _deps.push(timeRangeId); + } + return _deps; + }, [ + counter, + fnDeps, + options.skipTimeRangeRefreshUpdate, + preservePreviousData, + showToastOnError, + timeRangeId, + ]); + useEffect(() => { let controller: AbortController = new AbortController(); @@ -175,14 +191,7 @@ export function useFetcher( controller.abort(); }; /* eslint-disable react-hooks/exhaustive-deps */ - }, [ - counter, - preservePreviousData, - timeRangeId, - showToastOnError, - ...fnDeps, - /* eslint-enable react-hooks/exhaustive-deps */ - ]); + }, deps); return useMemo(() => { return { diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts index 7c5977ddbf854..6e14972c64cbf 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { sum } from 'lodash'; import objectHash from 'object-hash'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { rangeQuery } from '@kbn/observability-plugin/server'; @@ -41,6 +40,7 @@ export const getStats = async ({ filter, numBuckets, offset, + withTimeseries, }: { apmEventClient: APMEventClient; start: number; @@ -48,6 +48,7 @@ export const getStats = async ({ filter: QueryDslQueryContainer[]; numBuckets: number; offset?: string; + withTimeseries: boolean; }) => { const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ start, @@ -61,6 +62,7 @@ export const getStats = async ({ endWithOffset, filter, numBuckets, + withTimeseries, }); return ( @@ -85,27 +87,15 @@ export const getStats = async ({ type: NodeType.dependency as const, }, value: { - count: sum(bucket.timeseries.buckets.map((dateBucket) => dateBucket.count.value ?? 0)), - latency_sum: sum( - bucket.timeseries.buckets.map((dateBucket) => dateBucket.latency_sum.value ?? 0) - ), - error_count: sum( - bucket.timeseries.buckets.flatMap( - (dateBucket) => - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0 - ) - ), + count: bucket.doc_count ?? 0, + latency_sum: bucket.total_latency_sum.value ?? 0, + error_count: bucket.error_count.doc_count ?? 0, }, - timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + timeseries: bucket.timeseries?.buckets.map((dateBucket) => ({ x: dateBucket.key + offsetInMs, - count: dateBucket.count.value ?? 0, - latency_sum: dateBucket.latency_sum.value ?? 0, - error_count: - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0, + count: dateBucket.doc_count ?? 0, + latency_sum: dateBucket.total_latency_sum.value ?? 0, + error_count: dateBucket.error_count.doc_count ?? 0, })), }; }) ?? [] @@ -118,6 +108,7 @@ async function getConnectionStats({ endWithOffset, filter, numBuckets, + withTimeseries, }: { apmEventClient: APMEventClient; startWithOffset: number; @@ -125,7 +116,27 @@ async function getConnectionStats({ filter: QueryDslQueryContainer[]; numBuckets: number; after?: { serviceName: string | number; dependencyName: string | number }; + withTimeseries: boolean; + dependencyNames?: string[]; }) { + const statsAggs = { + total_latency_sum: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, + }, + }, + total_latency_count: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + }, + }, + error_count: { + filter: { + bool: { filter: [{ terms: { [EVENT_OUTCOME]: [EventOutcome.failure] } }] }, + }, + }, + }; + return apmEventClient.search('get_connection_stats', { apm: { sources: [ @@ -169,79 +180,27 @@ async function getConnectionStats({ }, ] as const), }, - aggs: { - sample: { - top_metrics: { - size: 1, - metrics: asMutableArray([ - { - field: SERVICE_ENVIRONMENT, - }, - { - field: AGENT_NAME, - }, - { - field: SPAN_TYPE, - }, - { - field: SPAN_SUBTYPE, - }, - ] as const), - sort: { - '@timestamp': 'desc', - }, - }, - }, - total_latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - total_latency_count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, - }, - }, - aggs: { - latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - [EVENT_OUTCOME]: { - terms: { - field: EVENT_OUTCOME, - }, - aggs: { - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, + ...statsAggs, + ...(withTimeseries + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start: startWithOffset, + end: endWithOffset, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, }, }, + aggs: statsAggs, }, - }, - }, - }, + } + : undefined), }, }, }, diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/index.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/index.ts index eccd8fa227d3b..16e51ccbfa1eb 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/index.ts @@ -9,12 +9,12 @@ import { ValuesType } from 'utility-types'; import { merge } from 'lodash'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getStats } from './get_stats'; -import { getDestinationMap } from './get_destination_map'; -import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import { withApmSpan } from '../../../utils/with_apm_span'; -import { APMEventClient } from '../../helpers/create_es_client/create_apm_event_client'; -import { RandomSampler } from '../../helpers/get_random_sampler'; +import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; +import type { APMEventClient } from '../../helpers/create_es_client/create_apm_event_client'; +import type { RandomSampler } from '../../helpers/get_random_sampler'; +import { getDestinationMap } from './get_destination_map'; +import { getStats } from './get_stats'; export function getConnectionStats({ apmEventClient, @@ -25,6 +25,7 @@ export function getConnectionStats({ collapseBy, offset, randomSampler, + withTimeseries = true, }: { apmEventClient: APMEventClient; start: number; @@ -34,6 +35,7 @@ export function getConnectionStats({ collapseBy: 'upstream' | 'downstream'; offset?: string; randomSampler: RandomSampler; + withTimeseries?: boolean; }) { return withApmSpan('get_connection_stats_and_map', async () => { const [allMetrics, { nodesBydependencyName: destinationMap, sampled }] = await Promise.all([ @@ -44,6 +46,7 @@ export function getConnectionStats({ filter, numBuckets, offset, + withTimeseries, }), getDestinationMap({ apmEventClient, @@ -84,12 +87,15 @@ export function getConnectionStats({ latency_sum: prev.value.latency_sum + current.value.latency_sum, error_count: prev.value.error_count + current.value.error_count, }, - timeseries: joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ - x: a.x, - count: a.count + b.count, - latency_sum: a.latency_sum + b.latency_sum, - error_count: a.error_count + b.error_count, - })), + timeseries: + prev.timeseries && current.timeseries + ? joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ + x: a.x, + count: a.count + b.count, + latency_sum: a.latency_sum + b.latency_sum, + error_count: a.error_count + b.error_count, + })) + : undefined, }; }, { @@ -108,14 +114,14 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? mergedStats.value.latency_sum / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? point.latency_sum / point.count : null, })), }, totalTime: { value: mergedStats.value.latency_sum, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.latency_sum, })), @@ -129,7 +135,7 @@ export function getConnectionStats({ value: mergedStats.value.count, }) : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 @@ -146,7 +152,7 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? (mergedStats.value.error_count ?? 0) / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? (point.error_count ?? 0) / point.count : null, })), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts new file mode 100644 index 0000000000000..37582a8b2f56d --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { DependenciesTimeseriesBuckes } from './get_dependencies_timeseries_statistics'; +import { parseDependenciesStats } from './get_dependencies_timeseries_statistics'; + +describe('parseDependenciesStats', () => { + const offsetInMs = 1000; + + test('should parse dependency stats correctly with all values', () => { + const dependencies = [ + { + key: 'service-A', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 10, + total_count: { value: 10 }, + failures: { total_count: { value: 2 }, doc_count: 2 }, + latency_sum: { value: 5000 }, + latency_count: { value: 10 }, + throughput: { value: 50 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-A': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: 50 }], + }, + }); + }); + + test('should handle missing optional values correctly', () => { + const dependencies = [ + { + key: 'service-B', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 5, + failures: { doc_count: 1 }, + latency_sum: { value: 2000 }, + latency_count: { value: 5 }, + throughput: {}, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-B': { + latency: [{ x: 1700000001000, y: 400 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: undefined }], + }, + }); + }); + + test('should handle missing failures field', () => { + const dependencies = [ + { + key: 'service-C', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 8, + failures: { doc_count: 0 }, + total_count: { value: 8 }, + latency_sum: { value: 4000 }, + latency_count: { value: 8 }, + throughput: { value: 30 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-C': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0 }], + throughput: [{ x: 1700000001000, y: 30 }], + }, + }); + }); + + test('should return an empty object when dependencies are empty', () => { + const result = parseDependenciesStats({ dependencies: [], offsetInMs }); + expect(result).toEqual({}); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts new file mode 100644 index 0000000000000..af9a33935128b --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { EVENT_OUTCOME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { EventOutcome } from '../../../common/event_outcome'; +import { getBucketSize } from '../../../common/utils/get_bucket_size'; +import { environmentQuery } from '../../../common/utils/environment_query'; +import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; +import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; +import { + getDocCountFieldForServiceDestinationStatistics, + getDocumentTypeFilterForServiceDestinationStatistics, + getLatencyFieldForServiceDestinationStatistics, + getProcessorEventForServiceDestinationStatistics, +} from '../../lib/helpers/spans/get_is_using_service_destination_metrics'; + +interface Options { + dependencyNames: string[]; + searchServiceDestinationMetrics: boolean; + apmEventClient: APMEventClient; + start: number; + end: number; + environment: string; + kuery: string; + offset?: string; + numBuckets: number; +} + +interface Statistics { + latency: Array<{ x: number; y: number }>; + errorRate: Array<{ x: number; y: number }>; + throughput: Array<{ x: number; y: number | null }>; +} + +async function fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + environment, + kuery, + numBuckets, +}: Options) { + const response = await apmEventClient.search('get_latency_for_dependency', { + apm: { + events: [getProcessorEventForServiceDestinationStatistics(searchServiceDestinationMetrics)], + }, + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...rangeQuery(start, end), + ...getDocumentTypeFilterForServiceDestinationStatistics(searchServiceDestinationMetrics), + { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, + ], + }, + }, + aggs: { + dependencies: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start, + end, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: start, + max: end, + }, + }, + aggs: { + // latency + latency_sum: { + sum: { + field: getLatencyFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + ...(searchServiceDestinationMetrics + ? { + latency_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + // error + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, + }, + }, + aggs: { + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + }, + }, + // throughput + throughput: { + rate: { + ...(searchServiceDestinationMetrics + ? { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + } + : {}), + unit: 'minute', + }, + }, + }, + }, + }, + }, + }, + }); + + return response.aggregations?.dependencies.buckets || []; +} + +export type DependenciesTimeseriesBuckes = Awaited< + ReturnType +>; + +export function parseDependenciesStats({ + dependencies, + offsetInMs, +}: { + dependencies: DependenciesTimeseriesBuckes; + offsetInMs: number; +}) { + return ( + dependencies.reduce>((acc, bucket) => { + const stats: Statistics = { + latency: [], + errorRate: [], + throughput: [], + }; + + for (const statsBucket of bucket.timeseries.buckets) { + const totalCount = statsBucket.total_count?.value ?? statsBucket.doc_count; + const failureCount = + statsBucket.failures.total_count?.value ?? statsBucket.failures.doc_count; + const x = statsBucket.key + offsetInMs; + + stats.latency.push({ + x, + y: + (statsBucket.latency_sum.value ?? 0) / + (statsBucket.latency_count?.value ?? statsBucket.doc_count), + }); + stats.errorRate.push({ x, y: failureCount / totalCount }); + stats.throughput.push({ x, y: statsBucket.throughput.value }); + } + + acc[bucket.key] = stats; + return acc; + }, {}) ?? {} + ); +} + +export interface DependenciesTimeseriesStatisticsResponse { + currentTimeseries: Record; + comparisonTimeseries: Record | null; +} + +export async function getDependenciesTimeseriesStatistics({ + apmEventClient, + dependencyNames, + start, + end, + environment, + kuery, + searchServiceDestinationMetrics, + offset, + numBuckets, +}: Options): Promise { + const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ + start, + end, + offset, + }); + + const [currentTimeseries, comparisonTimeseries] = await Promise.all([ + fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + kuery, + environment, + numBuckets, + }), + offset + ? fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start: startWithOffset, + end: endWithOffset, + kuery, + environment, + numBuckets, + }) + : null, + ]); + + return { + currentTimeseries: parseDependenciesStats({ dependencies: currentTimeseries, offsetInMs: 0 }), + comparisonTimeseries: comparisonTimeseries?.length + ? parseDependenciesStats({ dependencies: comparisonTimeseries, offsetInMs }) + : null, + }; +} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependencies.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependencies.ts index eb7bcab85296e..9da55463de8e7 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependencies.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependencies.ts @@ -27,6 +27,7 @@ interface Options { offset?: string; kuery: string; randomSampler: RandomSampler; + withTimeseries: boolean; } interface TopDependenciesForTimeRange { @@ -44,6 +45,7 @@ async function getTopDependenciesForTimeRange({ offset, kuery, randomSampler, + withTimeseries, }: Options): Promise { const { statsItems, sampled } = await getConnectionStats({ apmEventClient, @@ -54,6 +56,7 @@ async function getTopDependenciesForTimeRange({ offset, collapseBy: 'downstream', randomSampler, + withTimeseries, }); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/route.ts index 4459a127ea13e..40d481bf9e4cf 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/route.ts @@ -5,17 +5,17 @@ * 2.0. */ -import { toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; +import { jsonRt, toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { offsetRt } from '../../../common/comparison_rt'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { getRandomSampler } from '../../lib/helpers/get_random_sampler'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; -import { - DependencyLatencyDistributionResponse, - getDependencyLatencyDistribution, -} from './get_dependency_latency_distribution'; +import type { DependenciesTimeseriesStatisticsResponse } from './get_dependencies_timeseries_statistics'; +import { getDependenciesTimeseriesStatistics } from './get_dependencies_timeseries_statistics'; +import type { DependencyLatencyDistributionResponse } from './get_dependency_latency_distribution'; +import { getDependencyLatencyDistribution } from './get_dependency_latency_distribution'; import { getErrorRateChartsForDependency } from './get_error_rate_charts_for_dependency'; import { getLatencyChartsForDependency, @@ -39,14 +39,9 @@ import { const topDependenciesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/dependencies/top_dependencies', - params: t.intersection([ - t.type({ - query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), - }), - t.partial({ - query: offsetRt, - }), - ]), + params: t.type({ + query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), + }), options: { tags: ['access:apm'], }, @@ -60,7 +55,7 @@ const topDependenciesRoute = createApmServerRoute({ getApmEventClient(resources), getRandomSampler({ security, request, probability: 1 }), ]); - const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { environment, numBuckets, kuery, start, end } = resources.params.query; return getTopDependencies({ apmEventClient, @@ -69,8 +64,39 @@ const topDependenciesRoute = createApmServerRoute({ numBuckets, environment, kuery, - offset, randomSampler, + withTimeseries: false, + }); + }, +}); + +const topDependenciesStatisticsRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: t.type({ + query: t.intersection([ + t.intersection([environmentRt, kueryRt, rangeRt, offsetRt]), + t.type({ numBuckets: toNumberRt }), + ]), + body: t.type({ dependencyNames: jsonRt.pipe(t.array(t.string)) }), + }), + options: { + tags: ['access:apm'], + }, + handler: async (resources): Promise => { + const apmEventClient = await getApmEventClient(resources); + const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { dependencyNames } = resources.params.body; + + return getDependenciesTimeseriesStatistics({ + apmEventClient, + start, + end, + environment, + kuery, + offset, + dependencyNames, + searchServiceDestinationMetrics: true, + numBuckets, }); }, }); @@ -432,4 +458,5 @@ export const dependencisRouteRepository = { ...dependencyOperationsRoute, ...dependencyLatencyDistributionChartsRoute, ...topDependencySpansRoute, + ...topDependenciesStatisticsRoute, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_dependencies_breakdown.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_dependencies_breakdown.ts index 270ffc6852918..d4d9ea8d90cfc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_dependencies_breakdown.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_dependencies_breakdown.ts @@ -54,7 +54,7 @@ export async function getServiceDependenciesBreakdown({ return { title: getNodeName(location), - data: stats.totalTime.timeseries, + data: stats.totalTime.timeseries || [], }; }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts new file mode 100644 index 0000000000000..d5164a0ea90c8 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { DependencyNode } from '@kbn/apm-plugin/common/connections'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { NodeType } from '@kbn/apm-plugin/common/connections'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; +import { roundNumber } from '../../utils/common'; +import { generateData, dataConfig } from './generate_data'; + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const bucketSize = Math.round((end - start) / (60 * 1000)); + + const serviceName = 'synth-go'; + + async function callApi() { + return await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/dependencies', + params: { + path: { serviceName }, + query: { + environment: 'production', + numBuckets: 20, + offset: '1d', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + }, + }, + }); + } + + describe('Dependency for service', () => { + describe('when data is not loaded', () => { + it('handles empty state #1', async () => { + const { status, body } = await callApi(); + + expect(status).to.be(200); + expect(body.serviceDependencies).to.be.empty(); + }); + }); + + describe('when specific data is loaded', () => { + let dependencies: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await generateData({ apmSynthtraceEsClient, start, end }); + const response = await callApi(); + dependencies = response.body; + }); + + after(() => apmSynthtraceEsClient.clean()); + + it('returns one dependency', () => { + expect(dependencies.serviceDependencies.length).to.be(1); + }); + + it('returns correct dependency information', () => { + const location = dependencies.serviceDependencies[0].location as DependencyNode; + const { span } = dataConfig; + + expect(location.type).to.be(NodeType.dependency); + expect(location.dependencyName).to.be(span.destination); + expect(location.spanType).to.be(span.type); + expect(location.spanSubtype).to.be(span.subType); + expect(location).to.have.property('id'); + }); + + it("doesn't have previous stats", () => { + expect(dependencies.serviceDependencies[0].previousStats).to.be(null); + }); + + it('has an "impact" property', () => { + expect(dependencies.serviceDependencies[0].currentStats).to.have.property('impact'); + }); + + it('returns the correct latency', () => { + const { + currentStats: { latency }, + } = dependencies.serviceDependencies[0]; + + const { transaction } = dataConfig; + + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect(latency.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); + }); + + it('returns the correct throughput', () => { + const { + currentStats: { throughput }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate } = dataConfig; + + const expectedThroughput = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(expectedThroughput)); + expect( + throughput.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedThroughput / bucketSize) + ) + ).to.be(true); + }); + + it('returns the correct total time', () => { + const { + currentStats: { totalTime }, + } = dependencies.serviceDependencies[0]; + const { rate, transaction, errorRate } = dataConfig; + + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); + expect( + totalTime.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedValuePerBucket) + ) + ).to.be(true); + }); + + it('returns the correct error rate', () => { + const { + currentStats: { errorRate }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect(errorRate.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/dependencies/generate_data.ts b/x-pack/test/apm_api_integration/tests/dependencies/generate_data.ts index 58b708f0ab253..e306f52653c51 100644 --- a/x-pack/test/apm_api_integration/tests/dependencies/generate_data.ts +++ b/x-pack/test/apm_api_integration/tests/dependencies/generate_data.ts @@ -9,6 +9,7 @@ import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; export const dataConfig = { rate: 20, + errorRate: 5, transaction: { name: 'GET /api/product/list', duration: 1000, @@ -33,26 +34,45 @@ export async function generateData({ const instance = apm .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) .instance('instance-a'); - const { rate, transaction, span } = dataConfig; + const { rate, transaction, span, errorRate } = dataConfig; - await apmSynthtraceEsClient.index( - timerange(start, end) - .interval('1m') - .rate(rate) - .generator((timestamp) => - instance - .transaction({ transactionName: transaction.name }) - .timestamp(timestamp) - .duration(transaction.duration) - .success() - .children( - instance - .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) - .duration(transaction.duration) - .success() - .destination(span.destination) - .timestamp(timestamp) - ) - ) - ); + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); } diff --git a/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts b/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts index acdea5a3d54de..d86feee3798ac 100644 --- a/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts +++ b/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts @@ -12,6 +12,8 @@ import { dataConfig, generateData } from './generate_data'; import { roundNumber } from '../../utils'; type TopDependencies = APIReturnType<'GET /internal/apm/dependencies/top_dependencies'>; +type TopDependenciesStatistics = + APIReturnType<'POST /internal/apm/dependencies/top_dependencies/statistics'>; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); @@ -20,6 +22,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const bucketSize = Math.round((end - start) / (60 * 1000)); async function callApi() { return await apmApiClient.readUser({ @@ -31,7 +34,24 @@ export default function ApiTest({ getService }: FtrProviderContext) { environment: 'ENVIRONMENT_ALL', kuery: '', numBuckets: 20, - offset: '', + }, + }, + }); + } + + async function callStatisticsApi() { + return await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: { + query: { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + environment: 'ENVIRONMENT_ALL', + kuery: '', + numBuckets: 20, + }, + body: { + dependencyNames: JSON.stringify([dataConfig.span.destination]), }, }, }); @@ -53,11 +73,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { registry.when('Top dependencies', { config: 'basic', archives: [] }, () => { describe('when data is generated', () => { let topDependencies: TopDependencies; + let topDependenciesStats: TopDependenciesStatistics; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { await generateData({ apmSynthtraceEsClient, start, end }); - const response = await callApi(); + const [response, statisticsResponse] = await Promise.all([callApi(), callStatisticsApi()]); topDependencies = response.body; + topDependenciesStats = statisticsResponse.body; }); after(() => apmSynthtraceEsClient.clean()); @@ -93,6 +116,13 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(dependencies.currentStats).to.have.property('impact'); }); + it("doesn't have timeseries stats", () => { + expect(dependencies.currentStats.latency).to.not.have.property('timeseries'); + expect(dependencies.currentStats.totalTime).to.not.have.property('timeseries'); + expect(dependencies.currentStats.throughput).to.not.have.property('timeseries'); + expect(dependencies.currentStats.errorRate).to.not.have.property('timeseries'); + }); + it('returns the correct latency', () => { const { currentStats: { latency }, @@ -100,38 +130,52 @@ export default function ApiTest({ getService }: FtrProviderContext) { const { transaction } = dataConfig; - expect(latency.value).to.be(transaction.duration * 1000); - expect(latency.timeseries.every(({ y }) => y === transaction.duration * 1000)).to.be( - true - ); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].latency.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); it('returns the correct throughput', () => { const { currentStats: { throughput }, } = dependencies; - const { rate } = dataConfig; + const { rate, errorRate } = dataConfig; - expect(roundNumber(throughput.value)).to.be(roundNumber(rate)); + const totalRate = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(totalRate)); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].throughput.every( + ({ y }) => roundNumber(y) === roundNumber(totalRate) + ) + ).to.be(true); }); it('returns the correct total time', () => { const { currentStats: { totalTime }, } = dependencies; - const { rate, transaction } = dataConfig; + const { rate, transaction, errorRate } = dataConfig; - expect( - totalTime.timeseries.every(({ y }) => y === rate * transaction.duration * 1000) - ).to.be(true); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); }); it('returns the correct error rate', () => { const { currentStats: { errorRate }, } = dependencies; - expect(errorRate.value).to.be(0); - expect(errorRate.timeseries.every(({ y }) => y === 0)).to.be(true); + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].errorRate.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); }); }); diff --git a/x-pack/test/apm_api_integration/tests/dependencies/upstream_services.spec.ts b/x-pack/test/apm_api_integration/tests/dependencies/upstream_services.spec.ts index 1a7e958881d96..4b3ad94ef4839 100644 --- a/x-pack/test/apm_api_integration/tests/dependencies/upstream_services.spec.ts +++ b/x-pack/test/apm_api_integration/tests/dependencies/upstream_services.spec.ts @@ -64,7 +64,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { ]); const currentStatsLatencyValues = body.services[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + expect(currentStatsLatencyValues?.every(({ y }) => y === 1000000)).to.be(true); }); }); }); diff --git a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/generate_data.ts b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/generate_data.ts new file mode 100644 index 0000000000000..e306f52653c51 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/generate_data.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +export const dataConfig = { + rate: 20, + errorRate: 5, + transaction: { + name: 'GET /api/product/list', + duration: 1000, + }, + span: { + name: 'GET apm-*/_search', + type: 'db', + subType: 'elasticsearch', + destination: 'elasticsearch', + }, +}; + +export async function generateData({ + apmSynthtraceEsClient, + start, + end, +}: { + apmSynthtraceEsClient: ApmSynthtraceEsClient; + start: number; + end: number; +}) { + const instance = apm + .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) + .instance('instance-a'); + const { rate, transaction, span, errorRate } = dataConfig; + + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); +} From d92dec40286f09341621ef5dd549e9a77cd987b2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:42:03 +0000 Subject: [PATCH 2/6] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../dependencies_inventory_table/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx index 1b973eecf3564..a58de6d72329f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx @@ -7,11 +7,10 @@ import { useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; import { METRIC_TYPE } from '@kbn/analytics'; -import { usePerformanceContext } from '@kbn/ebt-tools'; import { i18n } from '@kbn/i18n'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; import { orderBy } from 'lodash'; -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import { v4 as uuidv4 } from 'uuid'; import { NodeType, getNodeName } from '../../../../../common/connections'; import { useApmParams } from '../../../../hooks/use_apm_params'; From 5c4674d884b8f853d2163e56dbabd024257774c8 Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Mon, 10 Mar 2025 14:44:02 -0300 Subject: [PATCH 3/6] fixing merge --- .../get_connection_stats/get_stats.ts | 62 ++++-- .../get_dependencies_timeseries_statistics.ts | 178 +++++++++--------- 2 files changed, 134 insertions(+), 106 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts index 6e14972c64cbf..6c11478c2b77e 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_stats.ts @@ -180,27 +180,51 @@ async function getConnectionStats({ }, ] as const), }, - ...statsAggs, - ...(withTimeseries - ? { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, - }, + aggs: { + sample: { + top_metrics: { + size: 1, + metrics: asMutableArray([ + { + field: SERVICE_ENVIRONMENT, + }, + { + field: AGENT_NAME, + }, + { + field: SPAN_TYPE, + }, + { + field: SPAN_SUBTYPE, }, - aggs: statsAggs, + ] as const), + sort: { + '@timestamp': 'desc', }, - } - : undefined), + }, + }, + ...statsAggs, + ...(withTimeseries + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start: startWithOffset, + end: endWithOffset, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, + }, + }, + aggs: statsAggs, + }, + } + : undefined), + }, }, }, }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts index af9a33935128b..a47e0c7c7290b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts @@ -51,102 +51,106 @@ async function fetchDependenciesTimeseriesStatistics({ apm: { events: [getProcessorEventForServiceDestinationStatistics(searchServiceDestinationMetrics)], }, - track_total_hits: false, - size: 0, - query: { - bool: { - filter: [ - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...rangeQuery(start, end), - ...getDocumentTypeFilterForServiceDestinationStatistics(searchServiceDestinationMetrics), - { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, - ], - }, - }, - aggs: { - dependencies: { - terms: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, + body: { + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...rangeQuery(start, end), + ...getDocumentTypeFilterForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, + ], }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start, - end, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: start, - max: end, - }, - }, - aggs: { - // latency - latency_sum: { - sum: { - field: getLatencyFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + }, + aggs: { + dependencies: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start, + end, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: start, + max: end, }, }, - ...(searchServiceDestinationMetrics - ? { - latency_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + aggs: { + // latency + latency_sum: { + sum: { + field: getLatencyFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + ...(searchServiceDestinationMetrics + ? { + latency_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, - }, - } - : {}), - // error - ...(searchServiceDestinationMetrics - ? { - total_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + } + : {}), + // error + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, + } + : {}), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, }, - } - : {}), - failures: { - filter: { - term: { - [EVENT_OUTCOME]: EventOutcome.failure, }, - }, - aggs: { - ...(searchServiceDestinationMetrics - ? { - total_count: { - sum: { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), + aggs: { + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, }, - }, - } - : {}), + } + : {}), + }, }, - }, - // throughput - throughput: { - rate: { - ...(searchServiceDestinationMetrics - ? { - field: getDocCountFieldForServiceDestinationStatistics( - searchServiceDestinationMetrics - ), - } - : {}), - unit: 'minute', + // throughput + throughput: { + rate: { + ...(searchServiceDestinationMetrics + ? { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + } + : {}), + unit: 'minute', + }, }, }, }, From e85781d8bffea43cc08c8d0051636cae1a1cd633 Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Mon, 10 Mar 2025 15:29:11 -0300 Subject: [PATCH 4/6] fixing merge --- .../dependencies/index.spec.ts | 140 ------------------ .../dependencies/top_dependencies.spec.ts | 1 - 2 files changed, 141 deletions(-) delete mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts deleted file mode 100644 index d5164a0ea90c8..0000000000000 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import expect from '@kbn/expect'; -import { DependencyNode } from '@kbn/apm-plugin/common/connections'; -import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; -import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { NodeType } from '@kbn/apm-plugin/common/connections'; -import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; -import { roundNumber } from '../../utils/common'; -import { generateData, dataConfig } from './generate_data'; - -export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { - const apmApiClient = getService('apmApi'); - const synthtrace = getService('synthtrace'); - const start = new Date('2021-01-01T00:00:00.000Z').getTime(); - const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; - const bucketSize = Math.round((end - start) / (60 * 1000)); - - const serviceName = 'synth-go'; - - async function callApi() { - return await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/dependencies', - params: { - path: { serviceName }, - query: { - environment: 'production', - numBuckets: 20, - offset: '1d', - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - }, - }, - }); - } - - describe('Dependency for service', () => { - describe('when data is not loaded', () => { - it('handles empty state #1', async () => { - const { status, body } = await callApi(); - - expect(status).to.be(200); - expect(body.serviceDependencies).to.be.empty(); - }); - }); - - describe('when specific data is loaded', () => { - let dependencies: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - let apmSynthtraceEsClient: ApmSynthtraceEsClient; - - before(async () => { - apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); - await generateData({ apmSynthtraceEsClient, start, end }); - const response = await callApi(); - dependencies = response.body; - }); - - after(() => apmSynthtraceEsClient.clean()); - - it('returns one dependency', () => { - expect(dependencies.serviceDependencies.length).to.be(1); - }); - - it('returns correct dependency information', () => { - const location = dependencies.serviceDependencies[0].location as DependencyNode; - const { span } = dataConfig; - - expect(location.type).to.be(NodeType.dependency); - expect(location.dependencyName).to.be(span.destination); - expect(location.spanType).to.be(span.type); - expect(location.spanSubtype).to.be(span.subType); - expect(location).to.have.property('id'); - }); - - it("doesn't have previous stats", () => { - expect(dependencies.serviceDependencies[0].previousStats).to.be(null); - }); - - it('has an "impact" property', () => { - expect(dependencies.serviceDependencies[0].currentStats).to.have.property('impact'); - }); - - it('returns the correct latency', () => { - const { - currentStats: { latency }, - } = dependencies.serviceDependencies[0]; - - const { transaction } = dataConfig; - - const expectedValue = transaction.duration * 1000; - expect(latency.value).to.be(expectedValue); - expect(latency.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); - }); - - it('returns the correct throughput', () => { - const { - currentStats: { throughput }, - } = dependencies.serviceDependencies[0]; - const { rate, errorRate } = dataConfig; - - const expectedThroughput = rate + errorRate; - expect(roundNumber(throughput.value)).to.be(roundNumber(expectedThroughput)); - expect( - throughput.timeseries?.every( - ({ y }) => roundNumber(y) === roundNumber(expectedThroughput / bucketSize) - ) - ).to.be(true); - }); - - it('returns the correct total time', () => { - const { - currentStats: { totalTime }, - } = dependencies.serviceDependencies[0]; - const { rate, transaction, errorRate } = dataConfig; - - const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; - expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); - expect( - totalTime.timeseries?.every( - ({ y }) => roundNumber(y) === roundNumber(expectedValuePerBucket) - ) - ).to.be(true); - }); - - it('returns the correct error rate', () => { - const { - currentStats: { errorRate }, - } = dependencies.serviceDependencies[0]; - const { rate, errorRate: dataConfigErroRate } = dataConfig; - const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); - expect(errorRate.value).to.be(expectedValue); - expect(errorRate.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); - }); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts b/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts index d86feee3798ac..656c694fc9811 100644 --- a/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts +++ b/x-pack/test/apm_api_integration/tests/dependencies/top_dependencies.spec.ts @@ -74,7 +74,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when data is generated', () => { let topDependencies: TopDependencies; let topDependenciesStats: TopDependenciesStatistics; - let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { await generateData({ apmSynthtraceEsClient, start, end }); From 5977168adc9ee4daa81674cf80a8d3b8023623ee Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Tue, 11 Mar 2025 09:35:19 -0300 Subject: [PATCH 5/6] fixing ci --- .../tests/dependencies/service_dependencies.spec.ts | 4 ++-- .../tests/service_overview/dependencies/index.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test/apm_api_integration/tests/dependencies/service_dependencies.spec.ts b/x-pack/test/apm_api_integration/tests/dependencies/service_dependencies.spec.ts index be6b0898030cf..3a94c3bf3763e 100644 --- a/x-pack/test/apm_api_integration/tests/dependencies/service_dependencies.spec.ts +++ b/x-pack/test/apm_api_integration/tests/dependencies/service_dependencies.spec.ts @@ -67,7 +67,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const currentStatsLatencyValues = body.serviceDependencies[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + expect(currentStatsLatencyValues?.every(({ y }) => y === 1000000)).to.be(true); }); }); }); @@ -105,7 +105,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const currentStatsLatencyValues = body.serviceDependencies[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + expect(currentStatsLatencyValues?.every(({ y }) => y === 1000000)).to.be(true); }); }); }); diff --git a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts index 0e841e26ddde4..bcedc4d026885 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts @@ -263,7 +263,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { impact: 100, }); - const firstValue = roundNumber(opbeansNode?.currentStats.latency.timeseries[0].y); + const firstValue = roundNumber(opbeansNode?.currentStats.latency.timeseries?.[0].y); const lastValue = roundNumber(last(opbeansNode?.currentStats.latency.timeseries)?.y); expect(firstValue).to.be(roundNumber(20 / 3)); From fe8cbde059aa43ee2fb5d74d92f1bd94833def49 Mon Sep 17 00:00:00 2001 From: Caue Marcondes Date: Tue, 11 Mar 2025 14:26:16 -0300 Subject: [PATCH 6/6] fixing test --- .../dependencies/index.spec.ts | 572 +++--------------- 1 file changed, 91 insertions(+), 481 deletions(-) diff --git a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts index bcedc4d026885..995c186e81534 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.spec.ts @@ -5,30 +5,38 @@ * 2.0. */ +import { type DependencyNode, NodeType } from '@kbn/apm-plugin/common/connections'; +import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; import expect from '@kbn/expect'; -import { last, omit, pick, sortBy } from 'lodash'; -import { ValuesType } from 'utility-types'; -import { Node, NodeType } from '@kbn/apm-plugin/common/connections'; -import { - ENVIRONMENT_ALL, - ENVIRONMENT_NOT_DEFINED, -} from '@kbn/apm-plugin/common/environment_filter_values'; -import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import type { FtrProviderContext } from '../../../common/ftr_provider_context'; import { roundNumber } from '../../../utils'; -import archives from '../../../common/fixtures/es_archiver/archives_metadata'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { apmDependenciesMapping, createServiceDependencyDocs } from './es_utils'; +import { dataConfig, generateData } from './generate_data'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); const apmApiClient = getService('apmApiClient'); - const es = getService('es'); - - const archiveName = 'apm_8.0.0'; - const { start, end } = archives[archiveName]; - - function getName(node: Node) { - return node.type === NodeType.service ? node.serviceName : node.dependencyName; + const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const bucketSize = Math.round((end - start) / (60 * 1000)); + + const serviceName = 'synth-go'; + + async function callApi() { + return await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/dependencies', + params: { + path: { serviceName }, + query: { + environment: 'production', + numBuckets: 20, + offset: '1d', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + }, + }, + }); } registry.when( @@ -36,497 +44,99 @@ export default function ApiTest({ getService }: FtrProviderContext) { { config: 'basic', archives: [] }, () => { it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/dependencies`, - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start, - end, - numBuckets: 20, - environment: ENVIRONMENT_ALL.value, - }, - }, - }); - - expect(response.status).to.be(200); - expect(response.body.serviceDependencies).to.eql([]); + const { status, body } = await callApi(); + expect(status).to.be(200); + expect(body.serviceDependencies).to.be.empty(); }); } ); - registry.when( - 'Service overview dependencies when specific data is loaded', - { config: 'basic', archives: [] }, - () => { - let response: { - status: number; - body: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - }; - - const indices = { - metric: 'apm-dependencies-metric', - transaction: 'apm-dependencies-transaction', - span: 'apm-dependencies-span', - }; - - const startTime = new Date(start).getTime(); - const endTime = new Date(end).getTime(); + registry.when('Service overview dependencies', { config: 'basic', archives: [] }, () => { + describe('when data is loaded ', () => { + let dependencies: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - after(async () => { - const allIndices = Object.values(indices).join(','); - const indexExists = await es.indices.exists({ index: allIndices }); - if (indexExists) { - await es.indices.delete({ - index: allIndices, - }); - } - }); + after(() => apmSynthtraceEsClient.clean()); before(async () => { - await es.indices.create({ - index: indices.metric, - body: { - mappings: apmDependenciesMapping, - }, - }); - - await es.indices.create({ - index: indices.transaction, - body: { - mappings: apmDependenciesMapping, - }, - }); - - await es.indices.create({ - index: indices.span, - body: { - mappings: apmDependenciesMapping, - }, - }); - - const docs = [ - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'success', - responseTime: { - count: 2, - sum: 10, - }, - time: startTime, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'failure', - responseTime: { - count: 1, - sum: 10, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'postgres', - outcome: 'success', - responseTime: { - count: 1, - sum: 3, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node-via-proxy', - outcome: 'success', - responseTime: { - count: 1, - sum: 1, - }, - time: endTime - 1, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ]; - - const bulkActions = docs.reduce( - (prev, doc) => { - return [...prev, { index: { _index: indices[doc.processor.event] } }, doc]; - }, - [] as Array< - | { - index: { - _index: string; - }; - } - | ValuesType - > - ); - - await es.bulk({ - body: bulkActions, - refresh: 'wait_for', - }); - - response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/dependencies`, - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start, - end, - numBuckets: 20, - environment: ENVIRONMENT_ALL.value, - }, - }, - }); + await generateData({ apmSynthtraceEsClient, start, end }); + const response = await callApi(); + dependencies = response.body; }); - it('returns a 200', () => { - expect(response.status).to.be(200); + it('returns one dependency', () => { + expect(dependencies.serviceDependencies.length).to.be(1); }); - it('returns two dependencies', () => { - expect(response.body.serviceDependencies.length).to.be(2); - }); - - it('returns opbeans-node as a dependency', () => { - const opbeansNode = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'opbeans-node' - ); - - expect(opbeansNode !== undefined).to.be(true); + it('returns correct dependency information', () => { + const location = dependencies.serviceDependencies[0].location as DependencyNode; + const { span } = dataConfig; - const values = { - latency: roundNumber(opbeansNode?.currentStats.latency.value), - throughput: roundNumber(opbeansNode?.currentStats.throughput.value), - errorRate: roundNumber(opbeansNode?.currentStats.errorRate.value), - impact: opbeansNode?.currentStats.impact, - ...pick(opbeansNode?.location, 'serviceName', 'type', 'agentName', 'environment'), - }; - - const count = 4; - const sum = 21; - const errors = 1; - - expect(values).to.eql({ - agentName: 'nodejs', - environment: ENVIRONMENT_NOT_DEFINED.value, - serviceName: 'opbeans-node', - type: 'service', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 100, - }); - - const firstValue = roundNumber(opbeansNode?.currentStats.latency.timeseries?.[0].y); - const lastValue = roundNumber(last(opbeansNode?.currentStats.latency.timeseries)?.y); - - expect(firstValue).to.be(roundNumber(20 / 3)); - expect(lastValue).to.be(1); + expect(location.type).to.be(NodeType.dependency); + expect(location.dependencyName).to.be(span.destination); + expect(location.spanType).to.be(span.type); + expect(location.spanSubtype).to.be(span.subType); + expect(location).to.have.property('id'); }); - it('returns postgres as an external dependency', () => { - const postgres = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'postgres' - ); - - expect(postgres !== undefined).to.be(true); - - const values = { - latency: roundNumber(postgres?.currentStats.latency.value), - throughput: roundNumber(postgres?.currentStats.throughput.value), - errorRate: roundNumber(postgres?.currentStats.errorRate.value), - impact: postgres?.currentStats.impact, - ...pick(postgres?.location, 'spanType', 'spanSubtype', 'dependencyName', 'type'), - }; - - const count = 1; - const sum = 3; - const errors = 0; - - expect(values).to.eql({ - spanType: 'external', - spanSubtype: 'http', - dependencyName: 'postgres', - type: 'dependency', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 0, - }); - }); - } - ); - - registry.when( - 'Service overview dependencies when data is loaded', - { config: 'basic', archives: [archiveName] }, - () => { - let response: { - status: number; - body: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - }; - // eslint-disable-next-line mocha/no-sibling-hooks - before(async () => { - response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/dependencies`, - params: { - path: { serviceName: 'opbeans-python' }, - query: { - start, - end, - numBuckets: 20, - environment: ENVIRONMENT_ALL.value, - }, - }, - }); + it("doesn't have previous stats", () => { + expect(dependencies.serviceDependencies[0].previousStats).to.be(null); }); - it('returns a successful response', () => { - expect(response.status).to.be(200); + it('has an "impact" property', () => { + expect(dependencies.serviceDependencies[0].currentStats).to.have.property('impact'); }); - it('returns at least one item', () => { - expect(response.body.serviceDependencies.length).to.be.greaterThan(0); + it('returns the correct latency', () => { + const { + currentStats: { latency }, + } = dependencies.serviceDependencies[0]; - expectSnapshot(response.body.serviceDependencies.length).toMatchInline(`4`); + const { transaction } = dataConfig; - const { currentStats, ...firstItem } = sortBy( - response.body.serviceDependencies, - 'currentStats.impact' - ).reverse()[0]; - - expectSnapshot(firstItem.location).toMatchInline(` - Object { - "agentName": "dotnet", - "dependencyName": "opbeans:3000", - "environment": "production", - "id": "5948c153c2d8989f92a9c75ef45bb845f53e200d", - "serviceName": "opbeans-dotnet", - "type": "service", - } - `); - - expectSnapshot( - omit(currentStats, [ - 'errorRate.timeseries', - 'throughput.timeseries', - 'latency.timeseries', - 'totalTime.timeseries', - ]) - ).toMatchInline(` - Object { - "errorRate": Object { - "value": 0.163636363636364, - }, - "impact": 100, - "latency": Object { - "value": 1117085.74545455, - }, - "throughput": Object { - "value": 1.83333333333333, - }, - "totalTime": Object { - "value": 61439716, - }, - } - `); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect(latency.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); - it('returns the right names', () => { - const names = response.body.serviceDependencies.map((item) => getName(item.location)); - expectSnapshot(names.sort()).toMatchInline(` - Array [ - "elasticsearch", - "opbeans-dotnet", - "postgresql", - "redis", - ] - `); - }); + it('returns the correct throughput', () => { + const { + currentStats: { throughput }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate } = dataConfig; - it('returns the right service names', () => { - const serviceNames = response.body.serviceDependencies - .map((item) => - item.location.type === NodeType.service ? getName(item.location) : undefined + const expectedThroughput = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(expectedThroughput)); + expect( + throughput.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedThroughput / bucketSize) ) - .filter(Boolean); - - expectSnapshot(serviceNames.sort()).toMatchInline(` - Array [ - "opbeans-dotnet", - ] - `); - }); - - it('returns the right latency values', () => { - const latencyValues = sortBy( - response.body.serviceDependencies.map((item) => ({ - name: getName(item.location), - latency: item.currentStats.latency.value, - })), - 'name' - ); - - expectSnapshot(latencyValues).toMatchInline(` - Array [ - Object { - "latency": 9496.32291666667, - "name": "elasticsearch", - }, - Object { - "latency": 1117085.74545455, - "name": "opbeans-dotnet", - }, - Object { - "latency": 27826.9968314322, - "name": "postgresql", - }, - Object { - "latency": 1468.27242524917, - "name": "redis", - }, - ] - `); + ).to.be(true); }); - it('returns the right throughput values', () => { - const throughputValues = sortBy( - response.body.serviceDependencies.map((item) => ({ - name: getName(item.location), - throughput: item.currentStats.throughput.value, - })), - 'name' - ); - - expectSnapshot(throughputValues).toMatchInline(` - Array [ - Object { - "name": "elasticsearch", - "throughput": 3.2, - }, - Object { - "name": "opbeans-dotnet", - "throughput": 1.83333333333333, - }, - Object { - "name": "postgresql", - "throughput": 52.6, - }, - Object { - "name": "redis", - "throughput": 40.1333333333333, - }, - ] - `); - }); + it('returns the correct total time', () => { + const { + currentStats: { totalTime }, + } = dependencies.serviceDependencies[0]; + const { rate, transaction, errorRate } = dataConfig; - it('returns the right impact values', () => { - const impactValues = sortBy( - response.body.serviceDependencies.map((item) => ({ - name: getName(item.location), - impact: item.currentStats.impact, - })), - 'name' - ); - - expectSnapshot(impactValues).toMatchInline(` - Array [ - Object { - "impact": 0, - "name": "elasticsearch", - }, - Object { - "impact": 100, - "name": "opbeans-dotnet", - }, - Object { - "impact": 71.0403531954737, - "name": "postgresql", - }, - Object { - "impact": 1.41447268043525, - "name": "redis", - }, - ] - `); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); + expect( + totalTime.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedValuePerBucket) + ) + ).to.be(true); }); - it('returns the right totalTime values', () => { - const totalTimeValues = sortBy( - response.body.serviceDependencies.map((item) => ({ - name: getName(item.location), - totalTime: item.currentStats.totalTime.value, - })), - 'name' - ); - - expectSnapshot(totalTimeValues).toMatchInline(` - Array [ - Object { - "name": "elasticsearch", - "totalTime": 911647, - }, - Object { - "name": "opbeans-dotnet", - "totalTime": 61439716, - }, - Object { - "name": "postgresql", - "totalTime": 43911001, - }, - Object { - "name": "redis", - "totalTime": 1767800, - }, - ] - `); + it('returns the correct error rate', () => { + const { + currentStats: { errorRate }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect(errorRate.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); - } - ); + }); + }); }