diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx index 6f7b743d8b779..b18f462b54171 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx @@ -13,7 +13,7 @@ import { getCytoscapeElements } from './get_cytoscape_elements'; import serviceMapResponse from './cytoscape-layout-test-response.json'; import { iconForNode } from './icons'; -const elementsFromResponses = getCytoscapeElements([serviceMapResponse], ''); +const elementsFromResponses = getCytoscapeElements(serviceMapResponse, ''); storiesOf('app/ServiceMap/Cytoscape', module).add( 'example', diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts index 9ba70646598fc..4017aa2e3cdd9 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/get_cytoscape_elements.ts @@ -4,166 +4,63 @@ * you may not use this file except in compliance with the Elastic License. */ import { ValuesType } from 'utility-types'; -import { sortBy, isEqual } from 'lodash'; -import { - Connection, - ConnectionNode -} from '../../../../../../../plugins/apm/common/service_map'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map'; import { getAPMHref } from '../../shared/Links/apm/APMLink'; -function getConnectionNodeId(node: ConnectionNode): string { - if ('destination.address' in node) { - // use a prefix to distinguish exernal destination ids from services - return `>${node['destination.address']}`; - } - return node['service.name']; -} - -function getConnectionId(connection: Connection) { - return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId( - connection.destination - )}`; -} export function getCytoscapeElements( - responses: ServiceMapAPIResponse[], + response: ServiceMapAPIResponse, search: string ) { - const discoveredServices = responses.flatMap( - response => response.discoveredServices - ); - - const serviceNodes = responses - .flatMap(response => response.services) - .map(service => ({ - ...service, - id: service['service.name'] - })); - - // maps destination.address to service.name if possible - function getConnectionNode(node: ConnectionNode) { - let mappedNode: ConnectionNode | undefined; - - if ('destination.address' in node) { - mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to; - } - - if (!mappedNode) { - mappedNode = node; - } - - return { - ...mappedNode, - id: getConnectionNodeId(mappedNode) - }; - } - - // build connections with mapped nodes - const connections = responses - .flatMap(response => response.connections) - .map(connection => { - const source = getConnectionNode(connection.source); - const destination = getConnectionNode(connection.destination); - - return { - source, - destination, - id: getConnectionId({ source, destination }) - }; - }) - .filter(connection => connection.source.id !== connection.destination.id); - - const nodes = connections - .flatMap(connection => [connection.source, connection.destination]) - .concat(serviceNodes); - - type ConnectionWithId = ValuesType; - type ConnectionNodeWithId = ValuesType; - - const connectionsById = connections.reduce((connectionMap, connection) => { - return { - ...connectionMap, - [connection.id]: connection - }; - }, {} as Record); + const { nodes, connections } = response; const nodesById = nodes.reduce((nodeMap, node) => { return { ...nodeMap, [node.id]: node }; - }, {} as Record); - - const cyNodes = (Object.values(nodesById) as ConnectionNodeWithId[]).map( - node => { - let data = {}; - - if ('service.name' in node) { - data = { - href: getAPMHref( - `/services/${node['service.name']}/service-map`, - search - ), - agentName: node['agent.name'], - frameworkName: node['service.framework.name'], - type: 'service' - }; - } - - if ('span.type' in node) { - data = { - // For nodes with span.type "db", convert it to "database". Otherwise leave it as-is. - type: node['span.type'] === 'db' ? 'database' : node['span.type'], - // Externals should not have a subtype so make it undefined if the type is external. - subtype: node['span.type'] !== 'external' && node['span.subtype'] - }; - } - - return { - group: 'nodes' as const, - data: { - id: node.id, - label: - 'service.name' in node - ? node['service.name'] - : node['destination.address'], - ...data - } + }, {} as Record>); + + const cyNodes = (Object.values(nodesById) as Array< + ValuesType + >).map(node => { + let data = {}; + + if ('service.name' in node) { + data = { + href: getAPMHref( + `/services/${node['service.name']}/service-map`, + search + ), + agentName: node['agent.name'], + frameworkName: node['service.framework.name'], + type: 'service' }; } - ); - - // instead of adding connections in two directions, - // we add a `bidirectional` flag to use in styling - // and hide the inverse edge when rendering - const dedupedConnections = (sortBy( - Object.values(connectionsById), - // make sure that order is stable - 'id' - ) as ConnectionWithId[]).reduce< - Array< - ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean } - > - >((prev, connection) => { - const reversedConnection = prev.find( - c => - c.destination.id === connection.source.id && - c.source.id === connection.destination.id - ); - if (reversedConnection) { - reversedConnection.bidirectional = true; - return prev.concat({ - ...connection, - isInverseEdge: true - }); + if ('span.type' in node) { + data = { + // For nodes with span.type "db", convert it to "database". Otherwise leave it as-is. + type: node['span.type'] === 'db' ? 'database' : node['span.type'], + // Externals should not have a subtype so make it undefined if the type is external. + subtype: node['span.type'] !== 'external' && node['span.subtype'] + }; } - return prev.concat(connection); - }, []); + return { + group: 'nodes' as const, + data: { + id: node.id, + label: + 'service.name' in node + ? node['service.name'] + : node['destination.address'], + ...data + } + }; + }); - const cyEdges = dedupedConnections.map(connection => { + const cyEdges = connections.map(connection => { return { group: 'edges' as const, classes: connection.isInverseEdge ? 'invisible' : undefined, diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx index 93aa3d406028c..6222a00a9e888 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -4,26 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { EuiBetaBadge } from '@elastic/eui'; import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { ElementDefinition } from 'cytoscape'; -import { find, isEqual } from 'lodash'; -import React, { - useCallback, - useEffect, - useMemo, - useRef, - useState -} from 'react'; -import { EuiBetaBadge } from '@elastic/eui'; +import React, { useMemo } from 'react'; import styled from 'styled-components'; import { isValidPlatinumLicense } from '../../../../../../../plugins/apm/common/service_map'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map'; -import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; import { useDeepObjectIdentity } from '../../../hooks/useDeepObjectIdentity'; +import { useFetcher } from '../../../hooks/useFetcher'; import { useLicense } from '../../../hooks/useLicense'; -import { useLoadingIndicator } from '../../../hooks/useLoadingIndicator'; import { useLocation } from '../../../hooks/useLocation'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { callApmApi } from '../../../services/rest/createCallApmApi'; @@ -64,13 +53,11 @@ const BetaBadgeContainer = styled.div` top: ${theme.gutterTypes.gutterSmall}; z-index: 1; /* The element containing the cytoscape canvas has z-index = 0. */ `; -const MAX_REQUESTS = 5; export function ServiceMap({ serviceName }: ServiceMapProps) { const license = useLicense(); const { search } = useLocation(); const { urlParams, uiFilters } = useUrlParams(); - const { notifications } = useApmPluginContext().core; const params = useDeepObjectIdentity({ start: urlParams.start, end: urlParams.end, @@ -82,95 +69,28 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { } }); - const renderedElements = useRef([]); - - const [responses, setResponses] = useState([]); - - const { setIsLoading } = useLoadingIndicator(); - - const [, _setUnusedState] = useState(false); - - const elements = useMemo(() => getCytoscapeElements(responses, search), [ - responses, - search - ]); - - const forceUpdate = useCallback(() => _setUnusedState(value => !value), []); - - const getNext = useCallback( - async (input: { reset?: boolean; after?: string | undefined }) => { - const { start, end, uiFilters: strippedUiFilters, ...query } = params; - - if (input.reset) { - renderedElements.current = []; - setResponses([]); - } - - if (start && end) { - setIsLoading(true); - try { - const data = await callApmApi({ - pathname: '/api/apm/service-map', - params: { - query: { - ...query, - start, - end, - uiFilters: JSON.stringify(strippedUiFilters), - after: input.after - } - } - }); - setResponses(resp => resp.concat(data)); - - const shouldGetNext = - responses.length + 1 < MAX_REQUESTS && data.after; - - if (shouldGetNext) { - await getNext({ after: data.after }); - } else { - setIsLoading(false); + const { data } = useFetcher(() => { + const { start, end } = params; + if (start && end) { + return callApmApi({ + pathname: '/api/apm/service-map', + params: { + query: { + ...params, + start, + end, + uiFilters: JSON.stringify(params.uiFilters) } - } catch (error) { - setIsLoading(false); - notifications.toasts.addError(error, { - title: i18n.translate('xpack.apm.errorServiceMapData', { - defaultMessage: `Error loading service connections` - }) - }); } - } - }, - [params, setIsLoading, responses.length, notifications.toasts] - ); - - useEffect(() => { - const loadServiceMaps = async () => { - await getNext({ reset: true }); - }; - - loadServiceMaps(); - - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [params]); - - useEffect(() => { - if (renderedElements.current.length === 0) { - renderedElements.current = elements; - return; + }); } + }, [params]); - const newElements = elements.filter(element => { - return !find(renderedElements.current, el => isEqual(el, element)); - }); - - if (newElements.length > 0 && renderedElements.current.length > 0) { - renderedElements.current = elements; - forceUpdate(); - } - }, [elements, forceUpdate]); + const elements = useMemo(() => { + return data ? getCytoscapeElements(data as any, search) : []; + }, [data, search]); - const { ref: wrapperRef, width, height } = useRefDimensions(); + const { ref, height, width } = useRefDimensions(); if (!license) { return null; @@ -179,10 +99,10 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { return isValidPlatinumLicense(license) ? (
${node['destination.address']}`; + } + return node['service.name']; +} + +function getConnectionId(connection: Connection) { + return `${getConnectionNodeId(connection.source)}~${getConnectionNodeId( + connection.destination + )}`; +} + +type ServiceMapResponse = ConnectionsResponse & { services: ServicesResponse }; + +export function dedupeConnections(response: ServiceMapResponse) { + const { discoveredServices, services, connections } = response; + + const serviceNodes = services.map(service => ({ + ...service, + id: service['service.name'] + })); + + // maps destination.address to service.name if possible + function getConnectionNode(node: ConnectionNode) { + let mappedNode: ConnectionNode | undefined; + + if ('destination.address' in node) { + mappedNode = discoveredServices.find(map => isEqual(map.from, node))?.to; + } + + if (!mappedNode) { + mappedNode = node; + } + + return { + ...mappedNode, + id: getConnectionNodeId(mappedNode) + }; + } + + // build connections with mapped nodes + const mappedConnections = connections + .map(connection => { + const source = getConnectionNode(connection.source); + const destination = getConnectionNode(connection.destination); + + return { + source, + destination, + id: getConnectionId({ source, destination }) + }; + }) + .filter(connection => connection.source.id !== connection.destination.id); + + const nodes = mappedConnections + .flatMap(connection => [connection.source, connection.destination]) + .concat(serviceNodes); + + const dedupedNodes: typeof nodes = []; + + nodes.forEach(node => { + if (!dedupedNodes.find(dedupedNode => isEqual(node, dedupedNode))) { + dedupedNodes.push(node); + } + }); + + type ConnectionWithId = ValuesType; + + const connectionsById = mappedConnections.reduce( + (connectionMap, connection) => { + return { + ...connectionMap, + [connection.id]: connection + }; + }, + {} as Record + ); + + // instead of adding connections in two directions, + // we add a `bidirectional` flag to use in styling + const dedupedConnections = (sortBy( + Object.values(connectionsById), + // make sure that order is stable + 'id' + ) as ConnectionWithId[]).reduce< + Array< + ConnectionWithId & { bidirectional?: boolean; isInverseEdge?: boolean } + > + >((prev, connection) => { + const reversedConnection = prev.find( + c => + c.destination.id === connection.source.id && + c.source.id === connection.destination.id + ); + + if (reversedConnection) { + reversedConnection.bidirectional = true; + return prev.concat({ + ...connection, + isInverseEdge: true + }); + } + + return prev.concat(connection); + }, []); + + return { + nodes: dedupedNodes, + connections: dedupedConnections + }; +} diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts index 85d71784b55c7..96acfb7986c68 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import { chunk } from 'lodash'; import { PromiseReturnType } from '../../../typings/common'; import { Setup, @@ -19,48 +19,61 @@ import { SERVICE_NAME, SERVICE_FRAMEWORK_NAME } from '../../../common/elasticsearch_fieldnames'; +import { dedupeConnections } from './dedupe_connections'; export interface IEnvOptions { setup: Setup & SetupTimeRange & SetupUIFilters; serviceName?: string; environment?: string; - after?: string; } async function getConnectionData({ setup, serviceName, - environment, - after + environment }: IEnvOptions) { - const { traceIds, after: nextAfter } = await getTraceSampleIds({ + const { traceIds } = await getTraceSampleIds({ setup, serviceName, - environment, - after + environment }); - const serviceMapData = traceIds.length - ? await getServiceMapFromTraceIds({ + const chunks = chunk( + traceIds, + setup.config['xpack.apm.serviceMapMaxTracesPerRequest'] + ); + + const init = { + connections: [], + discoveredServices: [] + }; + + if (!traceIds.length) { + return init; + } + + const chunkedResponses = await Promise.all( + chunks.map(traceIdsChunk => + getServiceMapFromTraceIds({ setup, serviceName, environment, - traceIds + traceIds: traceIdsChunk }) - : { connections: [], discoveredServices: [] }; + ) + ); - return { - after: nextAfter, - ...serviceMapData - }; + return chunkedResponses.reduce((prev, current) => { + return { + connections: prev.connections.concat(current.connections), + discoveredServices: prev.discoveredServices.concat( + current.discoveredServices + ) + }; + }); } async function getServicesData(options: IEnvOptions) { - // only return services on the first request for the global service map - if (options.after) { - return []; - } - const { setup } = options; const projection = getServicesProjection({ setup }); @@ -125,15 +138,19 @@ async function getServicesData(options: IEnvOptions) { ); } +export type ConnectionsResponse = PromiseReturnType; +export type ServicesResponse = PromiseReturnType; + export type ServiceMapAPIResponse = PromiseReturnType; + export async function getServiceMap(options: IEnvOptions) { const [connectionData, servicesData] = await Promise.all([ getConnectionData(options), getServicesData(options) ]); - return { + return dedupeConnections({ ...connectionData, services: servicesData - }; + }); } diff --git a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts index 463fe7f2cf640..f4e12df5d6a66 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts @@ -15,27 +15,24 @@ import { PROCESSOR_EVENT, SERVICE_NAME, SERVICE_ENVIRONMENT, - SPAN_TYPE, - SPAN_SUBTYPE, + TRACE_ID, DESTINATION_ADDRESS, - TRACE_ID + SPAN_TYPE, + SPAN_SUBTYPE } from '../../../common/elasticsearch_fieldnames'; -const MAX_CONNECTIONS_PER_REQUEST = 1000; const MAX_TRACES_TO_INSPECT = 1000; export async function getTraceSampleIds({ - after, serviceName, environment, setup }: { - after?: string; serviceName?: string; environment?: string; setup: Setup & SetupTimeRange & SetupUIFilters; }) { - const { start, end, client, indices } = setup; + const { start, end, client, indices, config } = setup; const rangeQuery = { range: rangeFilter(start, end) }; @@ -65,9 +62,15 @@ export async function getTraceSampleIds({ query.bool.filter.push({ term: { [SERVICE_ENVIRONMENT]: environment } }); } - const afterObj = after - ? { after: JSON.parse(Buffer.from(after, 'base64').toString()) } - : {}; + const fingerprintBucketSize = serviceName + ? config['xpack.apm.serviceMapFingerprintBucketSize'] + : config['xpack.apm.serviceMapFingerprintGlobalBucketSize']; + + const traceIdBucketSize = serviceName + ? config['xpack.apm.serviceMapTraceIdBucketSize'] + : config['xpack.apm.serviceMapTraceIdGlobalBucketSize']; + + const samplerShardSize = traceIdBucketSize * 10; const params = { index: [indices['apm_oss.spanIndices']], @@ -77,42 +80,57 @@ export async function getTraceSampleIds({ aggs: { connections: { composite: { - size: MAX_CONNECTIONS_PER_REQUEST, - ...afterObj, sources: [ - { [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } }, { - [SERVICE_ENVIRONMENT]: { - terms: { field: SERVICE_ENVIRONMENT, missing_bucket: true } + [DESTINATION_ADDRESS]: { + terms: { + field: DESTINATION_ADDRESS + } } }, { - [SPAN_TYPE]: { - terms: { field: SPAN_TYPE, missing_bucket: true } + [SERVICE_NAME]: { + terms: { + field: SERVICE_NAME + } } }, { - [SPAN_SUBTYPE]: { - terms: { field: SPAN_SUBTYPE, missing_bucket: true } + [SERVICE_ENVIRONMENT]: { + terms: { + field: SERVICE_ENVIRONMENT, + missing_bucket: true + } } }, { - [DESTINATION_ADDRESS]: { - terms: { field: DESTINATION_ADDRESS } + [SPAN_TYPE]: { + terms: { + field: SPAN_TYPE + } + } + }, + { + [SPAN_SUBTYPE]: { + terms: { + field: SPAN_SUBTYPE, + missing_bucket: true + } } } - ] + ], + size: fingerprintBucketSize }, aggs: { sample: { sampler: { - shard_size: 30 + shard_size: samplerShardSize }, aggs: { trace_ids: { terms: { field: TRACE_ID, - size: 10, + size: traceIdBucketSize, execution_hint: 'map' as const, // remove bias towards large traces by sorting on trace.id // which will be random-esque @@ -129,25 +147,9 @@ export async function getTraceSampleIds({ } }; - const tracesSampleResponse = await client.search< - { trace: { id: string } }, - typeof params - >(params); - - let nextAfter: string | undefined; - - const receivedAfterKey = - tracesSampleResponse.aggregations?.connections.after_key; - - if ( - receivedAfterKey && - (tracesSampleResponse.aggregations?.connections.buckets.length ?? 0) >= - MAX_CONNECTIONS_PER_REQUEST - ) { - nextAfter = Buffer.from(JSON.stringify(receivedAfterKey)).toString( - 'base64' - ); - } + const tracesSampleResponse = await client.search( + params + ); // make sure at least one trace per composite/connection bucket // is queried @@ -167,7 +169,6 @@ export async function getTraceSampleIds({ ); return { - after: nextAfter, traceIds }; } diff --git a/x-pack/plugins/apm/server/routes/service_map.ts b/x-pack/plugins/apm/server/routes/service_map.ts index bead0445d6ccc..a61a61e3ccaac 100644 --- a/x-pack/plugins/apm/server/routes/service_map.ts +++ b/x-pack/plugins/apm/server/routes/service_map.ts @@ -20,10 +20,12 @@ export const serviceMapRoute = createRoute(() => ({ path: '/api/apm/service-map', params: { query: t.intersection([ - t.partial({ environment: t.string, serviceName: t.string }), + t.partial({ + environment: t.string, + serviceName: t.string + }), uiFiltersRt, - rangeRt, - t.partial({ after: t.string }) + rangeRt ]) }, handler: async ({ context, request }) => { @@ -36,9 +38,9 @@ export const serviceMapRoute = createRoute(() => ({ const setup = await setupRequest(context, request); const { - query: { serviceName, environment, after } + query: { serviceName, environment } } = context.params; - return getServiceMap({ setup, serviceName, environment, after }); + return getServiceMap({ setup, serviceName, environment }); } }));