From 3404816c9c8d32eb63540c757dc70f3255c355b8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Oct 2021 19:48:01 +0200 Subject: [PATCH] [APM] Generate breakdown metrics (#114390) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/elastic-apm-generator/src/index.ts | 1 + .../src/lib/base_span.ts | 31 +++- .../elastic-apm-generator/src/lib/entity.ts | 4 + .../src/lib/output/to_elasticsearch_output.ts | 4 +- .../elastic-apm-generator/src/lib/service.ts | 1 + .../elastic-apm-generator/src/lib/span.ts | 12 -- .../src/lib/transaction.ts | 28 ++-- .../src/lib/utils/aggregate.ts | 45 ++++++ .../src/lib/utils/create_picker.ts | 16 ++ .../src/lib/utils/get_breakdown_metrics.ts | 145 ++++++++++++++++++ .../lib/utils/get_span_destination_metrics.ts | 54 +++---- .../src/lib/utils/get_transaction_metrics.ts | 90 +++++------ .../src/scripts/examples/01_simple_trace.ts | 20 ++- .../test/scenarios/01_simple_trace.test.ts | 2 + .../scenarios/04_breakdown_metrics.test.ts | 105 +++++++++++++ .../01_simple_trace.test.ts.snap | 30 ++++ ...ervice_instances_transaction_statistics.ts | 3 + .../apm_api_integration/common/trace_data.ts | 8 +- .../instances_main_statistics.ts | 145 ++++++++++++++++++ 19 files changed, 621 insertions(+), 123 deletions(-) create mode 100644 packages/elastic-apm-generator/src/lib/utils/aggregate.ts create mode 100644 packages/elastic-apm-generator/src/lib/utils/create_picker.ts create mode 100644 packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts create mode 100644 packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts diff --git a/packages/elastic-apm-generator/src/index.ts b/packages/elastic-apm-generator/src/index.ts index fd83ce483ad4f..7007e92012a66 100644 --- a/packages/elastic-apm-generator/src/index.ts +++ b/packages/elastic-apm-generator/src/index.ts @@ -12,3 +12,4 @@ export { getTransactionMetrics } from './lib/utils/get_transaction_metrics'; export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics'; export { getObserverDefaults } from './lib/defaults/get_observer_defaults'; export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output'; +export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics'; diff --git a/packages/elastic-apm-generator/src/lib/base_span.ts b/packages/elastic-apm-generator/src/lib/base_span.ts index 24a51282687f4..6288c16d339b6 100644 --- a/packages/elastic-apm-generator/src/lib/base_span.ts +++ b/packages/elastic-apm-generator/src/lib/base_span.ts @@ -8,10 +8,12 @@ import { Fields } from './entity'; import { Serializable } from './serializable'; +import { Span } from './span'; +import { Transaction } from './transaction'; import { generateTraceId } from './utils/generate_id'; export class BaseSpan extends Serializable { - private _children: BaseSpan[] = []; + private readonly _children: BaseSpan[] = []; constructor(fields: Fields) { super({ @@ -22,20 +24,29 @@ export class BaseSpan extends Serializable { }); } - traceId(traceId: string) { - this.fields['trace.id'] = traceId; + parent(span: BaseSpan) { + this.fields['trace.id'] = span.fields['trace.id']; + this.fields['parent.id'] = span.isSpan() + ? span.fields['span.id'] + : span.fields['transaction.id']; + + if (this.isSpan()) { + this.fields['transaction.id'] = span.fields['transaction.id']; + } this._children.forEach((child) => { - child.fields['trace.id'] = traceId; + child.parent(this); }); + return this; } children(...children: BaseSpan[]) { - this._children.push(...children); children.forEach((child) => { - child.traceId(this.fields['trace.id']!); + child.parent(this); }); + this._children.push(...children); + return this; } @@ -52,4 +63,12 @@ export class BaseSpan extends Serializable { serialize(): Fields[] { return [this.fields, ...this._children.flatMap((child) => child.serialize())]; } + + isSpan(): this is Span { + return this.fields['processor.event'] === 'span'; + } + + isTransaction(): this is Transaction { + return this.fields['processor.event'] === 'transaction'; + } } diff --git a/packages/elastic-apm-generator/src/lib/entity.ts b/packages/elastic-apm-generator/src/lib/entity.ts index e0a048c876213..2a4beee652cf7 100644 --- a/packages/elastic-apm-generator/src/lib/entity.ts +++ b/packages/elastic-apm-generator/src/lib/entity.ts @@ -10,9 +10,11 @@ export type Fields = Partial<{ '@timestamp': number; 'agent.name': string; 'agent.version': string; + 'container.id': string; 'ecs.version': string; 'event.outcome': string; 'event.ingested': number; + 'host.name': string; 'metricset.name': string; 'observer.version': string; 'observer.version_major': number; @@ -42,6 +44,8 @@ export type Fields = Partial<{ 'span.destination.service.type': string; 'span.destination.service.response_time.sum.us': number; 'span.destination.service.response_time.count': number; + 'span.self_time.count': number; + 'span.self_time.sum.us': number; }>; export class Entity { diff --git a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts index ded94f9ad2276..b4cae1b41b9a6 100644 --- a/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts +++ b/packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts @@ -14,10 +14,12 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string return events.map((event) => { const values = { ...event, + ...getObserverDefaults(), '@timestamp': new Date(event['@timestamp']!).toISOString(), 'timestamp.us': event['@timestamp']! * 1000, 'ecs.version': '1.4', - ...getObserverDefaults(), + 'service.node.name': + event['service.node.name'] || event['container.id'] || event['host.name'], }; const document = {}; diff --git a/packages/elastic-apm-generator/src/lib/service.ts b/packages/elastic-apm-generator/src/lib/service.ts index 8ddbd827e842e..859afa18aab03 100644 --- a/packages/elastic-apm-generator/src/lib/service.ts +++ b/packages/elastic-apm-generator/src/lib/service.ts @@ -14,6 +14,7 @@ export class Service extends Entity { return new Instance({ ...this.fields, ['service.node.name']: instanceName, + 'container.id': instanceName, }); } } diff --git a/packages/elastic-apm-generator/src/lib/span.ts b/packages/elastic-apm-generator/src/lib/span.ts index da9ba9cdff722..36f7f44816d01 100644 --- a/packages/elastic-apm-generator/src/lib/span.ts +++ b/packages/elastic-apm-generator/src/lib/span.ts @@ -19,18 +19,6 @@ export class Span extends BaseSpan { }); } - children(...children: BaseSpan[]) { - super.children(...children); - - children.forEach((child) => - child.defaults({ - 'parent.id': this.fields['span.id'], - }) - ); - - return this; - } - duration(duration: number) { this.fields['span.duration.us'] = duration * 1000; return this; diff --git a/packages/elastic-apm-generator/src/lib/transaction.ts b/packages/elastic-apm-generator/src/lib/transaction.ts index 14ed6ac1ea85e..f615f46710996 100644 --- a/packages/elastic-apm-generator/src/lib/transaction.ts +++ b/packages/elastic-apm-generator/src/lib/transaction.ts @@ -11,6 +11,8 @@ import { Fields } from './entity'; import { generateEventId } from './utils/generate_id'; export class Transaction extends BaseSpan { + private _sampled: boolean = true; + constructor(fields: Fields) { super({ ...fields, @@ -19,19 +21,25 @@ export class Transaction extends BaseSpan { 'transaction.sampled': true, }); } - children(...children: BaseSpan[]) { - super.children(...children); - children.forEach((child) => - child.defaults({ - 'transaction.id': this.fields['transaction.id'], - 'parent.id': this.fields['transaction.id'], - }) - ); - return this; - } duration(duration: number) { this.fields['transaction.duration.us'] = duration * 1000; return this; } + + sample(sampled: boolean = true) { + this._sampled = sampled; + return this; + } + + serialize() { + const [transaction, ...spans] = super.serialize(); + + const events = [transaction]; + if (this._sampled) { + events.push(...spans); + } + + return events; + } } diff --git a/packages/elastic-apm-generator/src/lib/utils/aggregate.ts b/packages/elastic-apm-generator/src/lib/utils/aggregate.ts new file mode 100644 index 0000000000000..81b72f6fa01e9 --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/aggregate.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; +import { pickBy } from 'lodash'; +import objectHash from 'object-hash'; +import { Fields } from '../entity'; +import { createPicker } from './create_picker'; + +export function aggregate(events: Fields[], fields: string[]) { + const picker = createPicker(fields); + + const metricsets = new Map(); + + function getMetricsetKey(span: Fields) { + const timestamp = moment(span['@timestamp']).valueOf(); + return { + '@timestamp': timestamp - (timestamp % (60 * 1000)), + ...pickBy(span, picker), + }; + } + + for (const event of events) { + const key = getMetricsetKey(event); + const id = objectHash(key); + + let metricset = metricsets.get(id); + + if (!metricset) { + metricset = { + key: { ...key, 'processor.event': 'metric', 'processor.name': 'metric' }, + events: [], + }; + metricsets.set(id, metricset); + } + + metricset.events.push(event); + } + + return Array.from(metricsets.values()); +} diff --git a/packages/elastic-apm-generator/src/lib/utils/create_picker.ts b/packages/elastic-apm-generator/src/lib/utils/create_picker.ts new file mode 100644 index 0000000000000..7fce23b6fc966 --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/create_picker.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +export function createPicker(fields: string[]) { + const wildcards = fields + .filter((field) => field.endsWith('.*')) + .map((field) => field.replace('*', '')); + + return (value: unknown, key: string) => { + return fields.includes(key) || wildcards.some((field) => key.startsWith(field)); + }; +} diff --git a/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts new file mode 100644 index 0000000000000..8eae0941c6bdd --- /dev/null +++ b/packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import objectHash from 'object-hash'; +import { groupBy, pickBy } from 'lodash'; +import { Fields } from '../entity'; +import { createPicker } from './create_picker'; + +const instanceFields = [ + 'container.*', + 'kubernetes.*', + 'agent.*', + 'process.*', + 'cloud.*', + 'service.*', + 'host.*', +]; + +const instancePicker = createPicker(instanceFields); + +const metricsetPicker = createPicker([ + 'transaction.type', + 'transaction.name', + 'span.type', + 'span.subtype', +]); + +export function getBreakdownMetrics(events: Fields[]) { + const txWithSpans = groupBy( + events.filter( + (event) => event['processor.event'] === 'span' || event['processor.event'] === 'transaction' + ), + (event) => event['transaction.id'] + ); + + const metricsets: Map = new Map(); + + Object.keys(txWithSpans).forEach((transactionId) => { + const txEvents = txWithSpans[transactionId]; + const transaction = txEvents.find((event) => event['processor.event'] === 'transaction')!; + + const eventsById: Record = {}; + const activityByParentId: Record> = {}; + for (const event of txEvents) { + const id = + event['processor.event'] === 'transaction' ? event['transaction.id'] : event['span.id']; + eventsById[id!] = event; + + const parentId = event['parent.id']; + + if (!parentId) { + continue; + } + + if (!activityByParentId[parentId]) { + activityByParentId[parentId] = []; + } + + const from = event['@timestamp']! * 1000; + const to = + from + + (event['processor.event'] === 'transaction' + ? event['transaction.duration.us']! + : event['span.duration.us']!); + + activityByParentId[parentId].push({ from, to }); + } + + // eslint-disable-next-line guard-for-in + for (const id in eventsById) { + const event = eventsById[id]; + const activities = activityByParentId[id] || []; + + const timeStart = event['@timestamp']! * 1000; + + let selfTime = 0; + let lastMeasurement = timeStart; + const changeTimestamps = [ + ...new Set([ + timeStart, + ...activities.flatMap((activity) => [activity.from, activity.to]), + timeStart + + (event['processor.event'] === 'transaction' + ? event['transaction.duration.us']! + : event['span.duration.us']!), + ]), + ]; + + for (const timestamp of changeTimestamps) { + const hasActiveChildren = activities.some( + (activity) => activity.from < timestamp && activity.to >= timestamp + ); + + if (!hasActiveChildren) { + selfTime += timestamp - lastMeasurement; + } + + lastMeasurement = timestamp; + } + + const key = { + '@timestamp': event['@timestamp']! - (event['@timestamp']! % (30 * 1000)), + 'transaction.type': transaction['transaction.type'], + 'transaction.name': transaction['transaction.name'], + ...pickBy(event, metricsetPicker), + }; + + const instance = pickBy(event, instancePicker); + + const metricsetId = objectHash(key); + + let metricset = metricsets.get(metricsetId); + + if (!metricset) { + metricset = { + ...key, + ...instance, + 'processor.event': 'metric', + 'processor.name': 'metric', + 'metricset.name': `span_breakdown`, + 'span.self_time.count': 0, + 'span.self_time.sum.us': 0, + }; + + if (event['processor.event'] === 'transaction') { + metricset['span.type'] = 'app'; + } else { + metricset['span.type'] = event['span.type']; + metricset['span.subtype'] = event['span.subtype']; + } + + metricsets.set(metricsetId, metricset); + } + + metricset['span.self_time.count']!++; + metricset['span.self_time.sum.us']! += selfTime; + } + }); + + return Array.from(metricsets.values()); +} diff --git a/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts index 3740ad685735e..decf2f71a9be4 100644 --- a/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts +++ b/packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts @@ -6,46 +6,34 @@ * Side Public License, v 1. */ -import { pick } from 'lodash'; -import moment from 'moment'; -import objectHash from 'object-hash'; import { Fields } from '../entity'; +import { aggregate } from './aggregate'; export function getSpanDestinationMetrics(events: Fields[]) { const exitSpans = events.filter((event) => !!event['span.destination.service.resource']); - const metricsets = new Map(); + const metricsets = aggregate(exitSpans, [ + 'event.outcome', + 'agent.name', + 'service.environment', + 'service.name', + 'span.destination.service.resource', + ]); - function getSpanBucketKey(span: Fields) { - return { - '@timestamp': moment(span['@timestamp']).startOf('minute').valueOf(), - ...pick(span, [ - 'event.outcome', - 'agent.name', - 'service.environment', - 'service.name', - 'span.destination.service.resource', - ]), - }; - } - - for (const span of exitSpans) { - const key = getSpanBucketKey(span); - const id = objectHash(key); + return metricsets.map((metricset) => { + let count = 0; + let sum = 0; - let metricset = metricsets.get(id); - if (!metricset) { - metricset = { - ['processor.event']: 'metric', - ...key, - 'span.destination.service.response_time.sum.us': 0, - 'span.destination.service.response_time.count': 0, - }; - metricsets.set(id, metricset); + for (const event of metricset.events) { + count++; + sum += event['span.duration.us']!; } - metricset['span.destination.service.response_time.count']! += 1; - metricset['span.destination.service.response_time.sum.us']! += span['span.duration.us']!; - } - return [...Array.from(metricsets.values())]; + return { + ...metricset.key, + ['metricset.name']: 'span_destination', + 'span.destination.service.response_time.sum.us': sum, + 'span.destination.service.response_time.count': count, + }; + }); } diff --git a/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts b/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts index 62ecb9e20006f..4d46461c6dcc9 100644 --- a/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts +++ b/packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts @@ -6,10 +6,9 @@ * Side Public License, v 1. */ -import { pick, sortBy } from 'lodash'; -import moment from 'moment'; -import objectHash from 'object-hash'; +import { sortBy } from 'lodash'; import { Fields } from '../entity'; +import { aggregate } from './aggregate'; function sortAndCompressHistogram(histogram?: { values: number[]; counts: number[] }) { return sortBy(histogram?.values).reduce( @@ -30,60 +29,45 @@ function sortAndCompressHistogram(histogram?: { values: number[]; counts: number } export function getTransactionMetrics(events: Fields[]) { - const transactions = events.filter((event) => event['processor.event'] === 'transaction'); + const transactions = events + .filter((event) => event['processor.event'] === 'transaction') + .map((transaction) => { + return { + ...transaction, + ['trace.root']: transaction['parent.id'] === undefined, + }; + }); - const metricsets = new Map(); + const metricsets = aggregate(transactions, [ + 'trace.root', + 'transaction.name', + 'transaction.type', + 'event.outcome', + 'transaction.result', + 'agent.name', + 'service.environment', + 'service.name', + 'service.version', + 'host.name', + 'container.id', + 'kubernetes.pod.name', + ]); - function getTransactionBucketKey(transaction: Fields) { - return { - '@timestamp': moment(transaction['@timestamp']).startOf('minute').valueOf(), - 'trace.root': transaction['parent.id'] === undefined, - ...pick(transaction, [ - 'transaction.name', - 'transaction.type', - 'event.outcome', - 'transaction.result', - 'agent.name', - 'service.environment', - 'service.name', - 'service.version', - 'host.name', - 'container.id', - 'kubernetes.pod.name', - ]), + return metricsets.map((metricset) => { + const histogram = { + values: [] as number[], + counts: [] as number[], }; - } - for (const transaction of transactions) { - const key = getTransactionBucketKey(transaction); - const id = objectHash(key); - let metricset = metricsets.get(id); - if (!metricset) { - metricset = { - ...key, - ['processor.event']: 'metric', - 'transaction.duration.histogram': { - values: [], - counts: [], - }, - }; - metricsets.set(id, metricset); + for (const transaction of metricset.events) { + histogram.counts.push(1); + histogram.values.push(Number(transaction['transaction.duration.us'])); } - metricset['transaction.duration.histogram']?.counts.push(1); - metricset['transaction.duration.histogram']?.values.push( - Number(transaction['transaction.duration.us']) - ); - } - return [ - ...Array.from(metricsets.values()).map((metricset) => { - return { - ...metricset, - ['transaction.duration.histogram']: sortAndCompressHistogram( - metricset['transaction.duration.histogram'] - ), - _doc_count: metricset['transaction.duration.histogram']!.values.length, - }; - }), - ]; + return { + ...metricset.key, + 'transaction.duration.histogram': sortAndCompressHistogram(histogram), + _doc_count: metricset.events.length, + }; + }); } diff --git a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts index eef3e6cc40560..7aae2986919c8 100644 --- a/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts +++ b/packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts @@ -7,17 +7,18 @@ */ import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics } from '../..'; +import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics'; export function simpleTrace(from: number, to: number) { const instance = service('opbeans-go', 'production', 'go').instance('instance'); const range = timerange(from, to); - const transactionName = '100rpm (75% success) failed 1000ms'; + const transactionName = '100rpm (80% success) failed 1000ms'; const successfulTraceEvents = range - .interval('1m') - .rate(75) + .interval('30s') + .rate(40) .flatMap((timestamp) => instance .transaction(transactionName) @@ -31,14 +32,14 @@ export function simpleTrace(from: number, to: number) { .success() .destination('elasticsearch') .timestamp(timestamp), - instance.span('custom_operation', 'app').duration(50).success().timestamp(timestamp) + instance.span('custom_operation', 'custom').duration(100).success().timestamp(timestamp) ) .serialize() ); const failedTraceEvents = range - .interval('1m') - .rate(25) + .interval('30s') + .rate(10) .flatMap((timestamp) => instance .transaction(transactionName) @@ -50,5 +51,10 @@ export function simpleTrace(from: number, to: number) { const events = successfulTraceEvents.concat(failedTraceEvents); - return events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)); + return [ + ...events, + ...getTransactionMetrics(events), + ...getSpanDestinationMetrics(events), + ...getBreakdownMetrics(events), + ]; } diff --git a/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts b/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts index 6bae70507dcbe..733093ce0a71c 100644 --- a/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts +++ b/packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts @@ -68,6 +68,7 @@ describe('simple trace', () => { expect(transaction).toEqual({ '@timestamp': 1609459200000, 'agent.name': 'java', + 'container.id': 'instance-1', 'event.outcome': 'success', 'processor.event': 'transaction', 'processor.name': 'transaction', @@ -89,6 +90,7 @@ describe('simple trace', () => { expect(span).toEqual({ '@timestamp': 1609459200050, 'agent.name': 'java', + 'container.id': 'instance-1', 'event.outcome': 'success', 'parent.id': 'e7433020f2745625', 'processor.event': 'span', diff --git a/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts b/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts new file mode 100644 index 0000000000000..aeb944f35faf6 --- /dev/null +++ b/packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { sumBy } from 'lodash'; +import { Fields } from '../../lib/entity'; +import { service } from '../../lib/service'; +import { timerange } from '../../lib/timerange'; +import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics'; + +describe('breakdown metrics', () => { + let events: Fields[]; + + const LIST_RATE = 2; + const LIST_SPANS = 2; + const ID_RATE = 4; + const ID_SPANS = 2; + const INTERVALS = 6; + + beforeEach(() => { + const javaService = service('opbeans-java', 'production', 'java'); + const javaInstance = javaService.instance('instance-1'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + + const range = timerange(start, start + INTERVALS * 30 * 1000 - 1); + + events = getBreakdownMetrics([ + ...range + .interval('30s') + .rate(LIST_RATE) + .flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/list') + .timestamp(timestamp) + .duration(1000) + .children( + javaInstance + .span('GET apm-*/_search', 'db', 'elasticsearch') + .timestamp(timestamp + 150) + .duration(500), + javaInstance.span('GET foo', 'db', 'redis').timestamp(timestamp).duration(100) + ) + .serialize() + ), + ...range + .interval('30s') + .rate(ID_RATE) + .flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/:id') + .timestamp(timestamp) + .duration(1000) + .children( + javaInstance + .span('GET apm-*/_search', 'db', 'elasticsearch') + .duration(500) + .timestamp(timestamp + 100) + .children( + javaInstance + .span('bar', 'external', 'http') + .timestamp(timestamp + 200) + .duration(100) + ) + ) + .serialize() + ), + ]).filter((event) => event['processor.event'] === 'metric'); + }); + + it('generates the right amount of breakdown metrics', () => { + expect(events.length).toBe(INTERVALS * (LIST_SPANS + 1 + ID_SPANS + 1)); + }); + + it('calculates breakdown metrics for the right amount of transactions and spans', () => { + expect(sumBy(events, (event) => event['span.self_time.count']!)).toBe( + INTERVALS * LIST_RATE * (LIST_SPANS + 1) + INTERVALS * ID_RATE * (ID_SPANS + 1) + ); + }); + + it('generates app metricsets for transaction self time', () => { + expect(events.some((event) => event['span.type'] === 'app' && !event['span.subtype'])).toBe( + true + ); + }); + + it('generates the right statistic', () => { + const elasticsearchSets = events.filter((event) => event['span.subtype'] === 'elasticsearch'); + + const expectedCountFromListTransaction = INTERVALS * LIST_RATE; + + const expectedCountFromIdTransaction = INTERVALS * ID_RATE; + + const expectedCount = expectedCountFromIdTransaction + expectedCountFromListTransaction; + + expect(sumBy(elasticsearchSets, (set) => set['span.self_time.count']!)).toBe(expectedCount); + + expect(sumBy(elasticsearchSets, (set) => set['span.self_time.sum.us']!)).toBe( + expectedCountFromListTransaction * 500 * 1000 + expectedCountFromIdTransaction * 400 * 1000 + ); + }); +}); diff --git a/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap b/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap index 6eec0ce38ba30..00a55cb87b125 100644 --- a/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap +++ b/packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap @@ -5,6 +5,7 @@ Array [ Object { "@timestamp": 1609459200000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -21,6 +22,7 @@ Array [ Object { "@timestamp": 1609459200050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "36c16f18e75058f8", "processor.event": "span", @@ -39,6 +41,7 @@ Array [ Object { "@timestamp": 1609459260000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -55,6 +58,7 @@ Array [ Object { "@timestamp": 1609459260050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "65ce74106eb050be", "processor.event": "span", @@ -73,6 +77,7 @@ Array [ Object { "@timestamp": 1609459320000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -89,6 +94,7 @@ Array [ Object { "@timestamp": 1609459320050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "91fa709d90625fff", "processor.event": "span", @@ -107,6 +113,7 @@ Array [ Object { "@timestamp": 1609459380000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -123,6 +130,7 @@ Array [ Object { "@timestamp": 1609459380050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "6c500d1d19835e68", "processor.event": "span", @@ -141,6 +149,7 @@ Array [ Object { "@timestamp": 1609459440000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -157,6 +166,7 @@ Array [ Object { "@timestamp": 1609459440050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "1b3246cc83595869", "processor.event": "span", @@ -175,6 +185,7 @@ Array [ Object { "@timestamp": 1609459500000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -191,6 +202,7 @@ Array [ Object { "@timestamp": 1609459500050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "12b49e3c83fe58d5", "processor.event": "span", @@ -209,6 +221,7 @@ Array [ Object { "@timestamp": 1609459560000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -225,6 +238,7 @@ Array [ Object { "@timestamp": 1609459560050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "d9272009dd4354a1", "processor.event": "span", @@ -243,6 +257,7 @@ Array [ Object { "@timestamp": 1609459620000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -259,6 +274,7 @@ Array [ Object { "@timestamp": 1609459620050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "bc52ca08063c505b", "processor.event": "span", @@ -277,6 +293,7 @@ Array [ Object { "@timestamp": 1609459680000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -293,6 +310,7 @@ Array [ Object { "@timestamp": 1609459680050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "186858dd88b75d59", "processor.event": "span", @@ -311,6 +329,7 @@ Array [ Object { "@timestamp": 1609459740000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -327,6 +346,7 @@ Array [ Object { "@timestamp": 1609459740050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "0d5f44d48189546c", "processor.event": "span", @@ -345,6 +365,7 @@ Array [ Object { "@timestamp": 1609459800000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -361,6 +382,7 @@ Array [ Object { "@timestamp": 1609459800050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "7483e0606e435c83", "processor.event": "span", @@ -379,6 +401,7 @@ Array [ Object { "@timestamp": 1609459860000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -395,6 +418,7 @@ Array [ Object { "@timestamp": 1609459860050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "f142c4cbc7f3568e", "processor.event": "span", @@ -413,6 +437,7 @@ Array [ Object { "@timestamp": 1609459920000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -429,6 +454,7 @@ Array [ Object { "@timestamp": 1609459920050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "2e3a47fa2d905519", "processor.event": "span", @@ -447,6 +473,7 @@ Array [ Object { "@timestamp": 1609459980000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -463,6 +490,7 @@ Array [ Object { "@timestamp": 1609459980050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "de5eaa1e47dc56b1", "processor.event": "span", @@ -481,6 +509,7 @@ Array [ Object { "@timestamp": 1609460040000, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "processor.event": "transaction", "processor.name": "transaction", @@ -497,6 +526,7 @@ Array [ Object { "@timestamp": 1609460040050, "agent.name": "java", + "container.id": "instance-1", "event.outcome": "success", "parent.id": "af7eac7ae61e576a", "processor.event": "span", diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts index 089282d6f1c34..ec76e0d35e5c0 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts @@ -109,6 +109,9 @@ export async function getServiceInstancesTransactionStatistics< filter: [ { term: { [SERVICE_NAME]: serviceName } }, { term: { [TRANSACTION_TYPE]: transactionType } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), ...rangeQuery(start, end), ...environmentQuery(environment), ...kqlQuery(kuery), diff --git a/x-pack/test/apm_api_integration/common/trace_data.ts b/x-pack/test/apm_api_integration/common/trace_data.ts index 9c96d3fa1e0b0..84bbb4beea4f4 100644 --- a/x-pack/test/apm_api_integration/common/trace_data.ts +++ b/x-pack/test/apm_api_integration/common/trace_data.ts @@ -6,6 +6,7 @@ */ import { + getBreakdownMetrics, getSpanDestinationMetrics, getTransactionMetrics, toElasticsearchOutput, @@ -20,7 +21,12 @@ export async function traceData(context: InheritedFtrProviderContext) { return { index: (events: any[]) => { const esEvents = toElasticsearchOutput( - events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)), + [ + ...events, + ...getTransactionMetrics(events), + ...getSpanDestinationMetrics(events), + ...getBreakdownMetrics(events), + ], '7.14.0' ); diff --git a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts index 2d165f4ceb902..cdf62053a821b 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { pick, sortBy } from 'lodash'; import moment from 'moment'; +import { service, timerange } from '@elastic/apm-generator'; import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; import { isFiniteNumber } from '../../../../plugins/apm/common/utils/is_finite_number'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -15,9 +16,12 @@ import archives from '../../common/fixtures/es_archiver/archives_metadata'; import { registry } from '../../common/registry'; import { LatencyAggregationType } from '../../../../plugins/apm/common/latency_aggregation_types'; +import { ENVIRONMENT_ALL } from '../../../../plugins/apm/common/environment_filter_values'; +import { SERVICE_NODE_NAME_MISSING } from '../../../../plugins/apm/common/service_nodes'; export default function ApiTest({ getService }: FtrProviderContext) { const apmApiClient = getService('apmApiClient'); + const traceData = getService('traceData'); const archiveName = 'apm_8.0.0'; const { start, end } = archives[archiveName]; @@ -278,4 +282,145 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } ); + + registry.when( + 'Service overview instances main statistics when data is generated', + { config: 'basic', archives: ['apm_8.0.0_empty'] }, + () => { + describe('for two go instances and one java instance', () => { + const GO_A_INSTANCE_RATE_SUCCESS = 10; + const GO_A_INSTANCE_RATE_FAILURE = 5; + const GO_B_INSTANCE_RATE_SUCCESS = 15; + + const JAVA_INSTANCE_RATE = 20; + + const rangeStart = new Date('2021-01-01T12:00:00.000Z').getTime(); + const rangeEnd = new Date('2021-01-01T12:15:00.000Z').getTime() - 1; + + before(async () => { + const goService = service('opbeans-go', 'production', 'go'); + const javaService = service('opbeans-java', 'production', 'java'); + + const goInstanceA = goService.instance('go-instance-a'); + const goInstanceB = goService.instance('go-instance-b'); + const javaInstance = javaService.instance('java-instance'); + + const interval = timerange(rangeStart, rangeEnd).interval('1m'); + + // include exit spans to generate span_destination metrics + // that should not be included + function withSpans(timestamp: number) { + return new Array(3).fill(undefined).map(() => + goInstanceA + .span('GET apm-*/_search', 'db', 'elasticsearch') + .timestamp(timestamp + 100) + .duration(300) + .destination('elasticsearch') + .success() + ); + } + + return traceData.index([ + ...interval.rate(GO_A_INSTANCE_RATE_SUCCESS).flatMap((timestamp) => + goInstanceA + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(GO_A_INSTANCE_RATE_FAILURE).flatMap((timestamp) => + goInstanceA + .transaction('GET /api/product/list') + .failure() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(GO_B_INSTANCE_RATE_SUCCESS).flatMap((timestamp) => + goInstanceB + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ...interval.rate(JAVA_INSTANCE_RATE).flatMap((timestamp) => + javaInstance + .transaction('GET /api/product/list') + .success() + .duration(500) + .timestamp(timestamp) + .children(...withSpans(timestamp)) + .serialize() + ), + ]); + }); + + after(async () => { + return traceData.clean(); + }); + + describe('for the go service', () => { + let body: APIReturnType<'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics'>; + + before(async () => { + body = ( + await apmApiClient.readUser({ + endpoint: + 'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics', + params: { + path: { + serviceName: 'opbeans-go', + }, + query: { + start: new Date(rangeStart).toISOString(), + end: new Date(rangeEnd + 1).toISOString(), + environment: ENVIRONMENT_ALL.value, + kuery: '', + latencyAggregationType: LatencyAggregationType.avg, + transactionType: 'request', + }, + }, + }) + ).body; + }); + + it('returns statistics for the go instances', () => { + const goAStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'go-instance-a' + ); + const goBStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'go-instance-b' + ); + + expect(goAStats?.throughput).to.eql( + GO_A_INSTANCE_RATE_SUCCESS + GO_A_INSTANCE_RATE_FAILURE + ); + + expect(goBStats?.throughput).to.eql(GO_B_INSTANCE_RATE_SUCCESS); + }); + + it('does not return data for the java service', () => { + const javaStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'java-instance' + ); + + expect(javaStats).to.be(undefined); + }); + + it('does not return data for missing service node name', () => { + const missingNameStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === SERVICE_NODE_NAME_MISSING + ); + + expect(missingNameStats).to.be(undefined); + }); + }); + }); + } + ); }