diff --git a/x-pack/plugins/apm/server/lib/service_group_query_with_overflow.ts b/x-pack/plugins/apm/server/lib/service_group_query_with_overflow.ts new file mode 100644 index 0000000000000..f0f1009b7d036 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/service_group_query_with_overflow.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { kqlQuery, termQuery } from '@kbn/observability-plugin/server'; +import { SERVICE_NAME } from '../../common/es_fields/apm'; +import { ServiceGroup } from '../../common/service_groups'; + +export function serviceGroupWithOverflowQuery( + serviceGroup?: ServiceGroup | null +): QueryDslQueryContainer[] { + if (serviceGroup) { + const serviceGroupQuery = kqlQuery(serviceGroup?.kuery); + const otherBucketQuery = termQuery(SERVICE_NAME, '_other'); + + return [ + { + bool: { + should: [ + { + bool: { + filter: serviceGroupQuery, + }, + }, + { + bool: { + filter: otherBucketQuery, + }, + }, + ], + }, + }, + ]; + } + return []; +} diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_service_alerts.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_service_alerts.ts index f212172364f2e..a8968f9bbb390 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_service_alerts.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_service_alerts.ts @@ -24,8 +24,8 @@ import { SERVICE_NAME } from '../../../../common/es_fields/apm'; import { ServiceGroup } from '../../../../common/service_groups'; import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client'; import { environmentQuery } from '../../../../common/utils/environment_query'; -import { serviceGroupQuery } from '../../../lib/service_group_query'; import { MAX_NUMBER_OF_SERVICES } from './get_services_items'; +import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow'; interface ServiceAggResponse { buckets: Array< @@ -69,7 +69,7 @@ export async function getServicesAlerts({ ...termQuery(ALERT_STATUS, ALERT_STATUS_ACTIVE), ...rangeQuery(start, end), ...kqlQuery(kuery), - ...serviceGroupQuery(serviceGroup), + ...serviceGroupWithOverflowQuery(serviceGroup), ...termQuery(SERVICE_NAME, serviceName), ...environmentQuery(environment), ], diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts index 3cd6cfd2698db..9557d130522a6 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts @@ -27,8 +27,8 @@ import { calculateFailedTransactionRate, getOutcomeAggregation, } from '../../../lib/helpers/transaction_error_rate'; -import { serviceGroupQuery } from '../../../lib/service_group_query'; import { maybe } from '../../../../common/utils/maybe'; +import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow'; interface AggregationParams { environment: string; @@ -102,7 +102,7 @@ export async function getServiceTransactionStats({ ...rangeQuery(start, end), ...environmentQuery(environment), ...kqlQuery(kuery), - ...serviceGroupQuery(serviceGroup), + ...serviceGroupWithOverflowQuery(serviceGroup), ], }, }, diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_services_without_transactions.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_services_without_transactions.ts index 05af47e61e8ec..0eedb8494f21b 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_services_without_transactions.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_services_without_transactions.ts @@ -14,12 +14,12 @@ import { SERVICE_NAME, } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; -import { serviceGroupQuery } from '../../../lib/service_group_query'; import { ServiceGroup } from '../../../../common/service_groups'; import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; +import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow'; export interface ServicesWithoutTransactionsResponse { services: Array<{ @@ -82,7 +82,7 @@ export async function getServicesWithoutTransactions({ ...rangeQuery(start, end), ...environmentQuery(environment), ...kqlQuery(kuery), - ...serviceGroupQuery(serviceGroup), + ...serviceGroupWithOverflowQuery(serviceGroup), ], }, }, diff --git a/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/es_utils.ts b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/es_utils.ts new file mode 100644 index 0000000000000..03a0def567957 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/es_utils.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function createServiceTransactionMetricsDocs({ + time, + service, + agentName, + overflowCount, +}: { + time: number; + service: { + name: string; + environment?: string; + language?: string; + }; + agentName?: string; + overflowCount?: number; +}) { + return { + processor: { + event: 'metric' as const, + }, + '@timestamp': new Date(time).toISOString(), + ...(agentName && { + agent: { + name: agentName, + }, + }), + event: { + ingested: new Date(time).toISOString(), + }, + metricset: { + name: 'service_transaction', + }, + service, + ...(overflowCount && { + service_transaction: { + aggregation: { + overflow_count: overflowCount, + }, + }, + }), + observer: { + version: '8.9.0', + }, + }; +} diff --git a/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/generate_data.ts b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/generate_data.ts new file mode 100644 index 0000000000000..e688e6ac6836a --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/generate_data.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +export async function generateData({ + synthtraceEsClient, + start, + end, +}: { + synthtraceEsClient: ApmSynthtraceEsClient; + start: string; + end: string; +}) { + const synthServices = [ + apm + .service({ name: 'synth-go', environment: 'testing', agentName: 'go' }) + .instance('instance-1'), + apm + .service({ name: 'synth-java', environment: 'testing', agentName: 'java' }) + .instance('instance-2'), + ]; + + await synthtraceEsClient.index( + synthServices.map((service) => + timerange(start, end) + .interval('5m') + .rate(1) + .generator((timestamp) => + service + .transaction({ + transactionName: 'GET /api/product/list', + transactionType: 'request', + }) + .duration(2000) + .timestamp(timestamp) + .children( + service + .span({ + spanName: '/_search', + spanType: 'db', + spanSubtype: 'elasticsearch', + }) + .destination('elasticsearch') + .duration(100) + .success() + .timestamp(timestamp) + ) + .errors(service.error({ message: 'error 1', type: 'foo' }).timestamp(timestamp)) + ) + ) + ); +} diff --git a/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/service_group_with_overflow.spec.ts b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/service_group_with_overflow.spec.ts new file mode 100644 index 0000000000000..dba8a21521a89 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/service_group_with_overflow.spec.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { ValuesType } from 'utility-types'; +import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values'; +import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; +import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { createServiceGroupApi, deleteAllServiceGroups } from '../service_groups_api_methods'; +import { createServiceTransactionMetricsDocs } from './es_utils'; +import { generateData } from './generate_data'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const apmApiClient = getService('apmApiClient'); + const es = getService('es'); + const synthtraceEsClient = getService('synthtraceEsClient'); + + registry.when( + 'Display overflow bucket in Service Groups', + { config: 'basic', archives: [] }, + () => { + const indexName = 'metrics-apm.service_transaction.1m-default'; + const start = '2023-06-21T06:50:15.910Z'; + const end = '2023-06-21T06:59:15.910Z'; + const startTime = new Date(start).getTime() + 1000; + const OVERFLOW_SERVICE_NAME = '_other'; + let serviceGroupId: string; + + after(async () => { + await deleteAllServiceGroups(apmApiClient); + synthtraceEsClient.clean(); + }); + + before(async () => { + await generateData({ start, end, synthtraceEsClient }); + + const docs = [ + createServiceTransactionMetricsDocs({ + time: startTime, + service: { + name: OVERFLOW_SERVICE_NAME, + }, + overflowCount: 13, + }), + ]; + + const bulkActions = docs.reduce( + (prev, doc) => { + return [...prev, { create: { _index: indexName } }, doc]; + }, + [] as Array< + | { + create: { + _index: string; + }; + } + | ValuesType + > + ); + + await es.bulk({ + body: bulkActions, + refresh: 'wait_for', + }); + + const serviceGroup = { + groupName: 'overflowGroup', + kuery: 'service.name: synth-go or service.name: synth-java', + }; + const createResponse = await createServiceGroupApi({ apmApiClient, ...serviceGroup }); + expect(createResponse.status).to.be(200); + serviceGroupId = createResponse.body.id; + }); + + it('get the overflow bucket even though its not added explicitly in the Service Group', async () => { + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services`, + params: { + query: { + start, + end, + environment: ENVIRONMENT_ALL.value, + kuery: '', + serviceGroup: serviceGroupId, + probability: 1, + documentType: ApmDocumentType.ServiceTransactionMetric, + rollupInterval: RollupInterval.OneMinute, + }, + }, + }); + + const overflowBucket = response.body.items.find( + (service) => service.serviceName === OVERFLOW_SERVICE_NAME + ); + expect(overflowBucket?.serviceName).to.equal(OVERFLOW_SERVICE_NAME); + }); + } + ); +}