From c7f99eb7a465ea8e262c8f8b2d5842000af0a990 Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Tue, 29 Apr 2025 12:16:58 +0200 Subject: [PATCH] [APM][OTel] Change the alerts query to include environment not defined value (#219228) Closes #217914 ## Summary This PR fixes the issue with the alerts filtering when the service environment is not defined. ![output](https://github.com/user-attachments/assets/05ef791a-1de4-4d06-bb2d-3b99b5f8afc4) ## Testing - Using synthtrace (any scenario) inject some data: for example: ``` node scripts/synthtrace simple_trace --scenarioOpts pipeline=apmToOtel --live --uniqueIds ``` - Change the scenario - the same one, so the same services have one synthrace env and one `undefined` (set the environment to `undefined`) - Run the scenario again in a different terminal (also using `live` without closing the previous one) - Configure some alert rules (that will generate alerts for both), for example, latency > 1ms or error count > 0 - Check the environment drop-down and the alerts tab (should behave the same as in the description example here - the env filter should be applied to the table results and the alert count badge - similar on the service inventory page - there we should retest with the https://github.com/elastic/kibana/pull/217899 changes when merged ) ![output1](https://github.com/user-attachments/assets/f77e6da1-8e60-4c8e-a1ee-c0e0e4a632e7) (cherry picked from commit bb025a82c94ef46105e914bf33a79cc7e13c3acc) # Conflicts: # x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts --- .../apm/common/environment_filter_values.ts | 8 +- .../common/utils/environment_query.test.ts | 23 +++-- .../apm/common/utils/environment_query.ts | 15 ++++ .../register_error_count_rule_type.test.ts | 4 +- ...ter_transaction_duration_rule_type.test.ts | 90 ++++++++++++++++++- ...r_transaction_error_rate_rule_type.test.ts | 2 +- .../get_derived_service_annotations.ts | 4 +- .../annotations/get_stored_annotations.ts | 4 +- 8 files changed, 129 insertions(+), 21 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts b/x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts index d8486c034822a..d4f86bbe4b00a 100644 --- a/x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts +++ b/x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts @@ -41,12 +41,8 @@ export const ENVIRONMENT_NOT_DEFINED = { label: getEnvironmentLabel(ENVIRONMENT_NOT_DEFINED_VALUE), }; -function isEnvironmentDefined(environment: string) { - return ( - environment && - environment !== ENVIRONMENT_NOT_DEFINED_VALUE && - environment !== ENVIRONMENT_ALL_VALUE - ); +export function isEnvironmentDefined(environment: string) { + return environment && environment !== ENVIRONMENT_ALL_VALUE; } export function getEnvironmentEsField(environment: string) { diff --git a/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.test.ts b/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.test.ts index f039b3f281fac..b996368096702 100644 --- a/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.test.ts +++ b/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.test.ts @@ -6,13 +6,17 @@ */ import { SERVICE_ENVIRONMENT } from '../es_fields/apm'; -import { ENVIRONMENT_NOT_DEFINED } from '../environment_filter_values'; +import { ENVIRONMENT_ALL, ENVIRONMENT_NOT_DEFINED } from '../environment_filter_values'; import { environmentQuery } from './environment_query'; describe('environmentQuery', () => { describe('when environment is an empty string', () => { - it('returns an empty query', () => { - expect(environmentQuery('')).toEqual([]); + it('returns ENVIRONMENT_NOT_DEFINED', () => { + expect(environmentQuery('')).toEqual([ + { + term: { [SERVICE_ENVIRONMENT]: ENVIRONMENT_NOT_DEFINED.value }, + }, + ]); }); }); @@ -24,10 +28,15 @@ describe('environmentQuery', () => { ]); }); + it('creates a query for all environments', () => { + expect(environmentQuery(ENVIRONMENT_ALL.value)).toEqual([]); + }); + it('creates a query for missing service environments', () => { - expect(environmentQuery(ENVIRONMENT_NOT_DEFINED.value)[0]).toHaveProperty( - ['bool', 'must_not', 'exists', 'field'], - SERVICE_ENVIRONMENT - ); + expect(environmentQuery(ENVIRONMENT_NOT_DEFINED.value)).toEqual([ + { + term: { [SERVICE_ENVIRONMENT]: ENVIRONMENT_NOT_DEFINED.value }, + }, + ]); }); }); diff --git a/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.ts b/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.ts index b31f1730d08b1..356a27f6a03f3 100644 --- a/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.ts +++ b/x-pack/solutions/observability/plugins/apm/common/utils/environment_query.ts @@ -13,6 +13,21 @@ import { SERVICE_NODE_NAME_MISSING } from '../service_nodes'; export function environmentQuery( environment: string | undefined, field: string = SERVICE_ENVIRONMENT +): QueryDslQueryContainer[] { + if (environment === ENVIRONMENT_ALL.value) { + return []; + } + + if (!environment || environment === ENVIRONMENT_NOT_DEFINED.value) { + return [{ term: { [field]: ENVIRONMENT_NOT_DEFINED.value } }]; + } + + return [{ term: { [field]: environment } }]; +} + +export function environmentQueryAnnotations( + environment: string | undefined, + field: string = SERVICE_ENVIRONMENT ): QueryDslQueryContainer[] { if (!environment || environment === ENVIRONMENT_ALL.value) { return []; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.test.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.test.ts index 463791e26a4aa..d1cad551ef4f9 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.test.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.test.ts @@ -820,7 +820,7 @@ describe('Error count alert', () => { threshold: 2, triggerValue: 5, viewInAppUrl: - 'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL', + 'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_NOT_DEFINED', }, id: 'foo_ENVIRONMENT_NOT_DEFINED', payload: { @@ -852,7 +852,7 @@ describe('Error count alert', () => { threshold: 2, triggerValue: 4, viewInAppUrl: - 'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL', + 'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_NOT_DEFINED', }, id: 'foo_ENVIRONMENT_NOT_DEFINED', payload: { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts index 9783062dc379c..8f31255c6dad8 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts @@ -347,7 +347,7 @@ describe('registerTransactionDurationRuleType', () => { transactionType: 'request', triggerValue: '5,500 ms', viewInAppUrl: - 'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_ALL', + 'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_NOT_DEFINED', }, id: 'opbeans-java_ENVIRONMENT_NOT_DEFINED_request_tx-java', payload: { @@ -363,6 +363,94 @@ describe('registerTransactionDurationRuleType', () => { }, }); }); + it('sends alert when service.environment is ENVIRONMENT_ALL', async () => { + const { services, dependencies, executor } = createRuleTypeMocks(); + + registerTransactionDurationRuleType(dependencies); + + services.scopedClusterClient.asCurrentUser.search.mockResponse({ + hits: { + hits: [], + total: { + relation: 'eq', + value: 2, + }, + }, + aggregations: { + series: { + buckets: [ + { + key: ['opbeans-java', 'ENVIRONMENT_ALL', 'request', 'tx-java'], + avgLatency: { + value: 5500000, + }, + }, + ], + }, + }, + took: 0, + timed_out: false, + _shards: { + failed: 0, + skipped: 0, + successful: 1, + total: 1, + }, + }); + services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' }); + + const params = { + threshold: 3000, + windowSize: 5, + windowUnit: 'm', + transactionType: 'request', + serviceName: 'opbeans-java', + aggregationType: 'avg', + groupBy: ['service.name', 'service.environment', 'transaction.type', 'transaction.name'], + }; + await executor({ params }); + expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(1); + expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({ + context: { + alertDetailsUrl: expect.stringContaining( + 'http://localhost:5601/eyr/app/observability/alerts/' + ), + environment: 'All', + grouping: { + service: { + environment: 'ENVIRONMENT_ALL', + name: 'opbeans-java', + }, + transaction: { + type: 'request', + name: 'tx-java', + }, + }, + interval: '5 mins', + reason: + 'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: All, type: request, name: tx-java. Alert when > 3.0 s.', + serviceName: 'opbeans-java', + threshold: 3000, + transactionName: 'tx-java', + transactionType: 'request', + triggerValue: '5,500 ms', + viewInAppUrl: + 'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_ALL', + }, + id: 'opbeans-java_ENVIRONMENT_ALL_request_tx-java', + payload: { + 'kibana.alert.evaluation.threshold': 3000000, + 'kibana.alert.evaluation.value': 5500000, + 'kibana.alert.reason': + 'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: All, type: request, name: tx-java. Alert when > 3.0 s.', + 'processor.event': 'transaction', + 'service.environment': 'ENVIRONMENT_ALL', + 'service.name': 'opbeans-java', + 'transaction.name': 'tx-java', + 'transaction.type': 'request', + }, + }); + }); it('sends alert when rule is configured with a filter query', async () => { const { services, dependencies, executor } = createRuleTypeMocks(); diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.test.ts b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.test.ts index 28d9687e9c781..2b3946fa4a987 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.test.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.test.ts @@ -516,7 +516,7 @@ describe('Transaction error rate alert', () => { transactionType: 'type-foo', triggerValue: '10', viewInAppUrl: - 'http://localhost:5601/eyr/app/apm/services/foo?transactionType=type-foo&environment=ENVIRONMENT_ALL', + 'http://localhost:5601/eyr/app/apm/services/foo?transactionType=type-foo&environment=ENVIRONMENT_NOT_DEFINED', }, id: 'foo_ENVIRONMENT_NOT_DEFINED_type-foo', payload: { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_derived_service_annotations.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_derived_service_annotations.ts index db1c5e028f449..1f2e1d4fa766a 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_derived_service_annotations.ts @@ -14,7 +14,7 @@ import { isFiniteNumber } from '../../../../common/utils/is_finite_number'; import type { Annotation } from '../../../../common/annotations'; import { AnnotationType } from '../../../../common/annotations'; import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; -import { environmentQuery } from '../../../../common/utils/environment_query'; +import { environmentQueryAnnotations } from '../../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, getProcessorEventForTransactions, @@ -39,7 +39,7 @@ export async function getDerivedServiceAnnotations({ const filter: ESFilter[] = [ { term: { [SERVICE_NAME]: serviceName } }, ...getBackwardCompatibleDocumentTypeFilter(searchAggregatedTransactions), - ...environmentQuery(environment), + ...environmentQueryAnnotations(environment), ]; const versions = diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_stored_annotations.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_stored_annotations.ts index 32c3de156faec..602aaf4190691 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_stored_annotations.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_stored_annotations.ts @@ -15,7 +15,7 @@ import { import type { ESSearchResponse } from '@kbn/es-types'; import type { Annotation as ESAnnotation } from '@kbn/observability-plugin/common/annotations'; import type { ScopedAnnotationsClient } from '@kbn/observability-plugin/server'; -import { environmentQuery } from '../../../../common/utils/environment_query'; +import { environmentQueryAnnotations } from '../../../../common/utils/environment_query'; import type { Annotation } from '../../../../common/annotations'; import { AnnotationType } from '../../../../common/annotations'; import { SERVICE_NAME } from '../../../../common/es_fields/apm'; @@ -48,7 +48,7 @@ export function getStoredAnnotations({ { term: { tags: 'apm' } }, { term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end), - ...environmentQuery(environment), + ...environmentQueryAnnotations(environment), ], }, },