[9.0] [APM][OTel] Change the alerts query to include environment not defined value (#219228)

x-pack/solutions/observability/plugins/apm/common/environment_filter_values.ts

@@ -41,12 +41,8 @@ export const ENVIRONMENT_NOT_DEFINED = {
   label: getEnvironmentLabel(ENVIRONMENT_NOT_DEFINED_VALUE),
 };
 
-function isEnvironmentDefined(environment: string) {
-  return (
-    environment &&
-    environment !== ENVIRONMENT_NOT_DEFINED_VALUE &&
-    environment !== ENVIRONMENT_ALL_VALUE
-  );
+export function isEnvironmentDefined(environment: string) {
+  return environment && environment !== ENVIRONMENT_ALL_VALUE;
 }
 
 export function getEnvironmentEsField(environment: string) {

x-pack/solutions/observability/plugins/apm/common/utils/environment_query.test.ts

@@ -6,13 +6,17 @@
  */
 
 import { SERVICE_ENVIRONMENT } from '../es_fields/apm';
-import { ENVIRONMENT_NOT_DEFINED } from '../environment_filter_values';
+import { ENVIRONMENT_ALL, ENVIRONMENT_NOT_DEFINED } from '../environment_filter_values';
 import { environmentQuery } from './environment_query';
 
 describe('environmentQuery', () => {
   describe('when environment is an empty string', () => {
-    it('returns an empty query', () => {
-      expect(environmentQuery('')).toEqual([]);
+    it('returns ENVIRONMENT_NOT_DEFINED', () => {
+      expect(environmentQuery('')).toEqual([
+        {
+          term: { [SERVICE_ENVIRONMENT]: ENVIRONMENT_NOT_DEFINED.value },
+        },
+      ]);
     });
   });
 
@@ -24,10 +28,15 @@ describe('environmentQuery', () => {
     ]);
   });
 
+  it('creates a query for all environments', () => {
+    expect(environmentQuery(ENVIRONMENT_ALL.value)).toEqual([]);
+  });
+
   it('creates a query for missing service environments', () => {
-    expect(environmentQuery(ENVIRONMENT_NOT_DEFINED.value)[0]).toHaveProperty(
-      ['bool', 'must_not', 'exists', 'field'],
-      SERVICE_ENVIRONMENT
-    );
+    expect(environmentQuery(ENVIRONMENT_NOT_DEFINED.value)).toEqual([
+      {
+        term: { [SERVICE_ENVIRONMENT]: ENVIRONMENT_NOT_DEFINED.value },
+      },
+    ]);
   });
 });

x-pack/solutions/observability/plugins/apm/common/utils/environment_query.ts

@@ -13,6 +13,21 @@ import { SERVICE_NODE_NAME_MISSING } from '../service_nodes';
 export function environmentQuery(
   environment: string | undefined,
   field: string = SERVICE_ENVIRONMENT
+): QueryDslQueryContainer[] {
+  if (environment === ENVIRONMENT_ALL.value) {
+    return [];
+  }
+
+  if (!environment || environment === ENVIRONMENT_NOT_DEFINED.value) {
+    return [{ term: { [field]: ENVIRONMENT_NOT_DEFINED.value } }];
+  }
+
+  return [{ term: { [field]: environment } }];
+}
+
+export function environmentQueryAnnotations(
+  environment: string | undefined,
+  field: string = SERVICE_ENVIRONMENT
 ): QueryDslQueryContainer[] {
   if (!environment || environment === ENVIRONMENT_ALL.value) {
     return [];

x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.test.ts

@@ -724,7 +724,7 @@ describe('Error count alert', () => {
         threshold: 2,
         triggerValue: 5,
         viewInAppUrl:
-          'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
+          'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_NOT_DEFINED',
       },
       id: 'foo_ENVIRONMENT_NOT_DEFINED',
       payload: {
@@ -750,7 +750,7 @@ describe('Error count alert', () => {
         threshold: 2,
         triggerValue: 4,
         viewInAppUrl:
-          'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
+          'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_NOT_DEFINED',
       },
       id: 'foo_ENVIRONMENT_NOT_DEFINED',
       payload: {

x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.test.ts

@@ -309,7 +309,7 @@ describe('registerTransactionDurationRuleType', () => {
         transactionType: 'request',
         triggerValue: '5,500 ms',
         viewInAppUrl:
-          'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_ALL',
+          'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_NOT_DEFINED',
       },
       id: 'opbeans-java_ENVIRONMENT_NOT_DEFINED_request_tx-java',
       payload: {
@@ -325,6 +325,84 @@ describe('registerTransactionDurationRuleType', () => {
       },
     });
   });
+  it('sends alert when service.environment is ENVIRONMENT_ALL', async () => {
+    const { services, dependencies, executor } = createRuleTypeMocks();
+
+    registerTransactionDurationRuleType(dependencies);
+
+    services.scopedClusterClient.asCurrentUser.search.mockResponse({
+      hits: {
+        hits: [],
+        total: {
+          relation: 'eq',
+          value: 2,
+        },
+      },
+      aggregations: {
+        series: {
+          buckets: [
+            {
+              key: ['opbeans-java', 'ENVIRONMENT_ALL', 'request', 'tx-java'],
+              avgLatency: {
+                value: 5500000,
+              },
+            },
+          ],
+        },
+      },
+      took: 0,
+      timed_out: false,
+      _shards: {
+        failed: 0,
+        skipped: 0,
+        successful: 1,
+        total: 1,
+      },
+    });
+    services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
+
+    const params = {
+      threshold: 3000,
+      windowSize: 5,
+      windowUnit: 'm',
+      transactionType: 'request',
+      serviceName: 'opbeans-java',
+      aggregationType: 'avg',
+      groupBy: ['service.name', 'service.environment', 'transaction.type', 'transaction.name'],
+    };
+    await executor({ params });
+    expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(1);
+    expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
+      context: {
+        alertDetailsUrl: expect.stringContaining(
+          'http://localhost:5601/eyr/app/observability/alerts/'
+        ),
+        environment: 'All',
+        interval: '5 mins',
+        reason:
+          'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: All, type: request, name: tx-java. Alert when > 3.0 s.',
+        serviceName: 'opbeans-java',
+        threshold: 3000,
+        transactionName: 'tx-java',
+        transactionType: 'request',
+        triggerValue: '5,500 ms',
+        viewInAppUrl:
+          'http://localhost:5601/eyr/app/apm/services/opbeans-java?transactionType=request&environment=ENVIRONMENT_ALL',
+      },
+      id: 'opbeans-java_ENVIRONMENT_ALL_request_tx-java',
+      payload: {
+        'kibana.alert.evaluation.threshold': 3000000,
+        'kibana.alert.evaluation.value': 5500000,
+        'kibana.alert.reason':
+          'Avg. latency is 5.5 s in the last 5 mins for service: opbeans-java, env: All, type: request, name: tx-java. Alert when > 3.0 s.',
+        'processor.event': 'transaction',
+        'service.environment': 'ENVIRONMENT_ALL',
+        'service.name': 'opbeans-java',
+        'transaction.name': 'tx-java',
+        'transaction.type': 'request',
+      },
+    });
+  });
 
   it('sends alert when rule is configured with a filter query', async () => {
     const { services, dependencies, executor } = createRuleTypeMocks();

x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.test.ts

@@ -479,7 +479,7 @@ describe('Transaction error rate alert', () => {
         transactionType: 'type-foo',
         triggerValue: '10',
         viewInAppUrl:
-          'http://localhost:5601/eyr/app/apm/services/foo?transactionType=type-foo&environment=ENVIRONMENT_ALL',
+          'http://localhost:5601/eyr/app/apm/services/foo?transactionType=type-foo&environment=ENVIRONMENT_NOT_DEFINED',
       },
       id: 'foo_ENVIRONMENT_NOT_DEFINED_type-foo',
       payload: {

x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_derived_service_annotations.ts

@@ -14,7 +14,7 @@ import { isFiniteNumber } from '../../../../common/utils/is_finite_number';
 import type { Annotation } from '../../../../common/annotations';
 import { AnnotationType } from '../../../../common/annotations';
 import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm';
-import { environmentQuery } from '../../../../common/utils/environment_query';
+import { environmentQueryAnnotations } from '../../../../common/utils/environment_query';
 import {
   getBackwardCompatibleDocumentTypeFilter,
   getProcessorEventForTransactions,
@@ -39,7 +39,7 @@ export async function getDerivedServiceAnnotations({
   const filter: ESFilter[] = [
     { term: { [SERVICE_NAME]: serviceName } },
     ...getBackwardCompatibleDocumentTypeFilter(searchAggregatedTransactions),
-    ...environmentQuery(environment),
+    ...environmentQueryAnnotations(environment),
   ];
 
   const versions =

x-pack/solutions/observability/plugins/apm/server/routes/services/annotations/get_stored_annotations.ts

@@ -15,7 +15,7 @@ import {
 import type { ESSearchResponse } from '@kbn/es-types';
 import type { Annotation as ESAnnotation } from '@kbn/observability-plugin/common/annotations';
 import type { ScopedAnnotationsClient } from '@kbn/observability-plugin/server';
-import { environmentQuery } from '../../../../common/utils/environment_query';
+import { environmentQueryAnnotations } from '../../../../common/utils/environment_query';
 import type { Annotation } from '../../../../common/annotations';
 import { AnnotationType } from '../../../../common/annotations';
 import { SERVICE_NAME } from '../../../../common/es_fields/apm';
@@ -48,7 +48,7 @@ export function getStoredAnnotations({
             { term: { tags: 'apm' } },
             { term: { [SERVICE_NAME]: serviceName } },
             ...rangeQuery(start, end),
-            ...environmentQuery(environment),
+            ...environmentQueryAnnotations(environment),
           ],
         },
       },