[APM] Support specific fields when creating service groups (#142201)

x-pack/plugins/apm/common/service_groups.test.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  isSupportedField,
+  validateServiceGroupKuery,
+  SERVICE_GROUP_SUPPORTED_FIELDS,
+} from './service_groups';
+import {
+  TRANSACTION_TYPE,
+  TRANSACTION_DURATION,
+  SERVICE_FRAMEWORK_VERSION,
+} from './elasticsearch_fieldnames';
+
+describe('service_groups common utils', () => {
+  describe('isSupportedField', () => {
+    it('should allow supported fields', () => {
+      SERVICE_GROUP_SUPPORTED_FIELDS.map((field) => {
+        expect(isSupportedField(field)).toBe(true);
+      });
+    });
+    it('should reject unsupported fields', () => {
+      const unsupportedFields = [
+        TRANSACTION_TYPE,
+        TRANSACTION_DURATION,
+        SERVICE_FRAMEWORK_VERSION,
+      ];
+      unsupportedFields.map((field) => {
+        expect(isSupportedField(field)).toBe(false);
+      });
+    });
+  });
+  describe('validateServiceGroupKuery', () => {
+    it('should validate supported KQL filter for a service group', () => {
+      const result = validateServiceGroupKuery(
+        `service.name: testbeans* or agent.name: "nodejs"`
+      );
+      expect(result).toHaveProperty('isValidFields', true);
+      expect(result).toHaveProperty('isValidSyntax', true);
+      expect(result).not.toHaveProperty('message');
+    });
+    it('should return validation error when unsupported fields are used', () => {
+      const result = validateServiceGroupKuery(
+        `service.name: testbeans* or agent.name: "nodejs" or transaction.type: request`
+      );
+      expect(result).toHaveProperty('isValidFields', false);
+      expect(result).toHaveProperty('isValidSyntax', true);
+      expect(result).toHaveProperty(
+        'message',
+        'Query filter for service group does not support fields [transaction.type]'
+      );
+    });
+    it('should return parsing error when KQL is incomplete', () => {
+      const result = validateServiceGroupKuery(
+        `service.name: testbeans* or agent.name: "nod`
+      );
+      expect(result).toHaveProperty('isValidFields', false);
+      expect(result).toHaveProperty('isValidSyntax', false);
+      expect(result).toHaveProperty('message');
+      expect(result).not.toBe('');
+    });
+  });
+});

x-pack/plugins/apm/common/service_groups.ts

@@ -5,6 +5,18 @@
  * 2.0.
  */
 
+import { fromKueryExpression } from '@kbn/es-query';
+import { i18n } from '@kbn/i18n';
+import { getKueryFields } from './utils/get_kuery_fields';
+import {
+  AGENT_NAME,
+  SERVICE_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_LANGUAGE_NAME,
+} from './elasticsearch_fieldnames';
+
+const LABELS = 'labels'; // implies labels.* wildcard
+
 export const APM_SERVICE_GROUP_SAVED_OBJECT_TYPE = 'apm-service-group';
 export const SERVICE_GROUP_COLOR_DEFAULT = '#D1DAE7';
 export const MAX_NUMBER_OF_SERVICE_GROUPS = 500;
@@ -20,3 +32,51 @@ export interface SavedServiceGroup extends ServiceGroup {
   id: string;
   updatedAt: number;
 }
+
+export const SERVICE_GROUP_SUPPORTED_FIELDS = [
+  AGENT_NAME,
+  SERVICE_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_LANGUAGE_NAME,
+  LABELS,
+];
+
+export function isSupportedField(fieldName: string) {
+  return (
+    fieldName.startsWith(LABELS) ||
+    SERVICE_GROUP_SUPPORTED_FIELDS.includes(fieldName)
+  );
+}
+
+export function validateServiceGroupKuery(kuery: string): {
+  isValidFields: boolean;
+  isValidSyntax: boolean;
+  message?: string;
+} {
+  try {
+    const kueryFields = getKueryFields([fromKueryExpression(kuery)]);
+    const unsupportedKueryFields = kueryFields.filter(
+      (fieldName) => !isSupportedField(fieldName)
+    );
+    if (unsupportedKueryFields.length === 0) {
+      return { isValidFields: true, isValidSyntax: true };
+    }
+    return {
+      isValidFields: false,
+      isValidSyntax: true,
+      message: i18n.translate('xpack.apm.serviceGroups.invalidFields.message', {
+        defaultMessage:
+          'Query filter for service group does not support fields [{unsupportedFieldNames}]',
+        values: {
+          unsupportedFieldNames: unsupportedKueryFields.join(', '),
+        },
+      }),
+    };
+  } catch (error) {
+    return {
+      isValidFields: false,
+      isValidSyntax: false,
+      message: error.message,
+    };
+  }
+}

x-pack/plugins/apm/common/utils/environment_query.ts

@@ -17,7 +17,7 @@ import {
 import { SERVICE_NODE_NAME_MISSING } from '../service_nodes';
 
 export function environmentQuery(
-  environment: string
+  environment: string | undefined
 ): QueryDslQueryContainer[] {
   if (!environment || environment === ENVIRONMENT_ALL.value) {
     return [];

x-pack/plugins/apm/public/components/alerting/utils/fields.tsx

@@ -38,7 +38,9 @@ export function ServiceField({
       })}
     >
       <SuggestionsSelect
-        customOptions={allowAll ? [ENVIRONMENT_ALL] : undefined}
+        customOptions={
+          allowAll ? [{ label: allOptionText, value: '' }] : undefined
+        }
         customOptionText={i18n.translate(
           'xpack.apm.serviceNamesSelectCustomOptionText',
           {
@@ -106,7 +108,7 @@ export function TransactionTypeField({
   return (
     <PopoverExpression value={currentValue || allOptionText} title={label}>
       <SuggestionsSelect
-        customOptions={[ENVIRONMENT_ALL]}
+        customOptions={[{ label: allOptionText, value: '' }]}
         customOptionText={i18n.translate(
           'xpack.apm.transactionTypesSelectCustomOptionText',
           {

x-pack/plugins/apm/public/components/app/service_groups/service_group_save/select_services.tsx

@@ -27,6 +27,10 @@ import { KueryBar } from '../../../shared/kuery_bar';
 import { ServiceListPreview } from './service_list_preview';
 import type { StagedServiceGroup } from './save_modal';
 import { getDateRange } from '../../../../context/url_params_context/helpers';
+import {
+  validateServiceGroupKuery,
+  isSupportedField,
+} from '../../../../../common/service_groups';
 
 const CentralizedContainer = styled.div`
   display: flex;
@@ -39,13 +43,6 @@ const MAX_CONTAINER_HEIGHT = 600;
 const MODAL_HEADER_HEIGHT = 180;
 const MODAL_FOOTER_HEIGHT = 80;
 
-const suggestedFieldsWhitelist = [
-  'agent.name',
-  'service.name',
-  'service.language.name',
-  'service.environment',
-];
-
 const Container = styled.div`
   width: 600px;
   height: ${MAX_CONTAINER_HEIGHT}px;
@@ -70,6 +67,9 @@ export function SelectServices({
 }: Props) {
   const [kuery, setKuery] = useState(serviceGroup?.kuery || '');
   const [stagedKuery, setStagedKuery] = useState(serviceGroup?.kuery || '');
+  const [kueryValidationMessage, setKueryValidationMessage] = useState<
+    string | undefined
+  >();
 
   useEffect(() => {
     if (isEdit) {
@@ -78,6 +78,14 @@ export function SelectServices({
     }
   }, [isEdit, serviceGroup.kuery]);
 
+  useEffect(() => {
+    if (!stagedKuery) {
+      return;
+    }
+    const { message } = validateServiceGroupKuery(stagedKuery);
+    setKueryValidationMessage(message);
+  }, [stagedKuery]);
+
   const { start, end } = useMemo(
     () =>
       getDateRange({
@@ -122,6 +130,11 @@ export function SelectServices({
               }
             )}
           </EuiText>
+          {kueryValidationMessage && (
+            <EuiText color="danger" size="s">
+              {kueryValidationMessage}
+            </EuiText>
+          )}
           <EuiFlexGroup gutterSize="s">
             <EuiFlexItem>
               <KueryBar
@@ -144,10 +157,7 @@ export function SelectServices({
                       },
                     } = querySuggestion;
 
-                    return (
-                      fieldName.startsWith('label') ||
-                      suggestedFieldsWhitelist.includes(fieldName)
-                    );
+                    return isSupportedField(fieldName);
                   }
                   return true;
                 }}

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -17,7 +17,7 @@ import {
   APM_SERVICE_GROUP_SAVED_OBJECT_TYPE,
   MAX_NUMBER_OF_SERVICE_GROUPS,
 } from '../../../../common/service_groups';
-import { getKueryFields } from '../../helpers/get_kuery_fields';
+import { getKueryFields } from '../../../../common/utils/get_kuery_fields';
 import {
   AGENT_NAME,
   AGENT_VERSION,

x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { firstValueFrom } from 'rxjs';
+import {
+  IScopedClusterClient,
+  SavedObjectsClientContract,
+} from '@kbn/core/server';
+import {
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+  TRANSACTION_TYPE,
+  TRANSACTION_DURATION,
+} from '../../../../../common/elasticsearch_fieldnames';
+import { alertingEsClient } from '../../alerting_es_client';
+import {
+  getServiceGroupFields,
+  getServiceGroupFieldsAgg,
+} from '../get_service_group_fields';
+import { getApmIndices } from '../../../settings/apm_indices/get_apm_indices';
+import { RegisterRuleDependencies } from '../../register_apm_rule_types';
+
+export async function getServiceGroupFieldsForAnomaly({
+  config$,
+  scopedClusterClient,
+  savedObjectsClient,
+  serviceName,
+  environment,
+  transactionType,
+  timestamp,
+  bucketSpan,
+}: {
+  config$: RegisterRuleDependencies['config$'];
+  scopedClusterClient: IScopedClusterClient;
+  savedObjectsClient: SavedObjectsClientContract;
+  serviceName: string;
+  environment: string;
+  transactionType: string;
+  timestamp: number;
+  bucketSpan: number;
+}) {
+  const config = await firstValueFrom(config$);
+  const indices = await getApmIndices({
+    config,
+    savedObjectsClient,
+  });
+  const { transaction: index } = indices;
+
+  const params = {
+    index,
+    body: {
+      size: 0,
+      track_total_hits: false,
+      query: {
+        bool: {
+          filter: [
+            { term: { [SERVICE_NAME]: serviceName } },
+            { term: { [TRANSACTION_TYPE]: transactionType } },
+            { term: { [SERVICE_ENVIRONMENT]: environment } },
+            {
+              range: {
+                '@timestamp': {
+                  gte: timestamp,
+                  lte: timestamp + bucketSpan * 1000,
+                  format: 'epoch_millis',
+                },
+              },
+            },
+          ],
+        },
+      },
+      aggs: {
+        ...getServiceGroupFieldsAgg({
+          sort: [{ [TRANSACTION_DURATION]: { order: 'desc' as const } }],
+        }),
+      },
+    },
+  };
+
+  const response = await alertingEsClient({
+    scopedClusterClient,
+    params,
+  });
+  if (!response.aggregations) {
+    return {};
+  }
+  return getServiceGroupFields(response.aggregations);
+}