[APM] Load list of services from ML/terms enum

x-pack/plugins/apm/common/service_inventory.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AgentName } from '../typings/es_schemas/ui/fields/agent';
+import { ServiceHealthStatus } from './service_health_status';
+
+export interface ServiceListItem {
+  serviceName: string;
+  healthStatus?: ServiceHealthStatus;
+  transactionType?: string;
+  agentName?: AgentName;
+  throughput?: number;
+  latency?: number | null;
+  transactionErrorRate?: number | null;
+  environments?: string[];
+}

x-pack/plugins/apm/public/components/app/service_inventory/index.tsx

@@ -12,21 +12,20 @@ import uuid from 'uuid';
 import { useAnomalyDetectionJobsContext } from '../../../context/anomaly_detection_jobs/use_anomaly_detection_jobs_context';
 import { useLegacyUrlParams } from '../../../context/url_params_context/use_url_params';
 import { useLocalStorage } from '../../../hooks/use_local_storage';
-import { useAnyOfApmParams } from '../../../hooks/use_apm_params';
+import { useApmParams } from '../../../hooks/use_apm_params';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
 import { useTimeRange } from '../../../hooks/use_time_range';
 import { SearchBar } from '../../shared/search_bar';
 import { getTimeRangeComparison } from '../../shared/time_comparison/get_time_range_comparison';
 import { ServiceList } from './service_list';
 import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout';
+import { joinByKey } from '../../../../common/utils/join_by_key';
 
 const initialData = {
   requestId: '',
-  mainStatisticsData: {
-    items: [],
-    hasHistoricalData: true,
-    hasLegacyData: false,
-  },
+  items: [],
+  hasHistoricalData: true,
+  hasLegacyData: false,
 };
 
 function useServicesFetcher() {
@@ -36,7 +35,7 @@ function useServicesFetcher() {
 
   const {
     query: { rangeFrom, rangeTo, environment, kuery },
-  } = useAnyOfApmParams('/services/{serviceName}', '/services');
+  } = useApmParams('/services');
 
   const { start, end } = useTimeRange({ rangeFrom, rangeTo });
 
@@ -47,7 +46,23 @@ function useServicesFetcher() {
     comparisonType,
   });
 
-  const { data = initialData, status: mainStatisticsStatus } = useFetcher(
+  const sortedAndFilteredServicesFetch = useFetcher(
+    (callApmApi) => {
+      return callApmApi('GET /internal/apm/sorted_and_filtered_services', {
+        params: {
+          query: {
+            start,
+            end,
+            environment,
+            kuery,
+          },
+        },
+      });
+    },
+    [start, end, environment, kuery]
+  );
+
+  const mainStatisticsFetch = useFetcher(
     (callApmApi) => {
       if (start && end) {
         return callApmApi('GET /internal/apm/services', {
@@ -62,17 +77,17 @@ function useServicesFetcher() {
         }).then((mainStatisticsData) => {
           return {
             requestId: uuid(),
-            mainStatisticsData,
+            ...mainStatisticsData,
           };
         });
       }
     },
     [environment, kuery, start, end]
   );
 
-  const { mainStatisticsData, requestId } = data;
+  const { data: mainStatisticsData = initialData } = mainStatisticsFetch;
 
-  const { data: comparisonData } = useFetcher(
+  const comparisonFetch = useFetcher(
     (callApmApi) => {
       if (start && end && mainStatisticsData.items.length) {
         return callApmApi('GET /internal/apm/services/detailed_statistics', {
@@ -96,20 +111,23 @@ function useServicesFetcher() {
     },
     // only fetches detailed statistics when requestId is invalidated by main statistics api call or offset is changed
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [requestId, offset],
+    [mainStatisticsData.requestId, offset],
     { preservePreviousData: false }
   );
 
   return {
-    mainStatisticsData,
-    mainStatisticsStatus,
-    comparisonData,
+    sortedAndFilteredServicesFetch,
+    mainStatisticsFetch,
+    comparisonFetch,
   };
 }
 
 export function ServiceInventory() {
-  const { mainStatisticsData, mainStatisticsStatus, comparisonData } =
-    useServicesFetcher();
+  const {
+    sortedAndFilteredServicesFetch,
+    mainStatisticsFetch,
+    comparisonFetch,
+  } = useServicesFetcher();
 
   const { anomalyDetectionSetupState } = useAnomalyDetectionJobsContext();
 
@@ -122,8 +140,13 @@ export function ServiceInventory() {
     !userHasDismissedCallout &&
     shouldDisplayMlCallout(anomalyDetectionSetupState);
 
-  const isLoading = mainStatisticsStatus === FETCH_STATUS.LOADING;
-  const isFailure = mainStatisticsStatus === FETCH_STATUS.FAILURE;
+  const isLoading =
+    sortedAndFilteredServicesFetch.status === FETCH_STATUS.LOADING ||
+    (sortedAndFilteredServicesFetch.status === FETCH_STATUS.SUCCESS &&
+      sortedAndFilteredServicesFetch.data?.services.length === 0 &&
+      mainStatisticsFetch.status === FETCH_STATUS.LOADING);
+
+  const isFailure = mainStatisticsFetch.status === FETCH_STATUS.FAILURE;
   const noItemsMessage = (
     <EuiEmptyPrompt
       title={
@@ -137,6 +160,14 @@ export function ServiceInventory() {
     />
   );
 
+  const items = joinByKey(
+    [
+      ...(sortedAndFilteredServicesFetch.data?.services ?? []),
+      ...(mainStatisticsFetch.data?.items ?? []),
+    ],
+    'serviceName'
+  );
+
   return (
     <>
       <SearchBar showTimeComparison />
@@ -154,8 +185,8 @@ export function ServiceInventory() {
           <ServiceList
             isLoading={isLoading}
             isFailure={isFailure}
-            items={mainStatisticsData.items}
-            comparisonData={comparisonData}
+            items={items}
+            comparisonData={comparisonFetch?.data}
             noItemsMessage={noItemsMessage}
           />
         </EuiFlexItem>

x-pack/plugins/apm/public/components/app/service_inventory/service_inventory.stories.tsx

@@ -30,6 +30,10 @@ const stories: Meta<{}> = {
             switch (endpoint) {
               case '/internal/apm/services':
                 return { items: [] };
+
+              case '/internal/apm/sorted_and_filtered_services':
+                return { services: [] };
+
               default:
                 return {};
             }

x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx

@@ -17,7 +17,6 @@ import { i18n } from '@kbn/i18n';
 import { TypeOf } from '@kbn/typed-react-router-config';
 import { orderBy } from 'lodash';
 import React, { useMemo } from 'react';
-import { ValuesType } from 'utility-types';
 import { NOT_AVAILABLE_LABEL } from '../../../../../common/i18n';
 import { ServiceHealthStatus } from '../../../../../common/service_health_status';
 import {
@@ -46,14 +45,11 @@ import {
   getTimeSeriesColor,
 } from '../../../shared/charts/helper/get_timeseries_color';
 import { HealthBadge } from './health_badge';
+import { ServiceListItem } from '../../../../../common/service_inventory';
 
-type ServiceListAPIResponse = APIReturnType<'GET /internal/apm/services'>;
-type Items = ServiceListAPIResponse['items'];
 type ServicesDetailedStatisticsAPIResponse =
   APIReturnType<'GET /internal/apm/services/detailed_statistics'>;
 
-type ServiceListItem = ValuesType<Items>;
-
 function formatString(value?: string | null) {
   return value || NOT_AVAILABLE_LABEL;
 }
@@ -239,7 +235,7 @@ export function getServiceColumns({
 }
 
 interface Props {
-  items: Items;
+  items: ServiceListItem[];
   comparisonData?: ServicesDetailedStatisticsAPIResponse;
   noItemsMessage?: React.ReactNode;
   isLoading: boolean;
@@ -287,9 +283,8 @@ export function ServiceList({
     ]
   );
 
-  const initialSortField = displayHealthStatus
-    ? 'healthStatus'
-    : 'transactionsPerMinute';
+  const initialSortField = displayHealthStatus ? 'healthStatus' : 'serviceName';
+  const initialSortDirection = displayHealthStatus ? 'desc' : 'asc';
 
   return (
     <EuiFlexGroup gutterSize="xs" direction="column" responsive={false}>
@@ -336,9 +331,9 @@ export function ServiceList({
           items={items}
           noItemsMessage={noItemsMessage}
           initialSortField={initialSortField}
-          initialSortDirection="desc"
+          initialSortDirection={initialSortDirection}
           sortFn={(itemsToSort, sortField, sortDirection) => {
-            // For healthStatus, sort items by healthStatus first, then by TPM
+            // For healthStatus, sort items by healthStatus first, then by name
             return sortField === 'healthStatus'
               ? orderBy(
                   itemsToSort,
@@ -348,9 +343,9 @@ export function ServiceList({
                         ? SERVICE_HEALTH_STATUS_ORDER.indexOf(item.healthStatus)
                         : -1;
                     },
-                    (item) => item.throughput ?? 0,
+                    (item) => item.serviceName.toLowerCase(),
                   ],
-                  [sortDirection, sortDirection]
+                  [sortDirection, sortDirection === 'asc' ? 'desc' : 'asc']
                 )
               : orderBy(
                   itemsToSort,

x-pack/plugins/apm/server/routes/services/get_services/get_health_statuses.ts

@@ -6,14 +6,16 @@
  */
 
 import { getSeverity } from '../../../../common/anomaly_detection';
-import { getServiceHealthStatus } from '../../../../common/service_health_status';
+import {
+  getServiceHealthStatus,
+  ServiceHealthStatus,
+} from '../../../../common/service_health_status';
 import { getServiceAnomalies } from '../../../routes/service_map/get_service_anomalies';
 import { ServicesItemsSetup } from './get_services_items';
 
 interface AggregationParams {
   environment: string;
   setup: ServicesItemsSetup;
-  searchAggregatedTransactions: boolean;
   start: number;
   end: number;
 }
@@ -23,7 +25,9 @@ export const getHealthStatuses = async ({
   setup,
   start,
   end,
-}: AggregationParams) => {
+}: AggregationParams): Promise<
+  Array<{ serviceName: string; healthStatus: ServiceHealthStatus }>
+> => {
   if (!setup.ml) {
     return [];
   }

x-pack/plugins/apm/server/routes/services/get_services/get_sorted_and_filtered_services.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from '@kbn/logging';
+import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames';
+import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values';
+import { Environment } from '../../../../common/environment_rt';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { joinByKey } from '../../../../common/utils/join_by_key';
+import { Setup } from '../../../lib/helpers/setup_request';
+import { getHealthStatuses } from './get_health_statuses';
+
+export async function getSortedAndFilteredServices({
+  setup,
+  start,
+  end,
+  environment,
+  logger,
+}: {
+  setup: Setup;
+  start: number;
+  end: number;
+  environment: Environment;
+  logger: Logger;
+}) {
+  const { apmEventClient } = setup;
+
+  async function getServicesFromTermsEnum() {
+    if (environment !== ENVIRONMENT_ALL.value) {
+      return [];
+    }
+    const response = await apmEventClient.termsEnum(
+      'get_services_from_terms_enum',
+      {
+        apm: {
+          events: [
+            ProcessorEvent.transaction,
+            ProcessorEvent.span,
+            ProcessorEvent.metric,
+            ProcessorEvent.error,
+          ],
+        },
+        body: {
+          size: 500,
+          field: SERVICE_NAME,
+        },
+      }
+    );
+
+    return response.terms;
+  }
+
+  const [servicesWithHealthStatuses, serviceNamesFromTermsEnum] =
+    await Promise.all([
+      getHealthStatuses({
+        setup,
+        start,
+        end,
+        environment,
+      }).catch((error) => {
+        logger.error(error);
+        return [];
+      }),
+      getServicesFromTermsEnum(),
+    ]);
+
+  const services = joinByKey(
+    [
+      ...servicesWithHealthStatuses,
+      ...serviceNamesFromTermsEnum.map((serviceName) => ({ serviceName })),
+    ],
+    'serviceName'
+  );
+
+  return services;
+}