[Index Management] Remove _cat/indices API from server code

x-pack/plugins/index_lifecycle_management/__jest__/extend_index_management.test.tsx

@@ -41,9 +41,9 @@ const indexWithoutLifecyclePolicy: Index = {
   status: 'open',
   name: 'noPolicy',
   uuid: 'BJ-nrZYuSrG-jaofr6SeLg',
-  primary: '1',
-  replica: '1',
-  documents: '1',
+  primary: 1,
+  replica: 1,
+  documents: 1,
   documents_deleted: '0',
   size: '3.4kb',
   primary_size: '3.4kb',
@@ -61,9 +61,9 @@ const indexWithLifecyclePolicy: Index = {
   status: 'open',
   name: 'testy3',
   uuid: 'XL11TLa3Tvq298_dMUzLHQ',
-  primary: '1',
-  replica: '1',
-  documents: '2',
+  primary: 1,
+  replica: 1,
+  documents: 2,
   documents_deleted: '0',
   size: '6.5kb',
   primary_size: '6.5kb',
@@ -89,9 +89,9 @@ const indexWithLifecycleError = {
   status: 'open',
   name: 'testy3',
   uuid: 'XL11TLa3Tvq298_dMUzLHQ',
-  primary: '1',
-  replica: '1',
-  documents: '2',
+  primary: 1,
+  replica: 1,
+  documents: 2,
   documents_deleted: '0',
   size: '6.5kb',
   primary_size: '6.5kb',

x-pack/plugins/index_management/common/types/indices.ts

@@ -52,14 +52,14 @@ export interface IndexSettings {
 }
 
 export interface Index {
-  health: string;
-  status: string;
+  health?: string;
+  status?: string;
   name: string;
-  uuid: string;
-  primary: string;
-  replica: string;
-  documents?: string;
-  size: any;
+  uuid?: string;
+  primary?: number | string;
+  replica?: number | string;
+  documents: number;
+  size: string;
   isFrozen: boolean;
   hidden: boolean;
   aliases: string | string[];

x-pack/plugins/index_management/server/lib/fetch_indices.test.ts

@@ -0,0 +1,185 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RequestMock, routeDependencies, RouterMock } from '../test/helpers';
+import { addBasePath } from '../routes/api';
+import { registerIndicesRoutes } from '../routes/api/indices';
+import {
+  createTestIndexResponse,
+  createTestIndexState,
+  createTestIndexStats,
+} from '../test/helpers/indices_fixtures';
+
+describe('[Index management API Routes] fetch indices lib function', () => {
+  const router = new RouterMock();
+
+  const getIndices = router.getMockESApiFn('indices.get');
+  const getIndicesStats = router.getMockESApiFn('indices.stats');
+  const mockRequest: RequestMock = {
+    method: 'get',
+    path: addBasePath('/indices'),
+  };
+
+  beforeAll(() => {
+    registerIndicesRoutes({
+      ...routeDependencies,
+      router,
+    });
+  });
+
+  test('regular index', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        regular_index: createTestIndexState(),
+      },
+    });
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          regular_index: createTestIndexStats({ uuid: 'regular_index' }),
+        },
+      },
+    });
+
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [createTestIndexResponse({ name: 'regular_index', uuid: 'regular_index' })],
+    });
+  });
+  test('index with aliases', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        index_with_aliases: createTestIndexState({
+          aliases: { test_alias: {}, another_alias: {} },
+        }),
+      },
+    });
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          index_with_aliases: createTestIndexStats({ uuid: 'index_with_aliases' }),
+        },
+      },
+    });
+
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [
+        createTestIndexResponse({
+          aliases: ['test_alias', 'another_alias'],
+          name: 'index_with_aliases',
+          uuid: 'index_with_aliases',
+        }),
+      ],
+    });
+  });
+  test('frozen index', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        frozen_index: createTestIndexState({
+          // @ts-expect-error
+          settings: { index: { number_of_shards: 1, number_of_replicas: 1, frozen: 'true' } },
+        }),
+      },
+    });
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          frozen_index: createTestIndexStats({ uuid: 'frozen_index' }),
+        },
+      },
+    });
+
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [
+        createTestIndexResponse({
+          name: 'frozen_index',
+          uuid: 'frozen_index',
+          isFrozen: true,
+        }),
+      ],
+    });
+  });
+  test('hidden index', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        hidden_index: createTestIndexState({
+          settings: { index: { number_of_shards: 1, number_of_replicas: 1, hidden: 'true' } },
+        }),
+      },
+    });
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          hidden_index: createTestIndexStats({ uuid: 'hidden_index' }),
+        },
+      },
+    });
+
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [
+        createTestIndexResponse({
+          name: 'hidden_index',
+          uuid: 'hidden_index',
+          hidden: true,
+        }),
+      ],
+    });
+  });
+  test('data stream index', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        data_stream_index: createTestIndexState({
+          data_stream: 'test_data_stream',
+        }),
+      },
+    });
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          data_stream_index: createTestIndexStats({ uuid: 'data_stream_index' }),
+        },
+      },
+    });
+
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [
+        createTestIndexResponse({
+          name: 'data_stream_index',
+          uuid: 'data_stream_index',
+          data_stream: 'test_data_stream',
+        }),
+      ],
+    });
+  });
+  test('index missing in stats call', async () => {
+    getIndices.mockResolvedValue({
+      body: {
+        index_missing_stats: createTestIndexState(),
+      },
+    });
+    // simulates when an index has been deleted after get indices call
+    // deleted index won't be present in the indices stats call response
+    getIndicesStats.mockResolvedValue({
+      body: {
+        indices: {
+          some_other_index: createTestIndexStats({ uuid: 'some_other_index' }),
+        },
+      },
+    });
+    await expect(router.runRequest(mockRequest)).resolves.toEqual({
+      body: [
+        createTestIndexResponse({
+          name: 'index_missing_stats',
+          uuid: undefined,
+          health: undefined,
+          status: undefined,
+          documents: 0,
+          size: '0b',
+        }),
+      ],
+    });
+  });
+});

x-pack/plugins/index_management/server/lib/fetch_indices.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ByteSizeValue } from '@kbn/config-schema';
 import { IScopedClusterClient } from 'kibana/server';
 import { IndexDataEnricher } from '../services';
 import { Index } from '../index';
@@ -19,49 +20,53 @@ async function fetchIndicesCall(
   const { body: indices } = await client.asCurrentUser.indices.get({
     index: indexNamesString,
     expand_wildcards: ['hidden', 'all'],
-    // only get specified properties in the response
-    filter_path: ['*.aliases', '*.settings.index.hidden', '*.data_stream'],
+    // only get specified index properties from ES to keep the response under 536MB
+    // node.js string length limit: https://github.com/nodejs/node/issues/33960
+    filter_path: [
+      '*.aliases',
+      '*.settings.index.number_of_shards',
+      '*.settings.index.number_of_replicas',
+      '*.settings.index.frozen',
+      '*.settings.index.hidden',
+      '*.data_stream',
+    ],
   });
 
   if (!Object.keys(indices).length) {
     return [];
   }
 
-  const { body: catHits } = await client.asCurrentUser.cat.indices({
-    format: 'json',
-    h: 'health,status,index,uuid,pri,rep,docs.count,sth,store.size',
-    expand_wildcards: ['hidden', 'all'],
+  const {
+    body: { indices: indicesStats = {} },
+  } = await client.asCurrentUser.indices.stats({
     index: indexNamesString,
+    expand_wildcards: ['hidden', 'all'],
+    forbid_closed_indices: false,
+    metric: ['docs', 'store'],
+  });
+  const indicesNames = Object.keys(indices);
+  return indicesNames.map((indexName: string) => {
+    const indexData = indices[indexName];
+    const indexStats = indicesStats[indexName];
+    const aliases = Object.keys(indexData.aliases!);
+    return {
+      // @ts-expect-error new property https://github.com/elastic/elasticsearch-specification/issues/1253
+      health: indexStats?.health,
+      // @ts-expect-error new property https://github.com/elastic/elasticsearch-specification/issues/1253
+      status: indexStats?.status,
+      name: indexName,
+      uuid: indexStats?.uuid,
+      primary: indexData.settings?.index?.number_of_shards,
+      replica: indexData.settings?.index?.number_of_replicas,
+      documents: indexStats?.total?.docs?.count ?? 0,
+      size: new ByteSizeValue(indexStats?.total?.store?.size_in_bytes ?? 0).toString(),
+      // @ts-expect-error
+      isFrozen: indexData.settings?.index?.frozen === 'true',
+      aliases: aliases.length ? aliases : 'none',
+      hidden: indexData.settings?.index?.hidden === 'true',
+      data_stream: indexData.data_stream,
+    };
   });
-
-  // System indices may show up in _cat APIs, as these APIs are primarily used for troubleshooting
-  // For now, we filter them out and only return index information for the indices we have
-  // In the future, we should migrate away from using cat APIs (https://github.com/elastic/kibana/issues/57286)
-  return catHits.reduce((decoratedIndices, hit) => {
-    const index = indices[hit.index!];
-
-    if (typeof index !== 'undefined') {
-      const aliases = Object.keys(index.aliases!);
-
-      decoratedIndices.push({
-        health: hit.health!,
-        status: hit.status!,
-        name: hit.index!,
-        uuid: hit.uuid!,
-        primary: hit.pri!,
-        replica: hit.rep!,
-        documents: hit['docs.count'],
-        size: hit['store.size'],
-        isFrozen: hit.sth === 'true', // sth value coming back as a string from ES
-        aliases: aliases.length ? aliases : 'none',
-        // @ts-expect-error @elastic/elasticsearch https://github.com/elastic/elasticsearch-specification/issues/532
-        hidden: index.settings?.index.hidden === 'true',
-        data_stream: index.data_stream!,
-      });
-    }
-
-    return decoratedIndices;
-  }, [] as Index[]);
 }
 
 export const fetchIndices = async (