[7.17] Appends the saved objects documents count to the CoreUsageData service (#124308)

docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md

@@ -0,0 +1,21 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md)
+
+## mergeSavedObjectMigrationMaps variable
+
+Merges two saved object migration maps.
+
+If there is a migration for a given version on only one of the maps, that migration function will be used:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '4.5.6': g }<!-- -->) -<!-- -->&gt; { '1.2.3': f, '4.5.6': g }
+
+If there is a migration for a given version on both maps, the migrations will be composed:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '1.2.3': g }<!-- -->) -<!-- -->&gt; { '1.2.3': (doc, context) =<!-- -->&gt; f(g(doc, context), context) }
+
+<b>Signature:</b>
+
+```typescript
+mergeSavedObjectMigrationMaps: (map1: SavedObjectMigrationMap, map2: SavedObjectMigrationMap) => SavedObjectMigrationMap
+```

src/core/server/core_usage_data/core_usage_data_service.mock.ts

@@ -138,6 +138,7 @@ const createStartContractMock = () => {
                 alias: 'test_index',
                 primaryStoreSizeBytes: 1,
                 storeSizeBytes: 1,
+                savedObjectsDocsCount: 1,
               },
             ],
             legacyUrlAliases: {

src/core/server/core_usage_data/core_usage_data_service.test.ts

@@ -177,6 +177,11 @@ describe('CoreUsageDataService', () => {
             },
           ],
         } as any);
+        elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({
+          body: {
+            count: '15',
+          },
+        } as any);
         elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({
           body: [
             {
@@ -188,6 +193,11 @@ describe('CoreUsageDataService', () => {
             },
           ],
         } as any);
+        elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({
+          body: {
+            count: '10',
+          },
+        } as any);
         elasticsearch.client.asInternalUser.search.mockResolvedValueOnce({
           body: {
             hits: { total: { value: 6 } },
@@ -343,13 +353,15 @@ describe('CoreUsageDataService', () => {
                     "docsCount": 10,
                     "docsDeleted": 10,
                     "primaryStoreSizeBytes": 2000,
+                    "savedObjectsDocsCount": "15",
                     "storeSizeBytes": 1000,
                   },
                   Object {
                     "alias": ".kibana_task_manager",
                     "docsCount": 20,
                     "docsDeleted": 20,
                     "primaryStoreSizeBytes": 4000,
+                    "savedObjectsDocsCount": "10",
                     "storeSizeBytes": 2000,
                   },
                 ],

src/core/server/core_usage_data/core_usage_data_service.ts

@@ -133,18 +133,19 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
             return index != null ? acc.add(index) : acc;
           }, new Set<string>())
           .values()
-      ).map((index) => {
+      ).map(async (index) => {
         // The _cat/indices API returns the _index_ and doesn't return a way
         // to map back from the index to the alias. So we have to make an API
-        // call for every alias
-        return elasticsearch.client.asInternalUser.cat
+        // call for every alias. The document count is the lucene document count.
+        const catIndicesResults = await elasticsearch.client.asInternalUser.cat
           .indices<any[]>({
             index,
             format: 'JSON',
             bytes: 'b',
           })
           .then(({ body }) => {
             const stats = body[0];
+
             return {
               alias: kibanaOrTaskManagerIndex(index, this.kibanaConfig!.index),
               docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0,
@@ -155,6 +156,27 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
                 : 0,
             };
           });
+        // We use the GET <index>/_count API to get the number of saved objects
+        // to monitor if the cluster will hit the scalling limit of saved object migrations
+        const savedObjectsCounts = await elasticsearch.client.asInternalUser
+          .count({
+            index,
+          })
+          .then(({ body }) => {
+            return {
+              savedObjectsDocsCount: body.count ? body.count : 0,
+            };
+          });
+        this.logger.debug(
+          `Lucene documents count ${catIndicesResults.docsCount} from index ${catIndicesResults.alias}`
+        );
+        this.logger.debug(
+          `Saved objects documents count ${savedObjectsCounts.savedObjectsDocsCount} from index ${catIndicesResults.alias}`
+        );
+        return {
+          ...catIndicesResults,
+          ...savedObjectsCounts,
+        };
       })
     );
   }

src/core/server/core_usage_data/types.ts

@@ -177,6 +177,7 @@ export interface CoreServicesUsageData {
       docsDeleted: number;
       storeSizeBytes: number;
       primaryStoreSizeBytes: number;
+      savedObjectsDocsCount: number;
     }[];
     legacyUrlAliases: {
       activeCount: number;

src/core/server/server.api.md

@@ -416,6 +416,7 @@ export interface CoreServicesUsageData {
             docsDeleted: number;
             storeSizeBytes: number;
             primaryStoreSizeBytes: number;
+            savedObjectsDocsCount: number;
         }[];
         legacyUrlAliases: {
             activeCount: number;

src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts

@@ -355,14 +355,14 @@ export function getCoreUsageCollector(
                 type: 'long',
                 _meta: {
                   description:
-                    'The number of documents in the index, including hidden nested documents.',
+                    'The number of lucene documents in the index, including hidden nested documents.',
                 },
               },
               docsDeleted: {
                 type: 'long',
                 _meta: {
                   description:
-                    'The number of deleted documents in the index, including hidden nested documents.',
+                    'The number of deleted lucene documents in the index, including hidden nested documents.',
                 },
               },
               alias: {
@@ -382,6 +382,12 @@ export function getCoreUsageCollector(
                   description: 'The size in bytes of the index, for primaries and replicas.',
                 },
               },
+              savedObjectsDocsCount: {
+                type: 'long',
+                _meta: {
+                  description: 'The number of saved objects documents in the index.',
+                },
+              },
             },
           },
           legacyUrlAliases: {

src/plugins/telemetry/schema/oss_plugins.json

@@ -6248,13 +6248,13 @@
                       "docsCount": {
                         "type": "long",
                         "_meta": {
-                          "description": "The number of documents in the index, including hidden nested documents."
+                          "description": "The number of lucene documents in the index, including hidden nested documents."
                         }
                       },
                       "docsDeleted": {
                         "type": "long",
                         "_meta": {
-                          "description": "The number of deleted documents in the index, including hidden nested documents."
+                          "description": "The number of deleted lucene documents in the index, including hidden nested documents."
                         }
                       },
                       "alias": {
@@ -6274,6 +6274,12 @@
                         "_meta": {
                           "description": "The size in bytes of the index, for primaries and replicas."
                         }
+                      },
+                      "savedObjectsDocsCount": {
+                        "type": "long",
+                        "_meta": {
+                          "description": "The number of saved objects documents in the index."
+                        }
                       }
                     }
                   }