From 599a239f7edf1fb6cdd87f2d813a5946453ca187 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Thu, 3 Feb 2022 07:56:39 -0700 Subject: [PATCH] Appends the saved objects documents count to the CoreUsageData service (#124308) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit ad7c8de75aef8c5890ea4db4e08d1e465cdd8db7) # Conflicts: # docs/development/core/server/kibana-plugin-core-server.md # docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md --- ...re-server.mergesavedobjectmigrationmaps.md | 21 ++++++++++++++ .../core_usage_data_service.mock.ts | 1 + .../core_usage_data_service.test.ts | 12 ++++++++ .../core_usage_data_service.ts | 28 +++++++++++++++++-- src/core/server/core_usage_data/types.ts | 1 + src/core/server/server.api.md | 1 + .../collectors/core/core_usage_collector.ts | 10 +++++-- src/plugins/telemetry/schema/oss_plugins.json | 10 +++++-- 8 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md diff --git a/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md b/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md new file mode 100644 index 0000000000000..68cd580b57882 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md @@ -0,0 +1,21 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md) + +## mergeSavedObjectMigrationMaps variable + +Merges two saved object migration maps. + +If there is a migration for a given version on only one of the maps, that migration function will be used: + +mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '4.5.6': g }) -> { '1.2.3': f, '4.5.6': g } + +If there is a migration for a given version on both maps, the migrations will be composed: + +mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '1.2.3': g }) -> { '1.2.3': (doc, context) => f(g(doc, context), context) } + +Signature: + +```typescript +mergeSavedObjectMigrationMaps: (map1: SavedObjectMigrationMap, map2: SavedObjectMigrationMap) => SavedObjectMigrationMap +``` diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts index 331a3bbb9c028..fd8f219fb2d55 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.mock.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -140,6 +140,7 @@ const createStartContractMock = () => { alias: 'test_index', primaryStoreSizeBytes: 1, storeSizeBytes: 1, + savedObjectsDocsCount: 1, }, ], legacyUrlAliases: { diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 59f673bbe3dfa..e6e890b1a7dab 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -218,6 +218,11 @@ describe('CoreUsageDataService', () => { }, ], } as any); + elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({ + body: { + count: '15', + }, + } as any); elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({ body: [ { @@ -229,6 +234,11 @@ describe('CoreUsageDataService', () => { }, ], } as any); + elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({ + body: { + count: '10', + }, + } as any); elasticsearch.client.asInternalUser.search.mockResolvedValueOnce({ body: { hits: { total: { value: 6 } }, @@ -384,6 +394,7 @@ describe('CoreUsageDataService', () => { "docsCount": 10, "docsDeleted": 10, "primaryStoreSizeBytes": 2000, + "savedObjectsDocsCount": "15", "storeSizeBytes": 1000, }, Object { @@ -391,6 +402,7 @@ describe('CoreUsageDataService', () => { "docsCount": 20, "docsDeleted": 20, "primaryStoreSizeBytes": 4000, + "savedObjectsDocsCount": "10", "storeSizeBytes": 2000, }, ], diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index 46cf44239cb5e..bf7a7ec34c1f0 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -124,11 +124,11 @@ export class CoreUsageDataService return acc.add(index); }, new Set()) .values() - ).map((index) => { + ).map(async (index) => { // The _cat/indices API returns the _index_ and doesn't return a way // to map back from the index to the alias. So we have to make an API - // call for every alias - return elasticsearch.client.asInternalUser.cat + // call for every alias. The document count is the lucene document count. + const catIndicesResults = await elasticsearch.client.asInternalUser.cat .indices({ index, format: 'JSON', @@ -136,6 +136,7 @@ export class CoreUsageDataService }) .then(({ body }) => { const stats = body[0]; + return { alias: kibanaOrTaskManagerIndex(index, kibanaIndex), docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0, @@ -146,6 +147,27 @@ export class CoreUsageDataService : 0, }; }); + // We use the GET /_count API to get the number of saved objects + // to monitor if the cluster will hit the scalling limit of saved object migrations + const savedObjectsCounts = await elasticsearch.client.asInternalUser + .count({ + index, + }) + .then(({ body }) => { + return { + savedObjectsDocsCount: body.count ? body.count : 0, + }; + }); + this.logger.debug( + `Lucene documents count ${catIndicesResults.docsCount} from index ${catIndicesResults.alias}` + ); + this.logger.debug( + `Saved objects documents count ${savedObjectsCounts.savedObjectsDocsCount} from index ${catIndicesResults.alias}` + ); + return { + ...catIndicesResults, + ...savedObjectsCounts, + }; }) ); } diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 59e220fac4efe..0b09444da493d 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -177,6 +177,7 @@ export interface CoreServicesUsageData { docsDeleted: number; storeSizeBytes: number; primaryStoreSizeBytes: number; + savedObjectsDocsCount: number; }[]; legacyUrlAliases: { activeCount: number; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index d86c0a9e8656c..3d834a16abd6d 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -425,6 +425,7 @@ export interface CoreServicesUsageData { docsDeleted: number; storeSizeBytes: number; primaryStoreSizeBytes: number; + savedObjectsDocsCount: number; }[]; legacyUrlAliases: { activeCount: number; diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts index b1cf0ecd2213e..a208832baf719 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts @@ -355,14 +355,14 @@ export function getCoreUsageCollector( type: 'long', _meta: { description: - 'The number of documents in the index, including hidden nested documents.', + 'The number of lucene documents in the index, including hidden nested documents.', }, }, docsDeleted: { type: 'long', _meta: { description: - 'The number of deleted documents in the index, including hidden nested documents.', + 'The number of deleted lucene documents in the index, including hidden nested documents.', }, }, alias: { @@ -382,6 +382,12 @@ export function getCoreUsageCollector( description: 'The size in bytes of the index, for primaries and replicas.', }, }, + savedObjectsDocsCount: { + type: 'long', + _meta: { + description: 'The number of saved objects documents in the index.', + }, + }, }, }, legacyUrlAliases: { diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index d40bf9aa5d610..a3f61bf49c372 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -6248,13 +6248,13 @@ "docsCount": { "type": "long", "_meta": { - "description": "The number of documents in the index, including hidden nested documents." + "description": "The number of lucene documents in the index, including hidden nested documents." } }, "docsDeleted": { "type": "long", "_meta": { - "description": "The number of deleted documents in the index, including hidden nested documents." + "description": "The number of deleted lucene documents in the index, including hidden nested documents." } }, "alias": { @@ -6274,6 +6274,12 @@ "_meta": { "description": "The size in bytes of the index, for primaries and replicas." } + }, + "savedObjectsDocsCount": { + "type": "long", + "_meta": { + "description": "The number of saved objects documents in the index." + } } } }