diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index ede29f5e3d2c8..450af99a5b234 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -244,7 +244,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | --- | --- | | [APP\_WRAPPER\_CLASS](./kibana-plugin-core-server.app_wrapper_class.md) | The class name for top level \*and\* nested application wrappers to ensure proper layout | | [kibanaResponseFactory](./kibana-plugin-core-server.kibanaresponsefactory.md) | Set of helpers used to create KibanaResponse to form HTTP response on an incoming request. Should be returned as a result of [RequestHandler](./kibana-plugin-core-server.requesthandler.md) execution. | -| [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md) | Merges two saved object migration maps. | +| [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md) | Merges two saved object migration maps.If there is a migration for a given version on only one of the maps, that migration function will be used:mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '4.5.6': g }) -> { '1.2.3': f, '4.5.6': g }If there is a migration for a given version on both maps, the migrations will be composed:mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '1.2.3': g }) -> { '1.2.3': (doc, context) => f(g(doc, context), context) } | | [pollEsNodesVersion](./kibana-plugin-core-server.pollesnodesversion.md) | | | [ServiceStatusLevels](./kibana-plugin-core-server.servicestatuslevels.md) | The current "level" of availability of a service. | | [validBodyOutput](./kibana-plugin-core-server.validbodyoutput.md) | The set of valid body.output | diff --git a/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md b/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md index 52d40097ca487..68cd580b57882 100644 --- a/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md +++ b/docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md @@ -6,6 +6,14 @@ Merges two saved object migration maps. +If there is a migration for a given version on only one of the maps, that migration function will be used: + +mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '4.5.6': g }) -> { '1.2.3': f, '4.5.6': g } + +If there is a migration for a given version on both maps, the migrations will be composed: + +mergeSavedObjectMigrationMaps({ '1.2.3': f }, { '1.2.3': g }) -> { '1.2.3': (doc, context) => f(g(doc, context), context) } + Signature: ```typescript diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts index 331a3bbb9c028..fd8f219fb2d55 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.mock.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -140,6 +140,7 @@ const createStartContractMock = () => { alias: 'test_index', primaryStoreSizeBytes: 1, storeSizeBytes: 1, + savedObjectsDocsCount: 1, }, ], legacyUrlAliases: { diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 59f673bbe3dfa..e6e890b1a7dab 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -218,6 +218,11 @@ describe('CoreUsageDataService', () => { }, ], } as any); + elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({ + body: { + count: '15', + }, + } as any); elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({ body: [ { @@ -229,6 +234,11 @@ describe('CoreUsageDataService', () => { }, ], } as any); + elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({ + body: { + count: '10', + }, + } as any); elasticsearch.client.asInternalUser.search.mockResolvedValueOnce({ body: { hits: { total: { value: 6 } }, @@ -384,6 +394,7 @@ describe('CoreUsageDataService', () => { "docsCount": 10, "docsDeleted": 10, "primaryStoreSizeBytes": 2000, + "savedObjectsDocsCount": "15", "storeSizeBytes": 1000, }, Object { @@ -391,6 +402,7 @@ describe('CoreUsageDataService', () => { "docsCount": 20, "docsDeleted": 20, "primaryStoreSizeBytes": 4000, + "savedObjectsDocsCount": "10", "storeSizeBytes": 2000, }, ], diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index 0665aed0e1950..824bfdffdceec 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -131,11 +131,11 @@ export class CoreUsageDataService return acc.add(index); }, new Set()) .values() - ).map((index) => { + ).map(async (index) => { // The _cat/indices API returns the _index_ and doesn't return a way // to map back from the index to the alias. So we have to make an API - // call for every alias - return elasticsearch.client.asInternalUser.cat + // call for every alias. The document count is the lucene document count. + const catIndicesResults = await elasticsearch.client.asInternalUser.cat .indices({ index, format: 'JSON', @@ -143,6 +143,7 @@ export class CoreUsageDataService }) .then(({ body }) => { const stats = body[0]; + return { alias: kibanaOrTaskManagerIndex(index, kibanaIndex), docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0, @@ -153,6 +154,27 @@ export class CoreUsageDataService : 0, }; }); + // We use the GET /_count API to get the number of saved objects + // to monitor if the cluster will hit the scalling limit of saved object migrations + const savedObjectsCounts = await elasticsearch.client.asInternalUser + .count({ + index, + }) + .then(({ body }) => { + return { + savedObjectsDocsCount: body.count ? body.count : 0, + }; + }); + this.logger.debug( + `Lucene documents count ${catIndicesResults.docsCount} from index ${catIndicesResults.alias}` + ); + this.logger.debug( + `Saved objects documents count ${savedObjectsCounts.savedObjectsDocsCount} from index ${catIndicesResults.alias}` + ); + return { + ...catIndicesResults, + ...savedObjectsCounts, + }; }) ); } diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 59e220fac4efe..0b09444da493d 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -177,6 +177,7 @@ export interface CoreServicesUsageData { docsDeleted: number; storeSizeBytes: number; primaryStoreSizeBytes: number; + savedObjectsDocsCount: number; }[]; legacyUrlAliases: { activeCount: number; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 2f68b82ac5635..a722e6eb98b02 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -426,6 +426,7 @@ export interface CoreServicesUsageData { docsDeleted: number; storeSizeBytes: number; primaryStoreSizeBytes: number; + savedObjectsDocsCount: number; }[]; legacyUrlAliases: { activeCount: number; diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts index b1cf0ecd2213e..a208832baf719 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts @@ -355,14 +355,14 @@ export function getCoreUsageCollector( type: 'long', _meta: { description: - 'The number of documents in the index, including hidden nested documents.', + 'The number of lucene documents in the index, including hidden nested documents.', }, }, docsDeleted: { type: 'long', _meta: { description: - 'The number of deleted documents in the index, including hidden nested documents.', + 'The number of deleted lucene documents in the index, including hidden nested documents.', }, }, alias: { @@ -382,6 +382,12 @@ export function getCoreUsageCollector( description: 'The size in bytes of the index, for primaries and replicas.', }, }, + savedObjectsDocsCount: { + type: 'long', + _meta: { + description: 'The number of saved objects documents in the index.', + }, + }, }, }, legacyUrlAliases: { diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 3269452ca8cc3..dcbf919698243 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -6248,13 +6248,13 @@ "docsCount": { "type": "long", "_meta": { - "description": "The number of documents in the index, including hidden nested documents." + "description": "The number of lucene documents in the index, including hidden nested documents." } }, "docsDeleted": { "type": "long", "_meta": { - "description": "The number of deleted documents in the index, including hidden nested documents." + "description": "The number of deleted lucene documents in the index, including hidden nested documents." } }, "alias": { @@ -6274,6 +6274,12 @@ "_meta": { "description": "The size in bytes of the index, for primaries and replicas." } + }, + "savedObjectsDocsCount": { + "type": "long", + "_meta": { + "description": "The number of saved objects documents in the index." + } } } }