diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index 1ff74905b0b0f..189f28bf2c803 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -61,6 +61,7 @@ export { ALERTING_CASES_SAVED_OBJECT_INDEX, SECURITY_SOLUTION_SAVED_OBJECT_INDEX, ANALYTICS_SAVED_OBJECT_INDEX, + USAGE_COUNTERS_SAVED_OBJECT_INDEX, ALL_SAVED_OBJECT_INDICES, } from './src/saved_objects_index_pattern'; export type { diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts index 93160c6b9bd45..e84403e29ca12 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts @@ -19,6 +19,8 @@ export const INGEST_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_ingest`; export const ALERTING_CASES_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_alerting_cases`; export const SECURITY_SOLUTION_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_security_solution`; export const ANALYTICS_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_analytics`; +export const USAGE_COUNTERS_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_usage_counters`; + export const ALL_SAVED_OBJECT_INDICES = [ MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX, @@ -26,4 +28,5 @@ export const ALL_SAVED_OBJECT_INDICES = [ INGEST_SAVED_OBJECT_INDEX, SECURITY_SOLUTION_SAVED_OBJECT_INDEX, ANALYTICS_SAVED_OBJECT_INDEX, + USAGE_COUNTERS_SAVED_OBJECT_INDEX, ]; diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index e0f2ae0f4f4ba..de4d0b258db5e 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -1029,6 +1029,12 @@ "createDate", "slug" ], + "usage-counter": [ + "counterName", + "counterType", + "domainId", + "source" + ], "usage-counters": [ "domainId" ], diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 778ed3c37992c..6a447dcbc26bf 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3407,6 +3407,23 @@ } } }, + "usage-counter": { + "dynamic": false, + "properties": { + "counterName": { + "type": "keyword" + }, + "counterType": { + "type": "keyword" + }, + "domainId": { + "type": "keyword" + }, + "source": { + "type": "keyword" + } + } + }, "usage-counters": { "dynamic": false, "properties": { diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 732685c732ff7..af946130df77d 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -168,6 +168,7 @@ describe('checking migration metadata changes on all registered SO types', () => "uptime-dynamic-settings": "b6756ff71d6b5258971b1c8fd433d167affbde52", "uptime-synthetics-api-key": "7ae976a461248f9dbd8442af14a179bdbc229eca", "url": "c923a4a5002a09c0080c9095e958f07d518e6704", + "usage-counter": "3a104db0c9867da2d0436e20604a11dc5d0bb59e", "usage-counters": "48782b3bcb6b5a23ba6f2bfe3a380d835e68890a", "visualization": "93a3e73994ad836fe2b1dccbe208238f41f63da0", "workplace_search_telemetry": "52b32b47ee576f554ac77cb1d5896dfbcfe9a1fb", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts index 9a89ecb13d71b..e79084ebdc34a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_target_mappings.test.ts @@ -122,7 +122,7 @@ describe('migration v2 - CHECK_TARGET_MAPPINGS', () => { // Check for migration steps present in the logs logs = await fs.readFile(logFilePath, 'utf-8'); - expect(logs).not.toMatch('CREATE_NEW_TARGET'); + expect(logs).not.toMatch('[.kibana] CREATE_NEW_TARGET'); expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES'); expect(logs).toMatch( 'UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK' diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index a137b905f07a7..2228b19956bb3 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -146,7 +146,8 @@ const previouslyRegisteredTypes = [ 'synthetics-dynamic-settings', 'uptime-synthetics-api-key', 'url', - 'usage-counters', + 'usage-counter', // added in 8.16.0: richer mappings, located in .kibana_usage_counters + 'usage-counters', // deprecated in favor of 'usage-counter' 'visualization', 'workplace_search_telemetry', ].sort(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index c4c56442c23f8..ba164e223be55 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -173,135 +173,15 @@ describe('split .kibana index into multiple system indices', () => { }) ); - expect(indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap) - .toMatchInlineSnapshot(` - Object { - ".kibana": Array [ - "action", - "action_task_params", - "ad_hoc_run_params", - "alert", - "api_key_pending_invalidation", - "apm-custom-dashboards", - "apm-indices", - "apm-server-schema", - "apm-service-group", - "apm-telemetry", - "app_search_telemetry", - "application_usage_daily", - "application_usage_totals", - "canvas-element", - "canvas-workpad-template", - "cases", - "cases-comments", - "cases-configure", - "cases-connector-mappings", - "cases-rules", - "cases-telemetry", - "cases-user-actions", - "cloud-security-posture-settings", - "config", - "config-global", - "connector_token", - "core-usage-stats", - "csp-rule-template", - "endpoint:unified-user-artifact-manifest", - "endpoint:user-artifact-manifest", - "enterprise_search_telemetry", - "entity-definition", - "entity-discovery-api-key", - "epm-packages", - "epm-packages-assets", - "event-annotation-group", - "event_loop_delays_daily", - "exception-list", - "exception-list-agnostic", - "file", - "file-upload-usage-collection-telemetry", - "fileShare", - "fleet-fleet-server-host", - "fleet-message-signing-keys", - "fleet-preconfiguration-deletion-record", - "fleet-proxy", - "fleet-setup-lock", - "fleet-uninstall-tokens", - "graph-workspace", - "guided-onboarding-guide-state", - "guided-onboarding-plugin-state", - "index-pattern", - "infra-custom-dashboards", - "infrastructure-monitoring-log-view", - "infrastructure-ui-source", - "ingest-agent-policies", - "ingest-download-sources", - "ingest-outputs", - "ingest-package-policies", - "ingest_manager_settings", - "inventory-view", - "kql-telemetry", - "legacy-url-alias", - "lens", - "lens-ui-telemetry", - "links", - "maintenance-window", - "map", - "metrics-data-source", - "metrics-explorer-view", - "ml-job", - "ml-module", - "ml-trained-model", - "monitoring-telemetry", - "observability-onboarding-state", - "osquery-manager-usage-metric", - "osquery-pack", - "osquery-pack-asset", - "osquery-saved-query", - "policy-settings-protection-updates-note", - "query", - "risk-engine-configuration", - "rules-settings", - "sample-data-telemetry", - "search-session", - "search-telemetry", - "security-rule", - "security-solution-signals-migration", - "siem-detection-engine-rule-actions", - "siem-ui-timeline", - "siem-ui-timeline-note", - "siem-ui-timeline-pinned-event", - "slo", - "slo-settings", - "space", - "spaces-usage-stats", - "synthetics-dynamic-settings", - "synthetics-monitor", - "synthetics-param", - "synthetics-privates-locations", - "tag", - "telemetry", - "threshold-explorer-view", - "ui-metric", - "upgrade-assistant-ml-upgrade-operation", - "upgrade-assistant-reindex-operation", - "uptime-dynamic-settings", - "uptime-synthetics-api-key", - "url", - "usage-counters", - "workplace_search_telemetry", - ], - ".kibana_so_search": Array [ - "search", - ], - ".kibana_so_ui": Array [ - "canvas-workpad", - "dashboard", - "visualization", - ], - ".kibana_task_manager": Array [ - "task", - ], - } - `); + const typesMap = indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap; + + expect(Array.isArray(typesMap['.kibana'])).toEqual(true); + expect(typesMap['.kibana'].length > 50).toEqual(true); + expect(typesMap['.kibana'].includes('action')).toEqual(true); + expect(typesMap['.kibana'].includes('cases')).toEqual(true); + expect(typesMap['.kibana_so_search']).toEqual(['search']); + expect(typesMap['.kibana_so_ui']).toEqual(['canvas-workpad', 'dashboard', 'visualization']); + expect(typesMap['.kibana_task_manager']).toEqual(['task']); const logs = await parseLogFile(logFilePathFirstRun); @@ -506,6 +386,7 @@ describe('split .kibana index into multiple system indices', () => { '.kibana_task_manager': { task: 5, }, + '.kibana_usage_counters': {}, }); }); diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts index 035da58c60fa1..0a75e27da0830 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts @@ -9,7 +9,7 @@ import Hapi from '@hapi/hapi'; import h2o2 from '@hapi/h2o2'; import { URL } from 'url'; -import { SavedObject, ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import type { SavedObject } from '@kbn/core-saved-objects-server'; import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server'; import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal'; import { Root } from '@kbn/core-root-server-internal'; @@ -18,7 +18,6 @@ import { createTestServers, type TestElasticsearchUtils, } from '@kbn/core-test-helpers-kbn-server'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; import { declareGetRoute, declareDeleteRoute, @@ -32,7 +31,8 @@ import { declarePassthroughRoute, declareIndexRoute, setProxyInterrupt, - allCombinationsPermutations, + getVersionedKibanaIndex, + getIndicesWithNamespaceAwareTypes, } from './repository_with_proxy_utils'; let esServer: TestElasticsearchUtils; @@ -93,29 +93,40 @@ describe('404s from proxies', () => { ? parseInt(process.env.TEST_PROXY_SERVER_PORT, 10) : 5698; + // Setup kibana configured to use proxy as ES backend + root = createRootWithCorePlugins({ + elasticsearch: { + hosts: [`http://${esHostname}:${proxyPort}`], + }, + migrations: { + skip: false, + }, + }); + await root.preboot(); + const setup = await root.setup(); + registerSOTypes(setup); + // Setup custom hapi hapiServer with h2o2 plugin for proxying hapiServer = Hapi.server({ port: proxyPort, }); + const mainIndex = getVersionedKibanaIndex(); await hapiServer.register(h2o2); // register specific routes to modify the response and a catch-all to relay the request/response as-is + declareGetRoute(hapiServer, esHostname, esPort, mainIndex); + declareDeleteRoute(hapiServer, esHostname, esPort, mainIndex); + declarePostUpdateRoute(hapiServer, esHostname, esPort, mainIndex); - allCombinationsPermutations( - ALL_SAVED_OBJECT_INDICES.map((indexPattern) => `${indexPattern}_${pkg.version}`) - ) - .map((indices) => indices.join(',')) - .forEach((kbnIndexPath) => { - declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath); - - declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath); - declareIndexRoute(hapiServer, esHostname, esPort, kbnIndexPath); - }); + declareGetSearchRoute(hapiServer, esHostname, esPort, mainIndex); + declarePostSearchRoute(hapiServer, esHostname, esPort, mainIndex); + declarePostPitRoute(hapiServer, esHostname, esPort, mainIndex); + declareIndexRoute(hapiServer, esHostname, esPort, mainIndex); + + // the deleteByNamespace performs an updateByQuery under the hood. + // It targets all SO indices that have namespace-aware types + const nsIndices = getIndicesWithNamespaceAwareTypes(setup.savedObjects.getTypeRegistry()); + declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, nsIndices); // register index-agnostic routes declarePostBulkRoute(hapiServer, esHostname, esPort); @@ -124,19 +135,6 @@ describe('404s from proxies', () => { await hapiServer.start(); - // Setup kibana configured to use proxy as ES backend - root = createRootWithCorePlugins({ - elasticsearch: { - hosts: [`http://${esHostname}:${proxyPort}`], - }, - migrations: { - skip: false, - }, - }); - await root.preboot(); - const setup = await root.setup(); - registerSOTypes(setup); - start = await root.start(); }); diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts index 6f40d19f609d9..8419d4865c5c7 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts @@ -6,7 +6,13 @@ * Side Public License, v 1. */ import Hapi from '@hapi/hapi'; -import { IncomingMessage } from 'http'; +import type { IncomingMessage } from 'http'; +import { LEGACY_URL_ALIAS_TYPE } from '@kbn/core-saved-objects-base-server-internal'; +import { + type ISavedObjectTypeRegistry, + MAIN_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; +import { kibanaPackageJson as pkg } from '@kbn/repo-info'; // proxy setup const defaultProxyOptions = (hostname: string, port: string) => ({ @@ -304,21 +310,32 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin }, }); -export function allCombinationsPermutations(collection: T[]): T[][] { - const recur = (subcollection: T[], size: number): T[][] => { - if (size <= 0) { - return [[]]; - } - const permutations: T[][] = []; - subcollection.forEach((value, index, array) => { - array = array.slice(); - array.splice(index, 1); - recur(array, size - 1).forEach((permutation) => { - permutation.unshift(value); - permutations.push(permutation); - }); - }); - return permutations; - }; - return collection.map((_, n) => recur(collection, n + 1)).flat(); -} +/** + * Obtain the versioned Kibana index, tipically used by the Elasticsearch client + * e.g. .kibana_8.15.0 + * @returns string + */ +export const getVersionedKibanaIndex = (): string => { + return `${MAIN_SAVED_OBJECT_INDEX}_${pkg.version}`; +}; + +/** + * Obtain a comma separated list of all SO indices that contain namespace-aware SO types + * inspired on delete_by_namespace.ts + * + * @param registry The SO type registry to query registered types + * @returns string + */ +export const getIndicesWithNamespaceAwareTypes = (registry: ISavedObjectTypeRegistry): string => { + const allTypes = registry.getAllTypes(); + const unique = (array: string[]) => [...new Set(array)]; + + return unique( + [ + ...allTypes + .filter((type) => !registry.isNamespaceAgnostic(type.name)) + .map(({ name }) => name), + LEGACY_URL_ALIAS_TYPE, + ].map((type) => `${registry.getIndex(type) || MAIN_SAVED_OBJECT_INDEX}_${pkg.version}`) + ).join(','); +}; diff --git a/src/plugins/kibana_usage_collection/server/collectors/common/__fixtures__/counters_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/common/__fixtures__/counters_saved_objects.ts new file mode 100644 index 0000000000000..cf9e9c361117a --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/common/__fixtures__/counters_saved_objects.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UsageCountersSavedObject } from '@kbn/usage-collection-plugin/server'; + +export const rawServerCounters: UsageCountersSavedObject[] = [ + { + type: 'usage-counter', + id: 'myApp:my_event:count:server:20210409', + attributes: { + domainId: 'myApp', + counterName: 'my_event', + counterType: 'count', + source: 'server', + count: 1, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:17:57.693Z', + }, + { + type: 'usage-counter', + id: 'Kibana_home:intersecting_event:loaded:server:20201023', + attributes: { + domainId: 'Kibana_home', + counterName: 'intersecting_event', + counterType: 'loaded', + source: 'server', + count: 60, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2020-10-23T11:27:57.067Z', + }, + { + type: 'usage-counter', + id: 'myApp:my_event_4457914848544:count:server:20210409', + attributes: { + domainId: 'myApp', + counterName: 'my_event_4457914848544', + counterType: 'count', + source: 'server', + count: 0, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counter', + id: 'myApp:my_event_malformed:count:server:20210409', + attributes: { + domainId: 'myApp', + counterName: 'my_event_malformed', + counterType: 'count', + source: 'server', + // @ts-expect-error + count: 'malformed', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counter', + id: 'myApp:my_event_4457914848544_2:count:server:20210409', + attributes: { + domainId: 'myApp', + counterName: 'my_event_4457914848544_2', + counterType: 'count', + source: 'server', + count: 8, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', + }, + { + type: 'usage-counter', + id: 'myApp:only_reported_in_usage_counters:count:server:20210409', + attributes: { + domainId: 'myApp', + counterName: 'only_reported_in_usage_counters', + counterType: 'count', + source: 'server', + count: 1, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', + }, + { + type: 'usage-counter', + id: 'myApp:namespaced_counter:count:server:20240627:first', + namespaces: ['first'], + attributes: { + domainId: 'myApp', + counterName: 'namespaced_counter', + counterType: 'count', + source: 'server', + count: 1, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2024-06-27T08:18:03.031Z', + }, + { + type: 'usage-counter', + id: 'myApp:namespaced_counter:count:server:20240627:second', + namespaces: ['second'], + attributes: { + domainId: 'myApp', + counterName: 'namespaced_counter', + counterType: 'count', + source: 'server', + count: 2, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2024-06-27T09:18:03.031Z', + }, + { + type: 'usage-counter', + id: 'myApp:namespaced_counter:count:server:20240627:third', + namespaces: ['third'], + attributes: { + domainId: 'myApp', + counterName: 'namespaced_counter', + counterType: 'count', + source: 'server', + count: 3, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2024-06-27T10:18:03.031Z', + }, + { + type: 'usage-counter', + id: 'myApp:namespaced_counter:count:server:20240627:default', + namespaces: ['default'], + attributes: { + domainId: 'myApp', + counterName: 'namespaced_counter', + counterType: 'count', + source: 'server', + count: 10, + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2024-06-27T11:18:03.031Z', + }, +]; + +export const rawUiCounters: UsageCountersSavedObject[] = rawServerCounters.map((counter) => ({ + ...counter, + id: counter.id.replace(':server:', ':ui:'), + attributes: { + ...counter.attributes, + source: 'ui', + }, +})); diff --git a/src/plugins/kibana_usage_collection/server/collectors/common/counters.test.ts b/src/plugins/kibana_usage_collection/server/collectors/common/counters.test.ts new file mode 100644 index 0000000000000..dfc701cfa555a --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/common/counters.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggingSystemMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { transformRawCounter, createCounterFetcher } from './counters'; +import { rawServerCounters, rawUiCounters } from './__fixtures__/counters_saved_objects'; + +const mockLogger = loggingSystemMock.create(); + +describe('transformRawCounter', () => { + it('transforms usage counters savedObject raw entries', () => { + const result = rawServerCounters.map(transformRawCounter); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "counterName": "my_event", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:17:57.693Z", + "total": 1, + }, + Object { + "counterName": "intersecting_event", + "counterType": "loaded", + "domainId": "Kibana_home", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 60, + }, + undefined, + undefined, + Object { + "counterName": "my_event_4457914848544_2", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 8, + }, + Object { + "counterName": "only_reported_in_usage_counters", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + }, + Object { + "counterName": "namespaced_counter", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2024-06-27T00:00:00Z", + "lastUpdatedAt": "2024-06-27T08:18:03.031Z", + "total": 1, + }, + Object { + "counterName": "namespaced_counter", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2024-06-27T00:00:00Z", + "lastUpdatedAt": "2024-06-27T09:18:03.031Z", + "total": 2, + }, + Object { + "counterName": "namespaced_counter", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2024-06-27T00:00:00Z", + "lastUpdatedAt": "2024-06-27T10:18:03.031Z", + "total": 3, + }, + Object { + "counterName": "namespaced_counter", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2024-06-27T00:00:00Z", + "lastUpdatedAt": "2024-06-27T11:18:03.031Z", + "total": 10, + }, + ] + `); + }); +}); + +describe('createCounterFetcher', () => { + const soClientMock = savedObjectsClientMock.create(); + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns saved objects only from a given source', async () => { + // @ts-expect-error incomplete mock implementation + soClientMock.find.mockImplementation(async ({ type, filter }) => { + if (type !== 'usage-counter') { + throw new Error(`unexpected type ${type}`); + } + + if (filter === 'counter.attributes.source: server') { + return { saved_objects: rawServerCounters }; + } else if (filter === 'counter.attributes.source: ui') { + return { saved_objects: rawUiCounters }; + } + + throw new Error(`unexpected filter ${filter}`); + }); + + const fetch = createCounterFetcher( + mockLogger.get(), + 'counter.attributes.source: server', + (dailyEvents) => ({ + dailyEvents, + }) + ); + // @ts-expect-error incomplete mock implementation + const { dailyEvents } = await fetch({ soClient: soClientMock }); + expect(dailyEvents).toHaveLength(5); + const intersectingEntry = dailyEvents.find( + ({ counterName, fromTimestamp }) => + counterName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z' + ); + + const onlyFromUICountersEntry = dailyEvents.find( + ({ counterName }) => counterName === 'only_reported_in_ui_counters' + ); + + const onlyFromUsageCountersEntry = dailyEvents.find( + ({ counterName }) => counterName === 'only_reported_in_usage_counters' + ); + + const invalidCountEntry = dailyEvents.find( + ({ counterName }) => counterName === 'my_event_malformed' + ); + + const zeroCountEntry = dailyEvents.find( + ({ counterName }) => counterName === 'my_event_4457914848544' + ); + + const nonUiCountersEntry = dailyEvents.find( + ({ counterName }) => counterName === 'some_event_name' + ); + + const namespacedCounterEntry = dailyEvents.find( + ({ counterName }) => counterName === 'namespaced_counter' + ); + + expect(invalidCountEntry).toBe(undefined); + expect(nonUiCountersEntry).toBe(undefined); + expect(zeroCountEntry).toBe(undefined); + expect(onlyFromUICountersEntry).toBe(undefined); + expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(` + Object { + "counterName": "only_reported_in_usage_counters", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + } + `); + expect(intersectingEntry).toMatchInlineSnapshot(` + Object { + "counterName": "intersecting_event", + "counterType": "loaded", + "domainId": "Kibana_home", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 60, + } + `); + // we sum counters from all namespaces: 1 + 2 + 3 + 10 + expect(namespacedCounterEntry).toMatchInlineSnapshot(` + Object { + "counterName": "namespaced_counter", + "counterType": "count", + "domainId": "myApp", + "fromTimestamp": "2024-06-27T00:00:00Z", + "lastUpdatedAt": "2024-06-27T11:18:03.031Z", + "total": 16, + } + `); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/common/counters.ts b/src/plugins/kibana_usage_collection/server/collectors/common/counters.ts new file mode 100644 index 0000000000000..f20e57ab47664 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/common/counters.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { Logger } from '@kbn/logging'; +import { + CollectorFetchContext, + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '@kbn/usage-collection-plugin/server'; + +export interface CounterEvent { + domainId: string; + counterName: string; + counterType: string; + lastUpdatedAt: string; + fromTimestamp: string; + total: number; +} + +export function createCounterFetcher( + logger: Logger, + filter: string, + transform: (counters: CounterEvent[]) => T +) { + return async ({ soClient }: CollectorFetchContext) => { + const finder = soClient.createPointInTimeFinder({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + namespaces: ['*'], + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter, + perPage: 100, + }); + + const dailyEvents: CounterEvent[] = []; + + for await (const { saved_objects: rawUsageCounters } of finder.find()) { + rawUsageCounters.forEach((raw) => { + try { + const event = transformRawCounter(raw); + if (event) { + dailyEvents.push(event); + } + } catch (err) { + // swallow error; allows sending successfully transformed objects. + logger.debug('Error collecting usage-counter details: ' + err.message); + } + }); + } + + return transform(mergeCounters(dailyEvents)); + }; +} + +export function transformRawCounter( + rawCounter: UsageCountersSavedObject +): CounterEvent | undefined { + const { + attributes: { domainId, counterType, counterName, count }, + updated_at: lastUpdatedAt, + } = rawCounter; + + if (typeof count !== 'number' || count < 1) { + return; + } + + const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); + + return { + domainId, + counterType, + counterName, + lastUpdatedAt: lastUpdatedAt!, + fromTimestamp, + total: count, + }; +} + +function mergeCounters(counters: CounterEvent[]): CounterEvent[] { + const mergedCounters = counters.reduce((acc, counter) => { + const { domainId, counterType, counterName, fromTimestamp } = counter; + const key = `${domainId}:${counterType}:${counterName}:${fromTimestamp}`; + + const existingCounter = acc[key]; + if (!existingCounter) { + acc[key] = counter; + return acc; + } else { + acc[key].total = existingCounter.total + counter.total; + acc[key].lastUpdatedAt = counter.lastUpdatedAt; + } + return acc; + }, {} as Record); + + return Object.values(mergedCounters); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.test.ts b/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.test.ts new file mode 100644 index 0000000000000..173f332c9f391 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import type { SavedObjectsFindResult } from '@kbn/core/server'; +import { + type UsageCountersSavedObjectAttributes, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '@kbn/usage-collection-plugin/server'; + +import { isSavedObjectOlderThan } from './saved_objects'; + +export const createMockSavedObjectDoc = ( + updatedAt: moment.Moment, + id: string, + namespace?: string +) => + ({ + id, + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + ...(namespace && { namespaces: [namespace] }), + attributes: { + count: 3, + counterName: 'testName', + counterType: 'count', + domainId: 'testDomain', + source: 'server', + }, + references: [], + updated_at: updatedAt.format(), + version: 'WzI5LDFd', + score: 0, + } as SavedObjectsFindResult); + +describe('isSavedObjectOlderThan', () => { + it(`returns true if doc is older than x days`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(true); + }); + + it(`returns false if doc is exactly x days old`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); + + it(`returns false if doc is younger than x days`, () => { + const numberOfDays = 2; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.ts new file mode 100644 index 0000000000000..7b42b528b266b --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/common/saved_objects.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { SavedObject } from '@kbn/core-saved-objects-api-server'; + +export function isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, +}: { + numberOfDays: number; + startDate: moment.Moment | string | number; + doc: Pick; +}): boolean { + const { updated_at: updatedAt } = doc; + const today = moment(startDate).startOf('day'); + const updateDay = moment(updatedAt).startOf('day'); + + const diffInDays = today.diff(updateDay, 'days'); + if (diffInDays > numberOfDays) { + return true; + } + + return false; +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts deleted file mode 100644 index 4516e5174d402..0000000000000 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { UsageCountersSavedObject } from '@kbn/usage-collection-plugin/server'; - -export const rawUsageCounters: UsageCountersSavedObject[] = [ - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event', - attributes: { - count: 1, - counterName: 'myApp:my_event', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:17:57.693Z', - }, - { - type: 'usage-counters', - id: 'uiCounter:23102020:loaded:Kibana_home:intersecting_event', - attributes: { - count: 60, - counterName: 'Kibana_home:intersecting_event', - counterType: 'loaded', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2020-10-23T11:27:57.067Z', - }, - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event_4457914848544', - attributes: { - count: 0, - counterName: 'myApp:my_event_4457914848544', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event_malformed', - attributes: { - // @ts-expect-error - count: 'malformed', - counterName: 'myApp:my_event_malformed', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { - count: 4, - counterName: 'some_event_name', - counterType: 'count', - domainId: 'anotherDomainId', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event_4457914848544_2', - attributes: { - count: 8, - counterName: 'myApp:my_event_4457914848544_2', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.031Z', - }, - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:only_reported_in_usage_counters', - attributes: { - count: 1, - counterName: 'myApp:only_reported_in_usage_counters', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.031Z', - }, -]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts index 0e84df3325d3d..6d35f5baf4046 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts @@ -6,124 +6,49 @@ * Side Public License, v 1. */ -import { transformRawUsageCounterObject, fetchUiCounters } from './register_ui_counters_collector'; -import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '@kbn/usage-collection-plugin/server'; - -describe('transformRawUsageCounterObject', () => { - it('transforms usage counters savedObject raw entries', () => { - const result = rawUsageCounters.map(transformRawUsageCounterObject); - expect(result).toMatchInlineSnapshot(` - Array [ - Object { - "appName": "myApp", - "counterType": "count", - "eventName": "my_event", - "fromTimestamp": "2021-04-09T00:00:00Z", - "lastUpdatedAt": "2021-04-09T08:17:57.693Z", - "total": 1, - }, - Object { - "appName": "Kibana_home", - "counterType": "loaded", - "eventName": "intersecting_event", - "fromTimestamp": "2020-10-23T00:00:00Z", - "lastUpdatedAt": "2020-10-23T11:27:57.067Z", - "total": 60, +import type { CounterEvent } from '../common/counters'; +import { toDailyEvents } from './register_ui_counters_collector'; + +describe('toDailyEvents', () => { + it('adapts counter events to have the expected UI properties', () => { + const counters: CounterEvent[] = [ + { + domainId: 'dashboards', + counterType: 'loaded', + counterName: 'lens', + lastUpdatedAt: '2024-06-19T12:03:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 18, + }, + { + domainId: 'dashboards', + counterType: 'updated', + counterName: 'lens', + lastUpdatedAt: '2024-06-19T14:13:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 4, + }, + ]; + + expect(toDailyEvents(counters)).toEqual({ + dailyEvents: [ + { + appName: 'dashboards', + counterType: 'loaded', + eventName: 'lens', + lastUpdatedAt: '2024-06-19T12:03:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 18, }, - undefined, - undefined, - undefined, - Object { - "appName": "myApp", - "counterType": "count", - "eventName": "my_event_4457914848544_2", - "fromTimestamp": "2021-04-09T00:00:00Z", - "lastUpdatedAt": "2021-04-09T08:18:03.031Z", - "total": 8, + { + appName: 'dashboards', + counterType: 'updated', + eventName: 'lens', + lastUpdatedAt: '2024-06-19T14:13:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 4, }, - Object { - "appName": "myApp", - "counterType": "count", - "eventName": "only_reported_in_usage_counters", - "fromTimestamp": "2021-04-09T00:00:00Z", - "lastUpdatedAt": "2021-04-09T08:18:03.031Z", - "total": 1, - }, - ] - `); - }); -}); - -describe('fetchUiCounters', () => { - const soClientMock = savedObjectsClientMock.create(); - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('returns saved objects only from usage_counters saved objects', async () => { - // @ts-expect-error incomplete mock implementation - soClientMock.find.mockImplementation(async ({ type }) => { - switch (type) { - case USAGE_COUNTERS_SAVED_OBJECT_TYPE: - return { saved_objects: rawUsageCounters }; - default: - throw new Error(`unexpected type ${type}`); - } + ], }); - - // @ts-expect-error incomplete mock implementation - const { dailyEvents } = await fetchUiCounters({ - soClient: soClientMock, - }); - expect(dailyEvents).toHaveLength(4); - const intersectingEntry = dailyEvents.find( - ({ eventName, fromTimestamp }) => - eventName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z' - ); - - const onlyFromUICountersEntry = dailyEvents.find( - ({ eventName }) => eventName === 'only_reported_in_ui_counters' - ); - - const onlyFromUsageCountersEntry = dailyEvents.find( - ({ eventName }) => eventName === 'only_reported_in_usage_counters' - ); - - const invalidCountEntry = dailyEvents.find( - ({ eventName }) => eventName === 'my_event_malformed' - ); - - const zeroCountEntry = dailyEvents.find( - ({ eventName }) => eventName === 'my_event_4457914848544' - ); - - const nonUiCountersEntry = dailyEvents.find(({ eventName }) => eventName === 'some_event_name'); - - expect(invalidCountEntry).toBe(undefined); - expect(nonUiCountersEntry).toBe(undefined); - expect(zeroCountEntry).toBe(undefined); - expect(onlyFromUICountersEntry).toBe(undefined); - expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(` - Object { - "appName": "myApp", - "counterType": "count", - "eventName": "only_reported_in_usage_counters", - "fromTimestamp": "2021-04-09T00:00:00Z", - "lastUpdatedAt": "2021-04-09T08:18:03.031Z", - "total": 1, - } - `); - expect(intersectingEntry).toMatchInlineSnapshot(` - Object { - "appName": "Kibana_home", - "counterType": "loaded", - "eventName": "intersecting_event", - "fromTimestamp": "2020-10-23T00:00:00Z", - "lastUpdatedAt": "2020-10-23T11:27:57.067Z", - "total": 60, - } - `); }); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts index 77212225cc14c..024d3b7a9f562 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts @@ -6,24 +6,20 @@ * Side Public License, v 1. */ -import moment from 'moment'; - +import type { Logger } from '@kbn/logging'; +import { UsageCounters } from '@kbn/usage-collection-plugin/common'; import { - CollectorFetchContext, - UsageCollectionSetup, + type UsageCollectionSetup, USAGE_COUNTERS_SAVED_OBJECT_TYPE, - UsageCountersSavedObject, - UsageCountersSavedObjectAttributes, } from '@kbn/usage-collection-plugin/server'; - -import { deserializeUiCounterName } from '@kbn/usage-collection-plugin/common/ui_counters'; +import { type CounterEvent, createCounterFetcher } from '../common/counters'; interface UiCounterEvent { appName: string; eventName: string; - lastUpdatedAt?: string; - fromTimestamp?: string; counterType: string; + lastUpdatedAt: string; + fromTimestamp: string; total: number; } @@ -31,58 +27,13 @@ export interface UiUsageCounters { dailyEvents: UiCounterEvent[]; } -export function transformRawUsageCounterObject( - rawUsageCounter: UsageCountersSavedObject -): UiCounterEvent | undefined { - const { - attributes: { count, counterName, counterType, domainId }, - updated_at: lastUpdatedAt, - } = rawUsageCounter; - - if (domainId !== 'uiCounter' || typeof count !== 'number' || count < 1) { - return; - } - - const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); - const { appName, eventName } = deserializeUiCounterName(counterName); - - return { - appName, - eventName, - lastUpdatedAt, - fromTimestamp, - counterType, - total: count, - }; -} - -export async function fetchUiCounters({ soClient }: CollectorFetchContext) { - const finder = soClient.createPointInTimeFinder({ - type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, - fields: ['count', 'counterName', 'counterType', 'domainId'], - filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, - perPage: 1000, - }); - - const dailyEvents: UiCounterEvent[] = []; - - for await (const { saved_objects: rawUsageCounters } of finder.find()) { - rawUsageCounters.forEach((raw) => { - try { - const event = transformRawUsageCounterObject(raw); - if (event) { - dailyEvents.push(event); - } - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - }); - } +const UI: UsageCounters.v1.CounterEventSource = 'ui'; +const UI_COUNTERS_FILTER = `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.source: ${UI}`; - return { dailyEvents }; -} - -export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) { +export function registerUiCountersUsageCollector( + usageCollection: UsageCollectionSetup, + logger: Logger +) { const collector = usageCollection.makeUsageCollector({ type: 'ui_counters', schema: { @@ -105,7 +56,10 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio type: 'date', _meta: { description: 'Time at which the metric was captured.' }, }, - counterType: { type: 'keyword', _meta: { description: 'The type of counter used.' } }, + counterType: { + type: 'keyword', + _meta: { description: 'The type of counter used.' }, + }, total: { type: 'integer', _meta: { description: 'The total number of times the event happened.' }, @@ -113,9 +67,24 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio }, }, }, - fetch: fetchUiCounters, + fetch: createCounterFetcher(logger, UI_COUNTERS_FILTER, toDailyEvents), isReady: () => true, }); usageCollection.registerCollector(collector); } + +export function toDailyEvents(counters: CounterEvent[]) { + return { + dailyEvents: counters.map(toUiCounter), + }; +} + +function toUiCounter(counter: CounterEvent): UiCounterEvent { + const { domainId: appName, counterName: eventName, ...props } = counter; + return { + appName, + eventName, + ...props, + }; +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts deleted file mode 100644 index 344d36542646d..0000000000000 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { UsageCountersSavedObject } from '@kbn/usage-collection-plugin/server'; - -export const rawUsageCounters: UsageCountersSavedObject[] = [ - { - type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event', - attributes: { - count: 13, - counterName: 'my_event', - counterType: 'count', - domainId: 'uiCounter', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { - count: 4, - counterName: 'some_event_name', - counterType: 'count', - domainId: 'anotherDomainId', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-09T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { - count: 4, - counterName: 'some_event_name', - counterType: 'count', - domainId: 'anotherDomainId', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-11T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId2:09042021:count:some_event_name', - attributes: { - count: 1, - counterName: 'some_event_name', - counterType: 'count', - domainId: 'anotherDomainId2', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-20T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId2:09042021:count:malformed_event', - attributes: { - // @ts-expect-error - count: 'malformed', - counterName: 'malformed_event', - counterType: 'count', - domainId: 'anotherDomainId2', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-20T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId2:09042021:custom_type:some_event_name', - attributes: { - count: 3, - counterName: 'some_event_name', - counterType: 'custom_type', - domainId: 'anotherDomainId2', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-20T08:18:03.030Z', - }, - { - type: 'usage-counters', - id: 'anotherDomainId3:09042021:custom_type:zero_count', - attributes: { - count: 0, - counterName: 'zero_count', - counterType: 'custom_type', - domainId: 'anotherDomainId3', - }, - references: [], - coreMigrationVersion: '8.0.0', - updated_at: '2021-04-20T08:18:03.030Z', - }, -]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/integration_tests/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/integration_tests/rollups.test.ts new file mode 100644 index 0000000000000..23e0867f4942e --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/integration_tests/rollups.test.ts @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment, { type MomentInput } from 'moment'; +/* + * Mocking the constructor of moment, so we can control the time of the day. + * This is to avoid flaky tests when starting to run before midnight and ending the test after midnight + * because the logic might remove one extra document since we moved to the next day. + */ +jest.doMock('moment', () => { + const mockedMoment = (date?: MomentInput) => moment(date ?? '2024-06-30T10:00:00.000Z'); + Object.setPrototypeOf(mockedMoment, moment); // inherit the prototype of `moment` so it has all the same methods. + return mockedMoment; +}); + +jest.mock('@kbn/core-saved-objects-api-server-internal/src/lib/apis/utils', () => ({ + ...jest.requireActual('@kbn/core-saved-objects-api-server-internal/src/lib/apis/utils'), + getCurrentTime: jest.fn(), +})); + +jest.mock('../../common/saved_objects', () => ({ + ...jest.requireActual('../../common/saved_objects'), + isSavedObjectOlderThan: jest.fn(), +})); + +import { getCurrentTime } from '@kbn/core-saved-objects-api-server-internal/src/lib/apis/utils'; +import type { Logger, ISavedObjectsRepository, SavedObject } from '@kbn/core/server'; +import { + type TestElasticsearchUtils, + type TestKibanaUtils, + createTestServers, + createRootWithCorePlugins, +} from '@kbn/core-test-helpers-kbn-server'; + +import { + serializeCounterKey, + type UsageCountersSavedObjectAttributes, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '@kbn/usage-collection-plugin/server'; +import { rollUsageCountersIndices } from '../rollups/rollups'; +import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from '../rollups/constants'; +import { isSavedObjectOlderThan } from '../../common/saved_objects'; + +const getCurrentTimeMock = getCurrentTime as jest.MockedFunction; +const isSavedObjectOlderThanMock = isSavedObjectOlderThan as jest.MockedFunction< + typeof isSavedObjectOlderThan +>; + +const ALL_COUNTERS = [ + 'domain1:a:count:server:20240624:default', + 'domain1:a:count:server:20240626:default', + 'domain1:b:count:server:20240624:one', + 'domain1:b:count:server:20240624:two', + 'domain1:b:count:server:20240626:one', + 'domain1:b:count:server:20240626:two', + 'domain2:a:count:server:20240624:default', + 'domain2:a:count:server:20240626:default', + 'domain2:c:count:server:20240626:default', +]; + +const RECENT_COUNTERS = ALL_COUNTERS.filter((key) => key.includes('20240626')); + +describe('usage-counters', () => { + let esServer: TestElasticsearchUtils; + let root: TestKibanaUtils['root']; + let internalRepository: ISavedObjectsRepository; + let logger: Logger; + + beforeAll(async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + }); + + esServer = await startES(); + root = createRootWithCorePlugins(); + + await root.preboot(); + await root.setup(); + const start = await root.start(); + + logger = root.logger.get('test daily rollups'); + internalRepository = start.savedObjects.createInternalRepository([ + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + ]); + }); + + it('deletes documents older that the retention period, from all namespaces', async () => { + // insert a bunch of usage counters in multiple namespaces + const old = [ + createCounter('domain1', 'a', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS + 1), + createCounter('domain1', 'b', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS + 1, 'one'), + createCounter('domain1', 'b', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS + 1, 'two'), + createCounter('domain2', 'a', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS + 1), + ]; + + const recent = [ + createCounter('domain1', 'a', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS - 1), + createCounter('domain1', 'b', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS - 1, 'one'), + createCounter('domain1', 'b', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS - 1, 'two'), + createCounter('domain2', 'a', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS - 1), + createCounter('domain2', 'c', USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS - 1), // different counterName (does not have an old counterpart) + ]; + + getCurrentTimeMock.mockReturnValue('2024-06-24T10:00:00.000Z'); // 6 days old + await Promise.all(old.map((counter) => incrementCounter(internalRepository, counter))); + + getCurrentTimeMock.mockReturnValue('2024-06-26T10:00:00.000Z'); // 4 days old + await Promise.all(recent.map((counter) => incrementCounter(internalRepository, counter))); + + // check that all documents are there + const beforeRollup = await internalRepository.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + namespaces: ['*'], + }); + expect( + beforeRollup.saved_objects + .map(({ attributes, updated_at: updatedAt, namespaces }) => + serializeCounterKey({ ...attributes, date: updatedAt, namespace: namespaces?.[0] }) + ) + .sort() + ).toEqual(ALL_COUNTERS); + + // run the rollup logic + isSavedObjectOlderThanMock.mockImplementation( + ({ doc }) => doc.updated_at === '2024-06-24T10:00:00.000Z' + ); + await rollUsageCountersIndices(logger, internalRepository); + + // check only recent counters are present + const afterRollup = await internalRepository.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + namespaces: ['*'], + }); + expect( + afterRollup.saved_objects + .map(({ attributes, updated_at: updatedAt, namespaces }) => + serializeCounterKey({ ...attributes, date: updatedAt, namespace: namespaces?.[0] }) + ) + .sort() + ).toEqual(RECENT_COUNTERS); + }); + + afterAll(async () => { + await esServer.stop(); + await root.shutdown(); + }); +}); + +function createCounter( + domainId: string, + counterName: string, + ageDays: number = 0, + namespace?: string +): SavedObject { + const date = moment('2024-06-30T10:00:00.000Z').subtract(ageDays, 'days'); + + const id = serializeCounterKey({ + domainId, + counterName, + counterType: 'count', + namespace, + source: 'server', + date, + }); + return { + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + id, + ...(namespace && { namespaces: [namespace] }), + updated_at: date.format(), // illustrative purpose only, overriden by SOR + attributes: { + domainId, + counterName, + counterType: 'count', + source: 'server', + count: 28, + }, + references: [], + }; +} + +async function incrementCounter( + internalRepository: ISavedObjectsRepository, + counter: SavedObject +) { + const namespace = counter.namespaces?.[0]; + return await internalRepository.incrementCounter( + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + counter.id, + [{ fieldName: 'count', incrementBy: counter.attributes.count }], + { + ...(namespace && { namespace }), + upsertAttributes: { + domainId: counter.attributes.domainId, + counterName: counter.attributes.counterName, + counterType: counter.attributes.counterType, + source: counter.attributes.source, + }, + } + ); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts index 945eb007fe23f..2412fbe2a9279 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts @@ -6,50 +6,49 @@ * Side Public License, v 1. */ -import { transformRawCounter } from './register_usage_counters_collector'; -import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; +import type { CounterEvent } from '../common/counters'; +import { toDailyEvents } from './register_usage_counters_collector'; -describe('transformRawCounter', () => { - it('transforms saved object raw entries', () => { - const result = rawUsageCounters.map(transformRawCounter); - expect(result).toMatchInlineSnapshot(` - Array [ - undefined, - Object { - "counterName": "some_event_name", - "counterType": "count", - "domainId": "anotherDomainId", - "fromTimestamp": "2021-04-09T00:00:00Z", - "lastUpdatedAt": "2021-04-09T08:18:03.030Z", - "total": 4, - }, - Object { - "counterName": "some_event_name", - "counterType": "count", - "domainId": "anotherDomainId", - "fromTimestamp": "2021-04-11T00:00:00Z", - "lastUpdatedAt": "2021-04-11T08:18:03.030Z", - "total": 4, - }, - Object { - "counterName": "some_event_name", - "counterType": "count", - "domainId": "anotherDomainId2", - "fromTimestamp": "2021-04-20T00:00:00Z", - "lastUpdatedAt": "2021-04-20T08:18:03.030Z", - "total": 1, +describe('toDailyEvents', () => { + it('adapts counter events to have the expected UI properties', () => { + const counters: CounterEvent[] = [ + { + domainId: 'foo', + counterType: 'bar', + counterName: 'count', + lastUpdatedAt: '2024-06-19T12:03:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 18, + }, + { + domainId: 'foo', + counterType: 'baz', + counterName: 'count', + lastUpdatedAt: '2024-06-19T14:13:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 4, + }, + ]; + + expect(toDailyEvents(counters)).toEqual({ + dailyEvents: [ + { + domainId: 'foo', + counterType: 'bar', + counterName: 'count', + lastUpdatedAt: '2024-06-19T12:03:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 18, }, - undefined, - Object { - "counterName": "some_event_name", - "counterType": "custom_type", - "domainId": "anotherDomainId2", - "fromTimestamp": "2021-04-20T00:00:00Z", - "lastUpdatedAt": "2021-04-20T08:18:03.030Z", - "total": 3, + { + domainId: 'foo', + counterType: 'baz', + counterName: 'count', + lastUpdatedAt: '2024-06-19T14:13:25.795Z', + fromTimestamp: '2024-06-19T00:00:00Z', + total: 4, }, - undefined, - ] - `); + ], + }); }); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts index 807f0189a4dd0..bd06615e8eb30 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts @@ -6,52 +6,25 @@ * Side Public License, v 1. */ -import moment from 'moment'; +import type { Logger } from '@kbn/logging'; +import { UsageCounters } from '@kbn/usage-collection-plugin/common'; import { - CollectorFetchContext, - UsageCollectionSetup, + type UsageCollectionSetup, USAGE_COUNTERS_SAVED_OBJECT_TYPE, - UsageCountersSavedObject, - UsageCountersSavedObjectAttributes, } from '@kbn/usage-collection-plugin/server'; - -interface UsageCounterEvent { - domainId: string; - counterName: string; - counterType: string; - lastUpdatedAt?: string; - fromTimestamp?: string; - total: number; -} +import { type CounterEvent, createCounterFetcher } from '../common/counters'; export interface UsageCounters { - dailyEvents: UsageCounterEvent[]; + dailyEvents: CounterEvent[]; } -export function transformRawCounter( - rawUsageCounter: UsageCountersSavedObject -): UsageCounterEvent | undefined { - const { - attributes: { count, counterName, counterType, domainId }, - updated_at: lastUpdatedAt, - } = rawUsageCounter; - const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); - - if (domainId === 'uiCounter' || typeof count !== 'number' || count < 1) { - return; - } - - return { - domainId, - counterName, - counterType, - lastUpdatedAt, - fromTimestamp, - total: count, - }; -} +const SERVER: UsageCounters.v1.CounterEventSource = 'server'; +const SERVER_COUNTERS_FILTER = `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.source: ${SERVER}`; -export function registerUsageCountersUsageCollector(usageCollection: UsageCollectionSetup) { +export function registerUsageCountersUsageCollector( + usageCollection: UsageCollectionSetup, + logger: Logger +) { const collector = usageCollection.makeUsageCollector({ type: 'usage_counters', schema: { @@ -85,33 +58,15 @@ export function registerUsageCountersUsageCollector(usageCollection: UsageCollec }, }, }, - fetch: async ({ soClient }: CollectorFetchContext) => { - const finder = soClient.createPointInTimeFinder({ - type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, - fields: ['count', 'counterName', 'counterType', 'domainId'], - filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, - perPage: 1000, - }); - - const dailyEvents: UsageCounterEvent[] = []; - - for await (const { saved_objects: rawUsageCounters } of finder.find()) { - rawUsageCounters.forEach((rawUsageCounter) => { - try { - const event = transformRawCounter(rawUsageCounter); - if (event) { - dailyEvents.push(event); - } - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - }); - } - - return { dailyEvents }; - }, + fetch: createCounterFetcher(logger, SERVER_COUNTERS_FILTER, toDailyEvents), isReady: () => true, }); usageCollection.registerCollector(collector); } + +export function toDailyEvents(counters: CounterEvent[]) { + return { + dailyEvents: counters, + }; +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts index 4397468244a1f..37c68021c9c73 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts @@ -7,70 +7,11 @@ */ import moment from 'moment'; -import { isSavedObjectOlderThan, rollUsageCountersIndices } from './rollups'; import { savedObjectsRepositoryMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { SavedObjectsFindResult } from '@kbn/core/server'; - -import { - UsageCountersSavedObjectAttributes, - USAGE_COUNTERS_SAVED_OBJECT_TYPE, -} from '@kbn/usage-collection-plugin/server'; - +import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '@kbn/usage-collection-plugin/server'; import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; - -const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) => - ({ - id, - type: 'usage-counter', - attributes: { - count: 3, - counterName: 'testName', - counterType: 'count', - domainId: 'testDomain', - }, - references: [], - updated_at: updatedAt.format(), - version: 'WzI5LDFd', - score: 0, - } as SavedObjectsFindResult); - -describe('isSavedObjectOlderThan', () => { - it(`returns true if doc is older than x days`, () => { - const numberOfDays = 1; - const startDate = moment().format(); - const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id'); - const result = isSavedObjectOlderThan({ - numberOfDays, - startDate, - doc, - }); - expect(result).toBe(true); - }); - - it(`returns false if doc is exactly x days old`, () => { - const numberOfDays = 1; - const startDate = moment().format(); - const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); - const result = isSavedObjectOlderThan({ - numberOfDays, - startDate, - doc, - }); - expect(result).toBe(false); - }); - - it(`returns false if doc is younger than x days`, () => { - const numberOfDays = 2; - const startDate = moment().format(); - const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); - const result = isSavedObjectOlderThan({ - numberOfDays, - startDate, - doc, - }); - expect(result).toBe(false); - }); -}); +import { createMockSavedObjectDoc } from '../../common/saved_objects.test'; +import { rollUsageCountersIndices } from './rollups'; describe('rollUsageCountersIndices', () => { let logger: ReturnType; @@ -106,7 +47,7 @@ describe('rollUsageCountersIndices', () => { createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'), createMockSavedObjectDoc(moment().subtract(9, 'days'), 'doc-id-1'), createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'), - createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'), + createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3', 'secondary'), ]; savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { @@ -128,7 +69,8 @@ describe('rollUsageCountersIndices', () => { expect(savedObjectClient.delete).toHaveBeenNthCalledWith( 2, USAGE_COUNTERS_SAVED_OBJECT_TYPE, - 'doc-id-3' + 'doc-id-3', + { namespace: 'secondary' } ); expect(logger.warn).toHaveBeenCalledTimes(0); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts index 621b40fb21e9c..a8cdfd92372d7 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts @@ -6,35 +6,15 @@ * Side Public License, v 1. */ -import type { ISavedObjectsRepository, Logger } from '@kbn/core/server'; import moment from 'moment'; +import type { ISavedObjectsRepository, Logger } from '@kbn/core/server'; import { - UsageCountersSavedObject, + type UsageCountersSavedObject, USAGE_COUNTERS_SAVED_OBJECT_TYPE, } from '@kbn/usage-collection-plugin/server'; import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; - -export function isSavedObjectOlderThan({ - numberOfDays, - startDate, - doc, -}: { - numberOfDays: number; - startDate: moment.Moment | string | number; - doc: Pick; -}): boolean { - const { updated_at: updatedAt } = doc; - const today = moment(startDate).startOf('day'); - const updateDay = moment(updatedAt).startOf('day'); - - const diffInDays = today.diff(updateDay, 'days'); - if (diffInDays > numberOfDays) { - return true; - } - - return false; -} +import { isSavedObjectOlderThan } from '../../common/saved_objects'; export async function rollUsageCountersIndices( logger: Logger, @@ -50,6 +30,7 @@ export async function rollUsageCountersIndices( const { saved_objects: rawUiCounterDocs } = await savedObjectsClient.find({ type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + namespaces: ['*'], perPage: 1000, // Process 1000 at a time as a compromise of speed and overload }); @@ -62,7 +43,11 @@ export async function rollUsageCountersIndices( ); return await Promise.all( - docsToDelete.map(({ id }) => savedObjectsClient.delete(USAGE_COUNTERS_SAVED_OBJECT_TYPE, id)) + docsToDelete.map(({ id, type, namespaces }) => + namespaces?.[0] + ? savedObjectsClient.delete(type, id, { namespace: namespaces[0] }) + : savedObjectsClient.delete(type, id) + ) ); } catch (err) { logger.warn(`Failed to rollup Usage Counters saved objects.`); diff --git a/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.test.ts b/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.test.ts index a2ac2366c0b3e..018e179042254 100644 --- a/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.test.ts +++ b/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.test.ts @@ -42,15 +42,17 @@ describe('registerEbtCounters', () => { code: 'test-code', count: 1, }); - expect(usageCollection.getUsageCounterByType).toHaveBeenCalledTimes(1); - expect(usageCollection.getUsageCounterByType).toHaveBeenCalledWith('ebt_counters.test-shipper'); + expect(usageCollection.getUsageCounterByDomainId).toHaveBeenCalledTimes(1); + expect(usageCollection.getUsageCounterByDomainId).toHaveBeenCalledWith( + 'ebt_counters.test-shipper' + ); expect(usageCollection.createUsageCounter).toHaveBeenCalledTimes(1); expect(usageCollection.createUsageCounter).toHaveBeenCalledWith('ebt_counters.test-shipper'); }); test('it reuses the usageCounter when it already exists', () => { const incrementCounterMock = jest.fn(); - usageCollection.getUsageCounterByType.mockReturnValue({ + usageCollection.getUsageCounterByDomainId.mockReturnValue({ incrementCounter: incrementCounterMock, }); registerEbtCounters(core.analytics, usageCollection); @@ -62,8 +64,10 @@ describe('registerEbtCounters', () => { code: 'test-code', count: 1, }); - expect(usageCollection.getUsageCounterByType).toHaveBeenCalledTimes(1); - expect(usageCollection.getUsageCounterByType).toHaveBeenCalledWith('ebt_counters.test-shipper'); + expect(usageCollection.getUsageCounterByDomainId).toHaveBeenCalledTimes(1); + expect(usageCollection.getUsageCounterByDomainId).toHaveBeenCalledWith( + 'ebt_counters.test-shipper' + ); expect(usageCollection.createUsageCounter).toHaveBeenCalledTimes(0); expect(incrementCounterMock).toHaveBeenCalledTimes(1); expect(incrementCounterMock).toHaveBeenCalledWith({ diff --git a/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.ts b/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.ts index ed2100dccf929..b075b198ce496 100644 --- a/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.ts +++ b/src/plugins/kibana_usage_collection/server/ebt_counters/register_ebt_counters.ts @@ -18,7 +18,7 @@ export function registerEbtCounters( // We create one counter per source ('client'|). const domainId = `ebt_counters.${source}`; const usageCounter = - usageCollection.getUsageCounterByType(domainId) ?? + usageCollection.getUsageCounterByDomainId(domainId) ?? usageCollection.createUsageCounter(domainId); usageCounter.incrementCounter({ diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index ff38116c1de44..e2b88b8881f66 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -72,7 +72,6 @@ export class KibanaUsageCollectionPlugin implements Plugin { public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { registerEbtCounters(coreSetup.analytics, usageCollection); - usageCollection.createUsageCounter('uiCounters'); this.eventLoopUsageCounter = usageCollection.createUsageCounter('eventLoop'); coreSetup.coreUsageData.registerUsageCounter(usageCollection.createUsageCounter('core')); this.registerUsageCollectors( @@ -127,14 +126,14 @@ export class KibanaUsageCollectionPlugin implements Plugin { const getUiSettingsClient = () => this.uiSettingsClient; const getCoreUsageDataService = () => this.coreUsageData!; - registerUiCountersUsageCollector(usageCollection); + registerUiCountersUsageCollector(usageCollection, this.logger); registerUsageCountersRollups( this.logger.get('usage-counters-rollup'), getSavedObjectsClient, pluginStop$ ); - registerUsageCountersUsageCollector(usageCollection); + registerUsageCountersUsageCollector(usageCollection, this.logger); registerOpsStatsCollector(usageCollection, metric$); diff --git a/src/plugins/kibana_usage_collection/tsconfig.json b/src/plugins/kibana_usage_collection/tsconfig.json index 2fb915d541052..84e9a38f3e970 100644 --- a/src/plugins/kibana_usage_collection/tsconfig.json +++ b/src/plugins/kibana_usage_collection/tsconfig.json @@ -18,6 +18,7 @@ "@kbn/core-test-helpers-kbn-server", "@kbn/core-usage-data-server", "@kbn/core-saved-objects-api-server", + "@kbn/core-saved-objects-api-server-internal", ], "exclude": [ "target/**/*", diff --git a/src/plugins/usage_collection/common/ui_counters.ts b/src/plugins/usage_collection/common/index.ts similarity index 51% rename from src/plugins/usage_collection/common/ui_counters.ts rename to src/plugins/usage_collection/common/index.ts index 3ed6e44aee419..3f0a94c72671f 100644 --- a/src/plugins/usage_collection/common/ui_counters.ts +++ b/src/plugins/usage_collection/common/index.ts @@ -6,18 +6,4 @@ * Side Public License, v 1. */ -export const serializeUiCounterName = ({ - appName, - eventName, -}: { - appName: string; - eventName: string; -}) => { - return `${appName}:${eventName}`; -}; - -export const deserializeUiCounterName = (key: string) => { - const [appName, ...restKey] = key.split(':'); - const eventName = restKey.join(':'); - return { appName, eventName }; -}; +export type { UsageCounters } from './types'; diff --git a/src/plugins/usage_collection/common/types/usage_counters/v1.ts b/src/plugins/usage_collection/common/types/usage_counters/v1.ts index fa53a9d9e93b0..c18af503d06e0 100644 --- a/src/plugins/usage_collection/common/types/usage_counters/v1.ts +++ b/src/plugins/usage_collection/common/types/usage_counters/v1.ts @@ -6,10 +6,14 @@ * Side Public License, v 1. */ +export type CounterEventSource = 'server' | 'ui'; + export interface CounterMetric { domainId: string; + namespace?: string; counterName: string; counterType: string; + source: CounterEventSource; incrementBy: number; } @@ -17,10 +21,14 @@ export interface CounterMetric { * Details about the counter to be incremented */ export interface IncrementCounterParams { + /** The namespace to increment this counter on */ + namespace?: string; /** The name of the counter **/ counterName: string; /** The counter type ("count" by default) **/ counterType?: string; + /** The source of the event we are counting */ + source?: CounterEventSource; /** Increment the counter by this number (1 if not specified) **/ incrementBy?: number; } diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index 38298b42ed9c1..fb2894d7f3903 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -23,10 +23,9 @@ export type { UsageCountersSavedObjectAttributes, IncrementCounterParams, UsageCounter, - SerializeCounterParams, } from './usage_counters'; -export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey } from './usage_counters'; +export { serializeCounterKey, USAGE_COUNTERS_SAVED_OBJECT_TYPE } from './usage_counters'; export type { UsageCollectionSetup } from './plugin'; export { config } from './config'; diff --git a/src/plugins/usage_collection/server/mocks.ts b/src/plugins/usage_collection/server/mocks.ts index bc59cf3cbef22..36f30357f6207 100644 --- a/src/plugins/usage_collection/server/mocks.ts +++ b/src/plugins/usage_collection/server/mocks.ts @@ -26,12 +26,12 @@ export const createUsageCollectionSetupMock = () => { executionContext: executionContextServiceMock.createSetupContract(), maximumWaitTimeForAllCollectorsInS: 1, }); - const { createUsageCounter, getUsageCounterByType } = + const { createUsageCounter, getUsageCounterByDomainId } = usageCountersServiceMock.createSetupContract(); const usageCollectionSetupMock: jest.Mocked = { createUsageCounter, - getUsageCounterByType, + getUsageCounterByDomainId, bulkFetch: jest.fn().mockImplementation(collectorSet.bulkFetch), getCollectorByType: jest.fn().mockImplementation(collectorSet.getCollectorByType), toApiFieldNames: jest.fn().mockImplementation(collectorSet.toApiFieldNames), diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index 7139de5091b50..ca551c129d84e 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -33,7 +33,7 @@ export interface UsageCollectionSetup { /** * Returns a usage counter by type */ - getUsageCounterByType: (type: string) => UsageCounter | undefined; + getUsageCounterByDomainId: (type: string) => UsageCounter | undefined; /** * Creates a usage collector to collect plugin telemetry data. * registerCollector must be called to connect the created collector with the service. @@ -112,13 +112,13 @@ export class UsageCollectionPlugin implements Plugin { bufferDurationMs: config.usageCounters.bufferDuration.asMilliseconds(), }); - const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core); + const usageCountersServiceSetup = this.usageCountersService.setup(core); + const { createUsageCounter, getUsageCounterByDomainId } = usageCountersServiceSetup; - const uiCountersUsageCounter = createUsageCounter('uiCounter'); const router = core.http.createRouter(); setupRoutes({ router, - uiCountersUsageCounter, + usageCountersServiceSetup, getSavedObjects: () => this.savedObjects, collectorSet, config: { @@ -141,7 +141,7 @@ export class UsageCollectionPlugin implements Plugin { toApiFieldNames: collectorSet.toApiFieldNames, toObject: collectorSet.toObject, createUsageCounter, - getUsageCounterByType, + getUsageCounterByDomainId, }; } diff --git a/src/plugins/usage_collection/server/report/index.ts b/src/plugins/usage_collection/server/report/index.ts index 1a4cbd0e209a0..38df62c7e2f07 100644 --- a/src/plugins/usage_collection/server/report/index.ts +++ b/src/plugins/usage_collection/server/report/index.ts @@ -6,5 +6,5 @@ * Side Public License, v 1. */ -export { storeReport } from './store_report'; +export { storeUiReport } from './store_ui_report'; export { reportSchema } from './schema'; diff --git a/src/plugins/usage_collection/server/report/schema.ts b/src/plugins/usage_collection/server/report/schema.ts index 1f76d3a4db76d..9f4e65c224e08 100644 --- a/src/plugins/usage_collection/server/report/schema.ts +++ b/src/plugins/usage_collection/server/report/schema.ts @@ -36,6 +36,7 @@ export const reportSchema = schema.object({ type: schema.string(), appName: schema.string(), eventName: schema.string(), + namespace: schema.maybe(schema.string()), total: schema.number(), }) ) diff --git a/src/plugins/usage_collection/server/report/store_report.test.mocks.ts b/src/plugins/usage_collection/server/report/store_ui_report.test.mocks.ts similarity index 100% rename from src/plugins/usage_collection/server/report/store_report.test.mocks.ts rename to src/plugins/usage_collection/server/report/store_ui_report.test.mocks.ts diff --git a/src/plugins/usage_collection/server/report/store_report.test.ts b/src/plugins/usage_collection/server/report/store_ui_report.test.ts similarity index 70% rename from src/plugins/usage_collection/server/report/store_report.test.ts rename to src/plugins/usage_collection/server/report/store_ui_report.test.ts index e1c33197142d9..4fb98c8897f0a 100644 --- a/src/plugins/usage_collection/server/report/store_report.test.ts +++ b/src/plugins/usage_collection/server/report/store_ui_report.test.ts @@ -6,22 +6,24 @@ * Side Public License, v 1. */ -import { storeApplicationUsageMock } from './store_report.test.mocks'; +import { storeApplicationUsageMock } from './store_ui_report.test.mocks'; import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; -import { storeReport } from './store_report'; -import { ReportSchemaType } from './schema'; import { METRIC_TYPE } from '@kbn/analytics'; +import type { ReportSchemaType } from './schema'; +import { storeUiReport } from './store_ui_report'; import { usageCountersServiceMock } from '../usage_counters/usage_counters_service.mock'; -describe('store_report', () => { - const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); - const uiCountersUsageCounter = usageCountersServiceSetup.createUsageCounter('uiCounter'); - +describe('store_ui_report', () => { let repository: ReturnType; + let usageCountersServiceSetup: ReturnType; + let usageCounterMock: ReturnType; beforeEach(() => { + usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); repository = savedObjectsRepositoryMock.create(); + usageCounterMock = usageCountersServiceSetup.createUsageCounter('dashboards'); + usageCountersServiceSetup.createUsageCounter.mockReturnValue(usageCounterMock); }); afterEach(() => { @@ -64,8 +66,22 @@ describe('store_report', () => { }, }, }; - await storeReport(repository, uiCountersUsageCounter, report); + await storeUiReport(repository, usageCountersServiceSetup, report); + await new Promise((resolve) => setTimeout(resolve, 4000)); + expect(usageCountersServiceSetup.createUsageCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "dashboards", + ], + Array [ + "test-app-name", + ], + Array [ + "test-app-name", + ], + ] + `); expect(repository.create.mock.calls).toMatchInlineSnapshot(` Array [ Array [ @@ -96,21 +112,22 @@ describe('store_report', () => { ], ] `); - expect((uiCountersUsageCounter.incrementCounter as jest.Mock).mock.calls) - .toMatchInlineSnapshot(` + expect((usageCounterMock.incrementCounter as jest.Mock).mock.calls).toMatchInlineSnapshot(` Array [ Array [ Object { - "counterName": "test-app-name:test-event-name", + "counterName": "test-event-name", "counterType": "loaded", "incrementBy": 1, + "source": "ui", }, ], Array [ Object { - "counterName": "test-app-name:test-event-name", + "counterName": "test-event-name", "counterType": "click", "incrementBy": 2, + "source": "ui", }, ], ] @@ -131,7 +148,7 @@ describe('store_report', () => { uiCounter: void 0, application_usage: void 0, }; - await storeReport(repository, uiCountersUsageCounter, report); + await storeUiReport(repository, usageCountersServiceSetup, report); expect(repository.bulkCreate).not.toHaveBeenCalled(); expect(repository.incrementCounter).not.toHaveBeenCalled(); diff --git a/src/plugins/usage_collection/server/report/store_report.ts b/src/plugins/usage_collection/server/report/store_ui_report.ts similarity index 86% rename from src/plugins/usage_collection/server/report/store_report.ts rename to src/plugins/usage_collection/server/report/store_ui_report.ts index 40ea5cbf414a8..1cf966e3466e8 100644 --- a/src/plugins/usage_collection/server/report/store_report.ts +++ b/src/plugins/usage_collection/server/report/store_ui_report.ts @@ -11,12 +11,11 @@ import moment from 'moment'; import { chain, sumBy } from 'lodash'; import { ReportSchemaType } from './schema'; import { storeApplicationUsage } from './store_application_usage'; -import { UsageCounter } from '../usage_counters'; -import { serializeUiCounterName } from '../../common/ui_counters'; +import { type UsageCountersServiceSetup } from '../usage_counters'; -export async function storeReport( +export async function storeUiReport( internalRepository: ISavedObjectsRepository, - uiCountersUsageCounter: UsageCounter, + counters: UsageCountersServiceSetup, report: ReportSchemaType ) { const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : []; @@ -60,11 +59,15 @@ export async function storeReport( // UI Counters ...uiCounters.map(async ([, metric]) => { const { appName, eventName, total, type } = metric; - const counterName = serializeUiCounterName({ appName, eventName }); - uiCountersUsageCounter.incrementCounter({ - counterName, + + const counter = + counters.getUsageCounterByDomainId(appName) ?? counters.createUsageCounter(appName); + + counter.incrementCounter({ + counterName: eventName, counterType: type, incrementBy: total, + source: 'ui', }); }), // Application Usage diff --git a/src/plugins/usage_collection/server/routes/index.ts b/src/plugins/usage_collection/server/routes/index.ts index 01d26b177218c..3bb42cf4865ce 100644 --- a/src/plugins/usage_collection/server/routes/index.ts +++ b/src/plugins/usage_collection/server/routes/index.ts @@ -16,16 +16,16 @@ import { Observable } from 'rxjs'; import { CollectorSet } from '../collector'; import { registerUiCountersRoute } from './ui_counters'; import { registerStatsRoute } from './stats'; -import type { UsageCounter } from '../usage_counters'; +import type { UsageCountersServiceSetup } from '../usage_counters'; export function setupRoutes({ router, - uiCountersUsageCounter, + usageCountersServiceSetup, getSavedObjects, ...rest }: { router: IRouter; getSavedObjects: () => ISavedObjectsRepository | undefined; - uiCountersUsageCounter: UsageCounter; + usageCountersServiceSetup: UsageCountersServiceSetup; config: { allowAnonymous: boolean; kibanaIndex: string; @@ -41,6 +41,6 @@ export function setupRoutes({ metrics: MetricsServiceSetup; overallStatus$: Observable; }) { - registerUiCountersRoute(router, getSavedObjects, uiCountersUsageCounter); + registerUiCountersRoute(router, getSavedObjects, usageCountersServiceSetup); registerStatsRoute({ router, ...rest }); } diff --git a/src/plugins/usage_collection/server/routes/ui_counters.ts b/src/plugins/usage_collection/server/routes/ui_counters.ts index 9752afeae3ef9..e19dd7c57ca6d 100644 --- a/src/plugins/usage_collection/server/routes/ui_counters.ts +++ b/src/plugins/usage_collection/server/routes/ui_counters.ts @@ -7,15 +7,15 @@ */ import { schema } from '@kbn/config-schema'; -import { IRouter, ISavedObjectsRepository } from '@kbn/core/server'; -import { storeReport, reportSchema } from '../report'; -import { UsageCounter } from '../usage_counters'; -import { UiCounters } from '../../common/types'; +import type { IRouter, ISavedObjectsRepository } from '@kbn/core/server'; +import { storeUiReport, reportSchema } from '../report'; +import type { UsageCountersServiceSetup } from '../usage_counters'; +import type { UiCounters } from '../../common/types'; export function registerUiCountersRoute( router: IRouter, getSavedObjects: () => ISavedObjectsRepository | undefined, - uiCountersUsageCounter: UsageCounter + usageCountersServiceSetup: UsageCountersServiceSetup ) { router.post( { @@ -33,7 +33,8 @@ export function registerUiCountersRoute( if (!internalRepository) { throw Error(`The saved objects client hasn't been initialised yet`); } - await storeReport(internalRepository, uiCountersUsageCounter, requestBody.report); + // we pass the whole usageCountersServiceSetup, so that we can create UI counters dynamically + await storeUiReport(internalRepository, usageCountersServiceSetup, requestBody.report); const bodyOk: UiCounters.v1.UiCountersResponseOk = { status: 'ok', }; diff --git a/src/plugins/usage_collection/server/usage_counters/index.ts b/src/plugins/usage_collection/server/usage_counters/index.ts index 05d36aaef502f..fcbcf9f02e681 100644 --- a/src/plugins/usage_collection/server/usage_counters/index.ts +++ b/src/plugins/usage_collection/server/usage_counters/index.ts @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { UsageCounters } from '../../common/types'; +import { UsageCounters } from '../../common'; export type IncrementCounterParams = UsageCounters.v1.IncrementCounterParams; export type { UsageCountersServiceSetup } from './usage_counters_service'; @@ -13,5 +13,4 @@ export type { UsageCountersSavedObjectAttributes, UsageCountersSavedObject } fro export type { IUsageCounter as UsageCounter } from './usage_counter'; export { UsageCountersService } from './usage_counters_service'; -export type { SerializeCounterParams } from './saved_objects'; -export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey } from './saved_objects'; +export { serializeCounterKey, USAGE_COUNTERS_SAVED_OBJECT_TYPE } from './saved_objects'; diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts index fa40fd5a8d2be..7a8934e4559fa 100644 --- a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts @@ -9,7 +9,7 @@ import { serializeCounterKey, storeCounter } from './saved_objects'; import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; -import { UsageCounters } from '../../common/types'; +import { UsageCounters } from '../../common'; import moment from 'moment'; @@ -19,10 +19,12 @@ describe('counterKey', () => { domainId: 'a', counterName: 'b', counterType: 'c', + namespace: 'default', + source: 'ui', date: moment('09042021', 'DDMMYYYY'), }); - expect(result).toMatchInlineSnapshot(`"a:09042021:c:b"`); + expect(result).toEqual('a:b:c:ui:20210409:default'); }); }); @@ -40,20 +42,22 @@ describe('storeCounter', () => { }); it('stores counter in a saved object', async () => { - const counterMetric: UsageCounters.v1.CounterMetric = { + const metric: UsageCounters.v1.CounterMetric = { domainId: 'a', counterName: 'b', counterType: 'c', + namespace: 'default', + source: 'ui', incrementBy: 13, }; - await storeCounter(counterMetric, internalRepository); + await storeCounter({ metric, soRepository: internalRepository }); expect(internalRepository.incrementCounter).toBeCalledTimes(1); expect(internalRepository.incrementCounter.mock.calls[0]).toMatchInlineSnapshot(` Array [ - "usage-counters", - "a:09042021:c:b", + "usage-counter", + "a:b:c:ui:20210409", Array [ Object { "fieldName": "count", @@ -61,10 +65,12 @@ describe('storeCounter', () => { }, ], Object { + "namespace": "default", "upsertAttributes": Object { "counterName": "b", "counterType": "c", "domainId": "a", + "source": "ui", }, }, ] diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts index 402dabb62b96b..21e51f81040df 100644 --- a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts @@ -6,13 +6,14 @@ * Side Public License, v 1. */ +import moment from 'moment'; +import { USAGE_COUNTERS_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import type { SavedObject, SavedObjectsRepository, SavedObjectsServiceSetup, } from '@kbn/core/server'; -import moment from 'moment'; -import { UsageCounters } from '../../common/types'; +import { UsageCounters } from '../../common'; /** * The attributes stored in the UsageCounters' SavedObjects @@ -24,6 +25,8 @@ export interface UsageCountersSavedObjectAttributes { counterName: string; /** The counter type **/ counterType: string; + /** The source of the event that is being counted: 'server' | 'ui' **/ + source: string; /** Number of times the event has occurred **/ count: number; } @@ -34,13 +37,30 @@ export interface UsageCountersSavedObjectAttributes { export type UsageCountersSavedObject = SavedObject; /** The Saved Objects type for Usage Counters **/ -export const USAGE_COUNTERS_SAVED_OBJECT_TYPE = 'usage-counters'; +export const USAGE_COUNTERS_SAVED_OBJECT_TYPE = 'usage-counter'; -export const registerUsageCountersSavedObjectType = ( +export const registerUsageCountersSavedObjectTypes = ( savedObjectsSetup: SavedObjectsServiceSetup ) => { savedObjectsSetup.registerType({ name: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + indexPattern: USAGE_COUNTERS_SAVED_OBJECT_INDEX, + hidden: false, + namespaceType: 'single', + mappings: { + dynamic: false, + properties: { + domainId: { type: 'keyword' }, + counterName: { type: 'keyword' }, + counterType: { type: 'keyword' }, + source: { type: 'keyword' }, + }, + }, + }); + + // DEPRECATED: we keep it just to ensure non-reindex migrations (serverless) + savedObjectsSetup.registerType({ + name: 'usage-counters', hidden: false, namespaceType: 'agnostic', mappings: { @@ -56,53 +76,68 @@ export const registerUsageCountersSavedObjectType = ( * Parameters to the `serializeCounterKey` method * @internal used in kibana_usage_collectors */ -export interface SerializeCounterParams { +export interface SerializeCounterKeyParams { /** The domain ID registered in the UsageCounter **/ domainId: string; /** The counter name **/ counterName: string; /** The counter type **/ counterType: string; - /** The date to which serialize the key **/ - date: moment.MomentInput; + /** The namespace of this counter */ + namespace?: string; + /** The source of the event we are counting */ + source: string; + /** The date to which serialize the key (defaults to 'now') **/ + date?: moment.MomentInput; } /** * Generates a key based on the UsageCounter details * @internal used in kibana_usage_collectors - * @param opts {@link SerializeCounterParams} + * @param opts {@link SerializeCounterKeyParams} */ export const serializeCounterKey = ({ domainId, counterName, counterType, + namespace, + source, date, -}: SerializeCounterParams) => { - const dayDate = moment(date).format('DDMMYYYY'); - return `${domainId}:${dayDate}:${counterType}:${counterName}`; +}: SerializeCounterKeyParams) => { + const dayDate = moment(date).format('YYYYMMDD'); + // e.g. 'dashboards:viewed:total:ui:20240628' // namespace-agnostic counters + // e.g. 'dashboards:viewed:total:ui:20240628:default' // namespaced counters + const namespaceSuffix = namespace ? `:${namespace}` : ''; + return `${domainId}:${counterName}:${counterType}:${source}:${dayDate}${namespaceSuffix}`; }; -export const storeCounter = async ( - counterMetric: UsageCounters.v1.CounterMetric, - internalRepository: Pick -) => { - const { counterName, counterType, domainId, incrementBy } = counterMetric; +export interface StoreCounterParams { + metric: UsageCounters.v1.CounterMetric; + soRepository: Pick; +} + +export const storeCounter = async ({ metric, soRepository }: StoreCounterParams) => { + const { namespace, counterName, counterType, domainId, source, incrementBy } = metric; + // same counter key can be used in different namespaces (no need to make namespace part of the key) const key = serializeCounterKey({ - date: moment.now(), domainId, counterName, counterType, + source, + date: moment.now(), }); - return await internalRepository.incrementCounter( + return await soRepository.incrementCounter( USAGE_COUNTERS_SAVED_OBJECT_TYPE, key, [{ fieldName: 'count', incrementBy }], { + ...(namespace && { namespace }), upsertAttributes: { domainId, counterName, counterType, + source, }, } ); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts index f3f6c2870ce25..aea0ab44f6cdd 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts @@ -5,10 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { UsageCounter } from './usage_counter'; -import type { UsageCounters } from '../../common/types'; import * as Rx from 'rxjs'; -import * as rxOp from 'rxjs'; +import { UsageCounter } from './usage_counter'; +import type { UsageCounters } from '../../common'; describe('UsageCounter', () => { const domainId = 'test-domain-id'; @@ -21,18 +20,37 @@ describe('UsageCounter', () => { describe('#incrementCounter', () => { it('#incrementCounter calls counter$.next', async () => { - const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); - usageCounter.incrementCounter({ counterName: 'test', counterType: 'type', incrementBy: 13 }); + const result = Rx.firstValueFrom(counter$.pipe(Rx.take(1), Rx.toArray())); + usageCounter.incrementCounter({ + counterName: 'test', + counterType: 'type', + incrementBy: 13, + source: 'ui', + namespace: 'second', + }); await expect(result).resolves.toEqual([ - { counterName: 'test', counterType: 'type', domainId: 'test-domain-id', incrementBy: 13 }, + { + domainId: 'test-domain-id', + counterType: 'type', + counterName: 'test', + source: 'ui', + namespace: 'second', + incrementBy: 13, + }, ]); }); it('passes default configs to counter$', async () => { - const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); + const result = Rx.firstValueFrom(counter$.pipe(Rx.take(1), Rx.toArray())); usageCounter.incrementCounter({ counterName: 'test' }); await expect(result).resolves.toEqual([ - { counterName: 'test', counterType: 'count', domainId: 'test-domain-id', incrementBy: 1 }, + { + domainId: 'test-domain-id', + counterType: 'count', + counterName: 'test', + source: 'server', + incrementBy: 1, + }, ]); }); }); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts index 80bd32ae4d1db..6f8c892f2627a 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts @@ -7,9 +7,9 @@ */ import * as Rx from 'rxjs'; -import { UsageCounters } from '../../common/types'; +import type { UsageCounters } from '../../common'; -export interface UsageCounterDeps { +export interface UsageCounterParams { domainId: string; counter$: Rx.Subject; } @@ -31,19 +31,27 @@ export class UsageCounter implements IUsageCounter { private domainId: string; private counter$: Rx.Subject; - constructor({ domainId, counter$ }: UsageCounterDeps) { + constructor({ domainId, counter$ }: UsageCounterParams) { this.domainId = domainId; this.counter$ = counter$; } public incrementCounter = (params: UsageCounters.v1.IncrementCounterParams) => { - const { counterName, counterType = 'count', incrementBy = 1 } = params; + const { + counterName, + counterType = 'count', + source = 'server', // default behavior before introducing the property + incrementBy = 1, + namespace, + } = params; this.counter$.next({ - counterName, domainId: this.domainId, + counterName, counterType, + source, incrementBy, + ...(namespace && { namespace }), }); }; } diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts index 0d96def540540..8e8627c5aea2d 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts @@ -13,7 +13,7 @@ import type { UsageCounter } from './usage_counter'; const createSetupContractMock = () => { const setupContract: jest.Mocked = { createUsageCounter: jest.fn(), - getUsageCounterByType: jest.fn(), + getUsageCounterByDomainId: jest.fn(), }; setupContract.createUsageCounter.mockReturnValue({ diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts index 1222924b6ec94..1350c8b706b87 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts @@ -40,7 +40,7 @@ describe('UsageCountersService', () => { const usageCounter = createUsageCounter('test-counter'); usageCounter.incrementCounter({ counterName: 'counterA' }); - usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA', namespace: 'second', source: 'ui' }); const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); usageCountersService['flushCache$'].next(); @@ -48,10 +48,10 @@ describe('UsageCountersService', () => { await expect(dataInSourcePromise).resolves.toHaveLength(2); }); - it('registers savedObject type during setup', () => { + it('registers savedObject types during setup', () => { const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); usageCountersService.setup(coreSetup); - expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1); + expect(coreSetup.savedObjects.registerType).toBeCalledTimes(2); }); it('flushes cached data on start', async () => { @@ -67,28 +67,31 @@ describe('UsageCountersService', () => { const usageCounter = createUsageCounter('test-counter'); usageCounter.incrementCounter({ counterName: 'counterA' }); - usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA', namespace: 'second', source: 'ui' }); const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); usageCountersService.start(coreStart); usageCountersService['source$'].complete(); await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(` - Array [ - Object { - "counterName": "counterA", - "counterType": "count", - "domainId": "test-counter", - "incrementBy": 1, - }, - Object { - "counterName": "counterA", - "counterType": "count", - "domainId": "test-counter", - "incrementBy": 1, - }, - ] - `); + Array [ + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + "source": "server", + }, + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + "namespace": "second", + "source": "ui", + }, + ] + `); }); it('buffers data into savedObject', async () => { @@ -114,8 +117,8 @@ describe('UsageCountersService', () => { expect(mockIncrementCounter.mock.calls).toMatchInlineSnapshot(` Array [ Array [ - "usage-counters", - "test-counter:09042021:count:counterA", + "usage-counter", + "test-counter:counterA:count:server:20210409", Array [ Object { "fieldName": "count", @@ -127,12 +130,13 @@ describe('UsageCountersService', () => { "counterName": "counterA", "counterType": "count", "domainId": "test-counter", + "source": "server", }, }, ], Array [ - "usage-counters", - "test-counter:09042021:count:counterB", + "usage-counter", + "test-counter:counterB:count:server:20210409", Array [ Object { "fieldName": "count", @@ -144,6 +148,7 @@ describe('UsageCountersService', () => { "counterName": "counterB", "counterType": "count", "domainId": "test-counter", + "source": "server", }, }, ], @@ -162,9 +167,9 @@ describe('UsageCountersService', () => { const mockError = new Error('failed.'); const mockIncrementCounter = jest.fn().mockImplementation((_, key) => { switch (key) { - case 'test-counter:09042021:count:counterA': + case 'test-counter:counterA:count:server:20210409': throw mockError; - case 'test-counter:09042021:count:counterB': + case 'test-counter:counterB:count:server:20210409': return 'pass'; default: throw new Error(`unknown key ${key}`); @@ -232,11 +237,11 @@ describe('UsageCountersService', () => { Array [ Object { "incrementBy": 2, - "key": "test-counter:09042021:count:counterA", + "key": "test-counter:counterA:count:server:20210409", }, Object { "incrementBy": 1, - "key": "test-counter:09042021:count:counterA", + "key": "test-counter:counterA:count:server:20210409", }, ] `); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts index c0e71bdd28dbf..28c3aaf2be148 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts @@ -8,18 +8,18 @@ import * as Rx from 'rxjs'; import * as rxOp from 'rxjs'; -import { +import moment from 'moment'; +import type { SavedObjectsRepository, SavedObjectsServiceSetup, SavedObjectsServiceStart, } from '@kbn/core/server'; import type { Logger, LogMeta } from '@kbn/core/server'; -import moment from 'moment'; -import { UsageCounter } from './usage_counter'; -import { UsageCounters } from '../../common/types'; +import { type IUsageCounter, UsageCounter } from './usage_counter'; +import type { UsageCounters } from '../../common'; import { - registerUsageCountersSavedObjectType, + registerUsageCountersSavedObjectTypes, storeCounter, serializeCounterKey, } from './saved_objects'; @@ -35,8 +35,8 @@ export interface UsageCountersServiceDeps { } export interface UsageCountersServiceSetup { - createUsageCounter: (type: string) => UsageCounter; - getUsageCounterByType: (type: string) => UsageCounter | undefined; + createUsageCounter: (domainId: string) => IUsageCounter; + getUsageCounterByDomainId: (domainId: string) => IUsageCounter | undefined; } /* internal */ @@ -95,11 +95,11 @@ export class UsageCountersService { storingCache$.next(false); }); - registerUsageCountersSavedObjectType(core.savedObjects); + registerUsageCountersSavedObjectTypes(core.savedObjects); return { createUsageCounter: this.createUsageCounter, - getUsageCounterByType: this.getUsageCounterByType, + getUsageCounterByDomainId: this.getUsageCounterByDomainId, }; }; @@ -137,11 +137,11 @@ export class UsageCountersService { private storeDate$( counters: UsageCounters.v1.CounterMetric[], - internalRepository: Pick + soRepository: Pick ) { return Rx.forkJoin( - counters.map((counter) => - Rx.defer(() => storeCounter(counter, internalRepository)).pipe( + counters.map((metric) => + Rx.defer(() => storeCounter({ metric, soRepository })).pipe( rxOp.retry(this.retryCount), rxOp.catchError((error) => { this.logger.warn(error); @@ -152,22 +152,17 @@ export class UsageCountersService { ); } - private createUsageCounter = (type: string): UsageCounter => { - if (this.counterSets.get(type)) { - throw new Error(`Usage counter set "${type}" already exists.`); + private createUsageCounter = (domainId: string): IUsageCounter => { + if (this.counterSets.get(domainId)) { + throw new Error(`Usage counter set "${domainId}" already exists.`); } - const counterSet = new UsageCounter({ - domainId: type, - counter$: this.source$, - }); - - this.counterSets.set(type, counterSet); - + const counterSet = new UsageCounter({ domainId, counter$: this.source$ }); + this.counterSets.set(domainId, counterSet); return counterSet; }; - private getUsageCounterByType = (type: string): UsageCounter | undefined => { + private getUsageCounterByDomainId = (type: string): IUsageCounter | undefined => { return this.counterSets.get(type); }; @@ -176,8 +171,15 @@ export class UsageCountersService { ): Record => { const date = moment.now(); return counters.reduce((acc, counter) => { - const { counterName, domainId, counterType } = counter; - const key = serializeCounterKey({ domainId, counterName, counterType, date }); + const { domainId, counterName, counterType, namespace, source } = counter; + const key = serializeCounterKey({ + domainId, + counterName, + counterType, + namespace, + source, + date, + }); const existingCounter = acc[key]; if (!existingCounter) { acc[key] = counter; diff --git a/src/plugins/usage_collection/tsconfig.json b/src/plugins/usage_collection/tsconfig.json index e7c24d604be96..53fba66071c5e 100644 --- a/src/plugins/usage_collection/tsconfig.json +++ b/src/plugins/usage_collection/tsconfig.json @@ -23,6 +23,7 @@ "@kbn/analytics-collection-utils", "@kbn/logging", "@kbn/ebt", + "@kbn/core-saved-objects-server", ], "exclude": [ "target/**/*", diff --git a/test/api_integration/apis/ui_counters/ui_counters.ts b/test/api_integration/apis/ui_counters/ui_counters.ts index 3016af858d6be..cf6b6eee51d21 100644 --- a/test/api_integration/apis/ui_counters/ui_counters.ts +++ b/test/api_integration/apis/ui_counters/ui_counters.ts @@ -29,7 +29,7 @@ export default function ({ getService }: FtrProviderContext) { const { body: { saved_objects: savedObjects }, } = await supertest - .get('/api/saved_objects/_find?type=usage-counters') + .get('/api/saved_objects/_find?type=usage-counter') .set('kbn-xsrf', 'kibana') .expect(200); @@ -51,7 +51,8 @@ export default function ({ getService }: FtrProviderContext) { counterType: UiCounterMetricType ): UsageCountersSavedObject[] => { const matchingEventName = savedObjects.filter( - ({ attributes }) => attributes.counterName === `${APP_NAME}:${eventName}` + ({ attributes: { domainId, counterName } }) => + domainId === APP_NAME && counterName === eventName ); if (!matchingEventName.length) { throw new Error( diff --git a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts index be4797b1cf61f..c7137470e0e7e 100644 --- a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts +++ b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts @@ -21,7 +21,7 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide await new Promise((res) => setTimeout(res, 10 * 1000)); return await supertest - .get('/api/saved_objects/_find?type=usage-counters') + .get('/api/saved_objects/_find?type=usage-counter') .set('kbn-xsrf', 'true') .expect(200) .then(({ body }) => { @@ -40,13 +40,13 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide describe('Usage Counters service', () => { before(async () => { const key = serializeCounterKey({ + domainId: 'usageCollectionTestPlugin', counterName: 'routeAccessed', counterType: 'count', - domainId: 'usageCollectionTestPlugin', - date: Date.now(), + source: 'server', }); - await supertest.delete(`/api/saved_objects/usage-counters/${key}`).set('kbn-xsrf', 'true'); + await supertest.delete(`/api/saved_objects/counter/${key}`).set('kbn-xsrf', 'true'); }); it('stores usage counters sent during start and setup', async () => {