From 18356ef718ab4652663cd0a7511a7892a984b89b Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 12:32:30 +0300 Subject: [PATCH 01/11] config usage service + collector --- .../core_usage_data_service.test.ts | 316 +++++++++++++++++- .../core_usage_data_service.ts | 90 ++++- src/core/server/core_usage_data/index.ts | 2 +- src/core/server/core_usage_data/types.ts | 15 +- src/core/server/index.ts | 4 + src/core/server/plugins/plugins_service.ts | 20 +- src/core/server/plugins/types.ts | 26 ++ src/core/server/server.ts | 1 + src/plugins/kibana_usage_collection/README.md | 6 +- .../server/collectors/config_usage/README.md | 62 ++++ .../server/collectors/config_usage/index.ts | 9 + .../register_config_usage_collector.ts | 40 +++ ...x.test.ts => core_usage_collector.test.ts} | 0 .../server/collectors/index.ts | 1 + .../kibana_usage_collection/server/plugin.ts | 2 + 15 files changed, 579 insertions(+), 15 deletions(-) create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts rename src/plugins/kibana_usage_collection/server/collectors/core/{index.test.ts => core_usage_collector.test.ts} (100%) diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 1c28eca1f1dec..9a5c7586e6ca7 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -35,7 +35,24 @@ describe('CoreUsageDataService', () => { }); let service: CoreUsageDataService; - const configService = configServiceMock.create(); + const mockConfig = { + ui_metric: {}, + elasticsearch: { username: 'kibana_system', password: 'changeme' }, + plugins: { paths: ['some_path', 'another_path'] }, + server: { port: 5603, basePath: '/zvt', rewriteBasePath: true }, + logging: { json: false }, + usageCollection: { + uiCounters: { + debug: true, + username: 'some_user', + }, + }, + }; + + const configService = configServiceMock.create({ + getConfig$: mockConfig, + }); + configService.atPath.mockImplementation((path) => { if (path === 'elasticsearch') { return new BehaviorSubject(RawElasticsearchConfig.schema.validate({})); @@ -146,6 +163,7 @@ describe('CoreUsageDataService', () => { const { getCoreUsageData } = service.start({ savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage: new Map(), elasticsearch, }); expect(getCoreUsageData()).resolves.toMatchInlineSnapshot(` @@ -274,6 +292,302 @@ describe('CoreUsageDataService', () => { `); }); }); + + describe('getConfigsUsageData', () => { + const elasticsearch = elasticsearchServiceMock.createStart(); + const typeRegistry = savedObjectsServiceMock.createTypeRegistryMock(); + let exposedConfigsToUsage: Map>; + beforeEach(() => { + exposedConfigsToUsage = new Map(); + }); + + describe('config explicitly exposed to usage', () => { + it('returns [redacted] on unsafe complete match', async () => { + exposedConfigsToUsage.set('usageCollection', { + 'uiCounters.debug': false, + }); + exposedConfigsToUsage.set('server', { + basePath: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'server.basePath', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "server.basePath": "[redacted]", + "usageCollection.uiCounters.debug": "[redacted]", + } + `); + }); + + it('returns config value on safe complete match', async () => { + exposedConfigsToUsage.set('server', { + basePath: true, + }); + + configService.getUsedPaths.mockResolvedValue(['server.basePath']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "server.basePath": "/zvt", + } + `); + }); + + it('returns [redacted] on unsafe parent match', async () => { + exposedConfigsToUsage.set('usageCollection', { + uiCounters: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'usageCollection.uiCounters.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "usageCollection.uiCounters.debug": "[redacted]", + "usageCollection.uiCounters.username": "[redacted]", + } + `); + }); + + it('returns config value on safe parent match', async () => { + exposedConfigsToUsage.set('usageCollection', { + uiCounters: true, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'usageCollection.uiCounters.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "usageCollection.uiCounters.debug": true, + "usageCollection.uiCounters.username": "some_user", + } + `); + }); + + it('returns [redacted] on implicit arrays', async () => { + exposedConfigsToUsage.set('plugins', { + paths: true, + }); + + configService.getUsedPaths.mockResolvedValue(['plugins.paths']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "plugins.paths": Array [ + "some_path", + "another_path", + ], + } + `); + }); + }); + + describe('config not explicitly exposed to usage', () => { + it('returns [redacted] for string configs', async () => { + exposedConfigsToUsage.set('usageCollection', { + uiCounters: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'usageCollection.uiCounters.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "usageCollection.uiCounters.debug": "[redacted]", + "usageCollection.uiCounters.username": "[redacted]", + } + `); + }); + + it('returns config value on safe parent match', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'elasticsearch.password', + 'elasticsearch.username', + 'usageCollection.uiCounters.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "elasticsearch.password": "[redacted]", + "elasticsearch.username": "[redacted]", + "usageCollection.uiCounters.username": "[redacted]", + } + `); + }); + + it('returns [redacted] on implicit arrays', async () => { + configService.getUsedPaths.mockResolvedValue(['plugins.paths']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "plugins.paths": "[redacted]", + } + `); + }); + + it('returns config value for numbers', async () => { + configService.getUsedPaths.mockResolvedValue(['server.port']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "server.port": 5603, + } + `); + }); + + it('returns config value for booleans', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'logging.json', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "logging.json": false, + "usageCollection.uiCounters.debug": true, + } + `); + }); + + it('ignores exposed to usage configs but not used', async () => { + exposedConfigsToUsage.set('usageCollection', { + uiCounters: true, + }); + + configService.getUsedPaths.mockResolvedValue(['logging.json']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "logging.json": false, + } + `); + }); + }); + + it('loops over all used configs once each', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'usageCollection.uiCounters.debug', + 'logging.json', + ]); + + exposedConfigsToUsage.set('usageCollection', { + uiCounters: true, + }); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + const mockGetMarkedAsSafe = jest.fn().mockReturnValue({}); + // @ts-ignore + service['getMarkedAsSafe'] = mockGetMarkedAsSafe; + await getConfigsUsageData(); + + expect(mockGetMarkedAsSafe).toBeCalledTimes(2); + expect(mockGetMarkedAsSafe.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Map { + "usageCollection" => Object { + "uiCounters": true, + }, + }, + "usageCollection", + "usageCollection.uiCounters.debug", + ], + Array [ + Map { + "usageCollection" => Object { + "uiCounters": true, + }, + }, + undefined, + "logging.json", + ], + ] + `); + }); + }); }); describe('setup and stop', () => { diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index dff68bf1c524f..41a3183187905 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -7,7 +7,9 @@ */ import { Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; +import { takeUntil, first } from 'rxjs/operators'; +import { get } from 'lodash'; +import { hasConfigPathIntersection } from '@kbn/config'; import { CoreService } from 'src/core/types'; import { Logger, SavedObjectsServiceStart, SavedObjectTypeRegistry } from 'src/core/server'; @@ -16,11 +18,12 @@ import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config'; import { HttpConfigType, InternalHttpServiceSetup } from '../http'; import { LoggingConfigType } from '../logging'; import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config'; -import { +import type { CoreServicesUsageData, CoreUsageData, CoreUsageDataStart, CoreUsageDataSetup, + ConfigUsageData, } from './types'; import { isConfigured } from './is_configured'; import { ElasticsearchServiceStart } from '../elasticsearch'; @@ -30,6 +33,8 @@ import { CORE_USAGE_STATS_TYPE } from './constants'; import { CoreUsageStatsClient } from './core_usage_stats_client'; import { MetricsServiceSetup, OpsMetrics } from '..'; +export type ExposedConfigsToUsage = Map>; + export interface SetupDeps { http: InternalHttpServiceSetup; metrics: MetricsServiceSetup; @@ -39,6 +44,7 @@ export interface SetupDeps { export interface StartDeps { savedObjects: SavedObjectsServiceStart; elasticsearch: ElasticsearchServiceStart; + exposedConfigsToUsage: ExposedConfigsToUsage; } /** @@ -256,6 +262,77 @@ export class CoreUsageDataService implements CoreService { + const fullPath = `${pluginId}.${exposeKey}`; + const hasIntersection = hasConfigPathIntersection(usedPath, fullPath); + + return hasIntersection; + }); + if (!exposeKeyDetails) { + return { explicitlyMarked: false, isSafe: false }; + } + + const isSafe = exposeDetails[exposeKeyDetails]; + if (typeof isSafe === 'boolean') { + return { explicitlyMarked: true, isSafe }; + } + + return { explicitlyMarked: false, isSafe: false }; + } + + private async getNonDefaultKibanaConfigs(exposedConfigsToUsage: ExposedConfigsToUsage): Promise { + const config = await this.configService.getConfig$().pipe(first()).toPromise(); + const nonDefaultConfigs = config.toRaw(); + const usedPaths = await this.configService.getUsedPaths(); + const exposedConfigsKeys = [...exposedConfigsToUsage.keys()]; + + return usedPaths.reduce((acc, usedPath) => { + const configValue = get(nonDefaultConfigs, usedPath); + const pluginId = exposedConfigsKeys.find(exposedConfigsKey => usedPath.startsWith(exposedConfigsKey)); + const { explicitlyMarked, isSafe } = this.getMarkedAsSafe(exposedConfigsToUsage, pluginId, usedPath); + + // explicitly marked as safe + if (explicitlyMarked && isSafe) { + acc[usedPath] = configValue; + } + + // explicitly marked as unsafe + if (explicitlyMarked && !isSafe) { + acc[usedPath] = '[redacted]'; + } + + /** + * not all types of values may contain sensitive values. + * Report boolean and number configs if not explicitly marked as unsafe. + */ + if (!explicitlyMarked) { + switch (typeof configValue) { + case 'number': + case 'boolean': + acc[usedPath] = configValue; + break; + case 'undefined': + acc[usedPath] = 'undefined'; + break; + default: { + acc[usedPath] = '[redacted]'; + } + } + } + + return acc; + }, {} as Record); + } + setup({ http, metrics, savedObjectsStartPromise }: SetupDeps) { metrics .getOpsMetrics$() @@ -316,11 +393,14 @@ export class CoreUsageDataService implements CoreService { - return this.getCoreUsageData(savedObjects, elasticsearch); + getCoreUsageData: async () => { + return await this.getCoreUsageData(savedObjects, elasticsearch); }, + getConfigsUsageData: async () => { + return await this.getNonDefaultKibanaConfigs(exposedConfigsToUsage); + } }; } diff --git a/src/core/server/core_usage_data/index.ts b/src/core/server/core_usage_data/index.ts index 4e0200ed1e4ea..638fc65522433 100644 --- a/src/core/server/core_usage_data/index.ts +++ b/src/core/server/core_usage_data/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export type { CoreUsageDataSetup, CoreUsageDataStart } from './types'; +export type { CoreUsageDataSetup, ConfigUsageData, CoreUsageDataStart } from './types'; export { CoreUsageDataService } from './core_usage_data_service'; export { CoreUsageStatsClient } from './core_usage_stats_client'; diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 46148e314bfee..79ce2494b0a23 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -7,7 +7,7 @@ */ import { CoreUsageStatsClient } from './core_usage_stats_client'; -import { ISavedObjectTypeRegistry, SavedObjectTypeRegistry } from '..'; +import { ISavedObjectTypeRegistry, SavedObjectTypeRegistry, PluginConfigUsageDescriptors } from '..'; /** * @internal @@ -122,6 +122,18 @@ export interface CoreUsageData extends CoreUsageStats { environment: CoreEnvironmentUsageData; } +/** + * Type describing Core's usage data payload + * @internal + */ +export type ConfigUsageData = Record + +/** + * Type describing Core's usage data payload + * @internal + */ +export type ExposedConfigsToUsage = Map>; + /** * Usage data from Core services * @internal @@ -263,4 +275,5 @@ export interface CoreUsageDataStart { * @internal * */ getCoreUsageData(): Promise; + getConfigsUsageData(): Promise; } diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 2c6fa74cb54a0..ad26ca39506fb 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -64,6 +64,7 @@ import { CoreUsageStats, CoreUsageData, CoreConfigUsageData, + ConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData, } from './core_usage_data'; @@ -74,6 +75,7 @@ export type { CoreConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData, + ConfigUsageData, }; export { bootstrap } from './bootstrap'; @@ -256,6 +258,7 @@ export type { PluginManifest, PluginName, SharedGlobalConfig, + MakeUsageFromSchema, } from './plugins'; export { @@ -411,6 +414,7 @@ export type { CoreStatus, ServiceStatus, ServiceStatusLevel, StatusServiceSetup export type { CoreUsageDataStart } from './core_usage_data'; + /** * Plugin specific context passed to a route handler. * diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts index 09be40ecaf2a2..73a431edcf0f0 100644 --- a/src/core/server/plugins/plugins_service.ts +++ b/src/core/server/plugins/plugins_service.ts @@ -9,14 +9,19 @@ import Path from 'path'; import { Observable } from 'rxjs'; import { filter, first, map, mergeMap, tap, toArray } from 'rxjs/operators'; -import { pick } from '@kbn/std'; +import { pick, getFlattenedObject } from '@kbn/std'; import { CoreService } from '../../types'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { discover, PluginDiscoveryError, PluginDiscoveryErrorType } from './discovery'; import { PluginWrapper } from './plugin'; -import { DiscoveredPlugin, PluginConfigDescriptor, PluginName, InternalPluginInfo } from './types'; +import { + DiscoveredPlugin, + PluginConfigDescriptor, + PluginName, + InternalPluginInfo, +} from './types'; import { PluginsConfig, PluginsConfigType } from './plugins_config'; import { PluginsSystem } from './plugins_system'; import { InternalCoreSetup, InternalCoreStart } from '../internal_types'; @@ -75,6 +80,7 @@ export class PluginsService implements CoreService; private readonly pluginConfigDescriptors = new Map(); private readonly uiPluginInternalInfo = new Map(); + private readonly pluginConfigUsageDescriptors = new Map>(); constructor(private readonly coreContext: CoreContext) { this.log = coreContext.logger.get('plugins-service'); @@ -109,6 +115,10 @@ export class PluginsService implements CoreService { * {@link PluginConfigSchema} */ schema: PluginConfigSchema; + /** + * Expose non-default configs to usage collection to be sent via telemetry. + * set a config to `true` to report the actual changed config value. + * set a config to `false` to report the changed config value as [redacted]. + * + * All changed configs except booleans and numbers will be reported + * as [redacted] unless otherwise specified. + * + * {@link MakeUsageFromSchema} + */ + exposeToUsage?: MakeUsageFromSchema; } +/** + * List of configuration values that will be exposed to usage collection. + * If parent node or actual config path is set to `true` then the actual value + * of these configs will be reoprted. + * If parent node or actual config path is set to `false` then the config + * will be reported as [redacted]. + * + * @public + */ +export type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends object + ? MakeUsageFromSchema | boolean + : boolean; +}; + /** * Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays * that use it as a key or value more obvious. diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 45d11f9013fed..30e552e24b299 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -247,6 +247,7 @@ export class Server { const coreUsageDataStart = this.coreUsageData.start({ elasticsearch: elasticsearchStart, savedObjects: savedObjectsStart, + exposedConfigsToUsage: this.plugins.getExposedPluginConfigsToUsage(), }); this.coreStart = { diff --git a/src/plugins/kibana_usage_collection/README.md b/src/plugins/kibana_usage_collection/README.md index 9ad2bd987e1f4..9e9438b1b5fee 100644 --- a/src/plugins/kibana_usage_collection/README.md +++ b/src/plugins/kibana_usage_collection/README.md @@ -4,6 +4,7 @@ This plugin registers the basic usage collectors from Kibana: - [Application Usage](./server/collectors/application_usage/README.md) - Core Metrics +- [Config Usage](./server/collectors/config_usage/README.md) - CSP configuration - Kibana: Number of Saved Objects per type - Localization data @@ -11,8 +12,3 @@ This plugin registers the basic usage collectors from Kibana: - Ops stats - UI Counts - UI Metrics - - - - - diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md new file mode 100644 index 0000000000000..02807c7060dee --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md @@ -0,0 +1,62 @@ +# Config Usage Collector + +The config usage collector reports non-default kibana configs. + +All non-default configs except booleans and numbers will be reported as `[redacted]` unless otherwise specified via `config.exposeToUsage` in the plugin config descriptor. + +```ts +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginConfigDescriptor } from 'src/core/server'; + +export const configSchema = schema.object({ + usageCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + retryCount: schema.number({ defaultValue: 1 }), + bufferDuration: schema.duration({ defaultValue: '5s' }), + }), + uiCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + debug: schema.boolean({ defaultValue: schema.contextRef('dev') }), + }), + maximumWaitTimeForAllCollectorsInS: schema.number({ + defaultValue: DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S, + }), +}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToUsage: { + uiCounters: true, + usageCounters: { + bufferDuration: true, + }, + maximumWaitTimeForAllCollectorsInS: false, + }, +}; +``` + +In the above example setting `uiCounters: true` in the `exposeToUsage` property marks all configs +under the path `uiCounters` as safe. The collector will send the actual non-default config value +when setting an exact config or its parent path to `true`. + +Settings the config path or its parent path to `false` will explicitly mark this config as unsafe. +The collector will send `[redacted]` for non-default configs +when setting an exact config or its parent path to `false`. + +Example output of the collector + +```json +{ + "kibana_config_usage": { + "xpack.apm.serviceMapTraceIdBucketSize": 30, + "elasticsearch.username": "[redacted]", + "elasticsearch.password": "[redacted]", + "plugins.paths": "[redacted]", + "server.port": 5603, + "server.basePath": "[redacted]", + "server.rewriteBasePath": true, + "logging.json": false, + "usageCollection.uiCounters.debug": true + } +} +``` \ No newline at end of file diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts new file mode 100644 index 0000000000000..5d37cfe5957ab --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerConfigUsageCollector } from './register_config_usage_collector'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts new file mode 100644 index 0000000000000..e8f0fae42980b --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { UsageCollectionSetup } from '../../../../usage_collection/server'; +import { ConfigUsageData, CoreUsageDataStart } from '../../../../../core/server'; + +export function registerConfigUsageCollector( + usageCollection: UsageCollectionSetup, + getCoreUsageDataService: () => CoreUsageDataStart +) { + + const collector = usageCollection.makeUsageCollector({ + type: 'kibana_config_usage', + isReady: () => typeof getCoreUsageDataService() !== 'undefined', + schema: { + // DYNAMIC_KEY: { + // type: 'text', + // _meta: { + // description: + // 'Translation file hash. If the hash is different it indicates that a custom translation file is used', + // }, + // }, + }, + fetch: async () => { + const coreUsageDataService = getCoreUsageDataService() + if (!coreUsageDataService) { + return; + } + + return await coreUsageDataService.getConfigsUsageData(); + } + }); + + usageCollection.registerCollector(collector); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts similarity index 100% rename from src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts rename to src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 10156b51ac183..8667e6df8829a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -14,6 +14,7 @@ export { registerOpsStatsCollector } from './ops_stats'; export { registerCspCollector } from './csp'; export { registerCoreUsageCollector } from './core'; export { registerLocalizationUsageCollector } from './localization'; +export { registerConfigUsageCollector } from './config_usage'; export { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index 5b903489e3ff3..c0e04f0b9c966 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -34,6 +34,7 @@ import { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, registerUiCountersRollups, + registerConfigUsageCollector, } from './collectors'; interface KibanaUsageCollectionPluginsDepsSetup { @@ -104,6 +105,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { ); registerCspCollector(usageCollection, coreSetup.http); registerCoreUsageCollector(usageCollection, getCoreUsageDataService); + registerConfigUsageCollector(usageCollection, getCoreUsageDataService); registerLocalizationUsageCollector(usageCollection, coreSetup.i18n); } } From 50d8ef0f555dbcefbbf3c815653547f0a166ec23 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 13:32:04 +0300 Subject: [PATCH 02/11] add more tests --- .../core_usage_data_service.mock.ts | 1 + .../server/plugins/plugins_service.mock.ts | 1 + .../server/plugins/plugins_service.test.ts | 100 +++++++++++++++++- .../register_config_usage_collector.test.ts | 44 ++++++++ .../core/core_usage_collector.test.ts | 6 +- 5 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts index 8ed627cebec7e..b2c150cc6cd2d 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.mock.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -132,6 +132,7 @@ const createStartContractMock = () => { }, }) ), + getConfigsUsageData: jest.fn(), }; return startContract; diff --git a/src/core/server/plugins/plugins_service.mock.ts b/src/core/server/plugins/plugins_service.mock.ts index 1d0ed7cb09299..f4f2263a1bdb0 100644 --- a/src/core/server/plugins/plugins_service.mock.ts +++ b/src/core/server/plugins/plugins_service.mock.ts @@ -19,6 +19,7 @@ const createStartContractMock = () => ({ contracts: new Map() }); const createServiceMock = (): PluginsServiceMock => ({ discover: jest.fn(), + getExposedPluginConfigsToUsage: jest.fn(), setup: jest.fn().mockResolvedValue(createSetupContractMock()), start: jest.fn().mockResolvedValue(createStartContractMock()), stop: jest.fn(), diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts index 6bf7a1fadb4d3..da76a3df300a0 100644 --- a/src/core/server/plugins/plugins_service.test.ts +++ b/src/core/server/plugins/plugins_service.test.ts @@ -78,7 +78,7 @@ const createPlugin = ( manifest: { id, version, - configPath: `${configPath}${disabled ? '-disabled' : ''}`, + configPath: disabled ? configPath.concat('-disabled') : configPath, kibanaVersion, requiredPlugins, requiredBundles, @@ -374,7 +374,6 @@ describe('PluginsService', () => { expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin); expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin); - expect(mockDiscover).toHaveBeenCalledTimes(1); expect(mockDiscover).toHaveBeenCalledWith( { @@ -472,6 +471,90 @@ describe('PluginsService', () => { expect(pluginPaths).toEqual(['/plugin-A-path', '/plugin-B-path']); }); + + it('ppopulates pluginConfigUsageDescriptors with plugins exposeToUsage property', async () => { + const pluginA = createPlugin('plugin-with-expose-usage', { + path: 'plugin-with-expose-usage', + configPath: 'pathA', + }); + + jest.doMock( + join('plugin-with-expose-usage', 'server'), + () => ({ + config: { + exposeToUsage: { + test: true, + nested: { + prop: true, + }, + }, + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + const pluginB = createPlugin('plugin-with-array-configPath', { + path: 'plugin-with-array-configPath', + configPath: ['plugin', 'pathB'], + }); + + jest.doMock( + join('plugin-with-array-configPath', 'server'), + () => ({ + config: { + exposeToUsage: { + test: true, + }, + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + jest.doMock( + join('plugin-without-expose', 'server'), + () => ({ + config: { + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + const pluginC = createPlugin('plugin-without-expose', { + path: 'plugin-without-expose', + configPath: 'pathC', + }); + + mockDiscover.mockReturnValue({ + error$: from([]), + plugin$: from([pluginA, pluginB, pluginC]), + }); + + const mockPluginConfigUsageDescriptors = new Map(); + // @ts-ignore + pluginsService['pluginConfigUsageDescriptors'] = mockPluginConfigUsageDescriptors; + await pluginsService.discover({ environment: environmentSetup }); + + expect(mockPluginConfigUsageDescriptors).toMatchInlineSnapshot(` + Map { + "pathA" => Object { + "nested.prop": true, + "test": true, + }, + "plugin.pathB" => Object { + "test": true, + }, + } + `); + }); }); describe('#generateUiPluginsConfigs()', () => { @@ -624,6 +707,19 @@ describe('PluginsService', () => { }); }); + describe('#getExposedPluginConfigsToUsage', () => { + it('returns pluginConfigUsageDescriptors', () => { + expect(pluginsService.getExposedPluginConfigsToUsage()).toEqual(new Map()); + + const mockPluginConfigUsageDescriptors = Symbol('test'); + // @ts-ignore + pluginsService['pluginConfigUsageDescriptors'] = mockPluginConfigUsageDescriptors; + expect(pluginsService.getExposedPluginConfigsToUsage()).toEqual( + mockPluginConfigUsageDescriptors + ); + }); + }); + describe('#stop()', () => { it('`stop` stops plugins system', async () => { await pluginsService.stop(); diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts new file mode 100644 index 0000000000000..7d4f03fd30edf --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + Collector, + createUsageCollectionSetupMock, + createCollectorFetchContextMock, +} from '../../../../usage_collection/server/mocks'; +import { registerConfigUsageCollector } from './register_config_usage_collector'; +import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; +import type { ConfigUsageData } from '../../../../../core/server'; + +const logger = loggingSystemMock.createLogger(); + +describe('kibana_config_usage', () => { + let collector: Collector; + + const usageCollectionMock = createUsageCollectionSetupMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = new Collector(logger, config); + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); + + const collectorFetchContext = createCollectorFetchContextMock(); + const coreUsageDataStart = coreUsageDataServiceMock.createStartContract(); + const mockConfigUsage = (Symbol('config usage telemetry') as any) as ConfigUsageData; + coreUsageDataStart.getConfigsUsageData.mockResolvedValue(mockConfigUsage); + + beforeAll(() => registerConfigUsageCollector(usageCollectionMock, () => coreUsageDataStart)); + + test('registered collector is set', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('kibana_config_usage'); + }); + + test('fetch', async () => { + expect(await collector.fetch(collectorFetchContext)).toEqual(mockConfigUsage); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts index cbc38129fdddf..b671a9f93d369 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts @@ -9,11 +9,11 @@ import { Collector, createUsageCollectionSetupMock, + createCollectorFetchContextMock, } from '../../../../usage_collection/server/mocks'; -import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; -import { registerCoreUsageCollector } from '.'; +import { registerCoreUsageCollector } from './core_usage_collector'; import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; -import { CoreUsageData } from 'src/core/server/'; +import type { CoreUsageData } from '../../../../../core/server'; const logger = loggingSystemMock.createLogger(); From de4c1614ed3d466d5433b69be7a2e91a31f2883b Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 14:30:23 +0300 Subject: [PATCH 03/11] add api_integration tests --- src/core/server/core_usage_data/types.ts | 2 +- .../apis/telemetry/telemetry_local.ts | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 79ce2494b0a23..701edce22bbdc 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -7,7 +7,7 @@ */ import { CoreUsageStatsClient } from './core_usage_stats_client'; -import { ISavedObjectTypeRegistry, SavedObjectTypeRegistry, PluginConfigUsageDescriptors } from '..'; +import { ISavedObjectTypeRegistry, SavedObjectTypeRegistry } from '..'; /** * @internal diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index 9b92576c84b3a..c2132b14742e7 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -15,6 +15,7 @@ import type { SavedObject } from '../../../../src/core/server'; import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; import ossPluginsTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_plugins.json'; import { assertTelemetryPayload, flatKeys } from './utils'; +import { merge } from 'lodash'; async function retrieveTelemetry( supertest: supertestAsPromised.SuperTest @@ -56,7 +57,18 @@ export default function ({ getService }: FtrProviderContext) { it('should pass the schema validation', () => { try { assertTelemetryPayload( - { root: ossRootTelemetrySchema, plugins: ossPluginsTelemetrySchema }, + { + root: ossRootTelemetrySchema, + /** + * set 'kibana_config_usage' type to 'pass_through' + * This collector is excluded in the telemetryrc.json file + */ + plugins: merge(ossPluginsTelemetrySchema, { + properties: { + kibana_config_usage: { type: 'pass_through' } + } + }), + }, stats ); } catch (err) { @@ -86,6 +98,33 @@ export default function ({ getService }: FtrProviderContext) { expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); + expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.be.an('object'); + // non-default kibana configs. Configs set at 'test/api_integration/config.js'. + expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.eql({ + 'elasticsearch.username': '[redacted]', + 'elasticsearch.password': '[redacted]', + 'elasticsearch.hosts': '[redacted]', + 'elasticsearch.healthCheck.delay': 3600000, + 'plugins.paths': '[redacted]', + 'logging.json': false, + 'server.port': 5620, + 'server.xsrf.disableProtection': true, + 'server.compression.referrerWhitelist': '[redacted]', + 'server.maxPayload': 1679958, + 'status.allowAnonymous': true, + 'home.disableWelcomeScreen': true, + 'data.search.aggs.shardDelay.enabled': true, + 'security.showInsecureClusterWarning': false, + 'telemetry.banner': false, + 'telemetry.url': '[redacted]', + 'telemetry.optInStatusUrl': '[redacted]', + 'telemetry.optIn': false, + 'newsfeed.service.urlRoot': '[redacted]', + 'newsfeed.service.pathTemplate': '[redacted]', + 'savedObjects.maxImportPayloadBytes': 10485760, + 'savedObjects.maxImportExportSize': 10001, + 'usageCollection.usageCounters.bufferDuration': 0 + }); // Testing stack_stats.data expect(stats.stack_stats.data).to.be.an('object'); From 115736f657c91953b88edd7f13129e3cb368115e Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 16:26:08 +0300 Subject: [PATCH 04/11] type check + docs --- ...-plugin-core-server.makeusagefromschema.md | 15 ++++++++++ .../core/server/kibana-plugin-core-server.md | 1 + ...er.pluginconfigdescriptor.exposetousage.md | 17 +++++++++++ ...ugin-core-server.pluginconfigdescriptor.md | 1 + .../core_usage_data_service.test.ts | 5 ++-- .../core_usage_data_service.ts | 30 ++++++++++++------- src/core/server/core_usage_data/types.ts | 2 +- src/core/server/index.ts | 1 - .../server/plugins/plugins_service.test.ts | 23 +++++++------- src/core/server/plugins/plugins_service.ts | 9 ++---- src/core/server/plugins/types.ts | 4 +-- src/core/server/server.api.md | 19 +++++++++--- .../register_config_usage_collector.ts | 5 ++-- .../apis/telemetry/telemetry_local.ts | 14 +++++---- 14 files changed, 96 insertions(+), 50 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md diff --git a/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md new file mode 100644 index 0000000000000..af499f9defff7 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) + +## MakeUsageFromSchema type + +List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to `true` then the actual value of these configs will be reoprted. If parent node or actual config path is set to `false` then the config will be reported as \[redacted\]. + +Signature: + +```typescript +export declare type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; +}; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 3bbdf8c703ab1..e33e9472d42a9 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -272,6 +272,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [LegacyElasticsearchClientConfig](./kibana-plugin-core-server.legacyelasticsearchclientconfig.md) | | | [LifecycleResponseFactory](./kibana-plugin-core-server.lifecycleresponsefactory.md) | Creates an object containing redirection or error response with error details, HTTP headers, and other data transmitted to the client. | | [LoggerConfigType](./kibana-plugin-core-server.loggerconfigtype.md) | | +| [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to true then the actual value of these configs will be reoprted. If parent node or actual config path is set to false then the config will be reported as \[redacted\]. | | [MetricsServiceStart](./kibana-plugin-core-server.metricsservicestart.md) | APIs to retrieves metrics gathered and exposed by the core platform. | | [MIGRATION\_ASSISTANCE\_INDEX\_ACTION](./kibana-plugin-core-server.migration_assistance_index_action.md) | | | [MIGRATION\_DEPRECATION\_LEVEL](./kibana-plugin-core-server.migration_deprecation_level.md) | | diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md new file mode 100644 index 0000000000000..8c50c2e339426 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginConfigDescriptor](./kibana-plugin-core-server.pluginconfigdescriptor.md) > [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) + +## PluginConfigDescriptor.exposeToUsage property + +Expose non-default configs to usage collection to be sent via telemetry. set a config to `true` to report the actual changed config value. set a config to `false` to report the changed config value as \[redacted\]. + +All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified. + +[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) + +Signature: + +```typescript +exposeToUsage?: MakeUsageFromSchema; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md index 5708c4f9a3f88..80e807a1361fd 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md @@ -46,5 +46,6 @@ export const config: PluginConfigDescriptor = { | --- | --- | --- | | [deprecations](./kibana-plugin-core-server.pluginconfigdescriptor.deprecations.md) | ConfigDeprecationProvider | Provider for the to apply to the plugin configuration. | | [exposeToBrowser](./kibana-plugin-core-server.pluginconfigdescriptor.exposetobrowser.md) | {
[P in keyof T]?: boolean;
} | List of configuration properties that will be available on the client-side plugin. | +| [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) | MakeUsageFromSchema<T> | Expose non-default configs to usage collection to be sent via telemetry. set a config to true to report the actual changed config value. set a config to false to report the changed config value as \[redacted\].All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | | [schema](./kibana-plugin-core-server.pluginconfigdescriptor.schema.md) | PluginConfigSchema<T> | Schema to use to validate the plugin configuration.[PluginConfigSchema](./kibana-plugin-core-server.pluginconfigschema.md) | diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 9a5c7586e6ca7..fdf269dc7ecc0 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -321,7 +321,6 @@ describe('CoreUsageDataService', () => { elasticsearch, }); - await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { "server.basePath": "[redacted]", @@ -559,8 +558,8 @@ describe('CoreUsageDataService', () => { }); const mockGetMarkedAsSafe = jest.fn().mockReturnValue({}); - // @ts-ignore - service['getMarkedAsSafe'] = mockGetMarkedAsSafe; + // @ts-expect-error + service.getMarkedAsSafe = mockGetMarkedAsSafe; await getConfigsUsageData(); expect(mockGetMarkedAsSafe).toBeCalledTimes(2); diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index 41a3183187905..068fcbfca72c9 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -262,16 +262,18 @@ export class CoreUsageDataService implements CoreService { + const exposeKeyDetails = Object.keys(exposeDetails).find((exposeKey) => { const fullPath = `${pluginId}.${exposeKey}`; const hasIntersection = hasConfigPathIntersection(usedPath, fullPath); @@ -289,7 +291,9 @@ export class CoreUsageDataService implements CoreService { + private async getNonDefaultKibanaConfigs( + exposedConfigsToUsage: ExposedConfigsToUsage + ): Promise { const config = await this.configService.getConfig$().pipe(first()).toPromise(); const nonDefaultConfigs = config.toRaw(); const usedPaths = await this.configService.getUsedPaths(); @@ -297,8 +301,14 @@ export class CoreUsageDataService implements CoreService { const configValue = get(nonDefaultConfigs, usedPath); - const pluginId = exposedConfigsKeys.find(exposedConfigsKey => usedPath.startsWith(exposedConfigsKey)); - const { explicitlyMarked, isSafe } = this.getMarkedAsSafe(exposedConfigsToUsage, pluginId, usedPath); + const pluginId = exposedConfigsKeys.find((exposedConfigsKey) => + usedPath.startsWith(exposedConfigsKey) + ); + const { explicitlyMarked, isSafe } = this.getMarkedAsSafe( + exposedConfigsToUsage, + pluginId, + usedPath + ); // explicitly marked as safe if (explicitlyMarked && isSafe) { @@ -313,7 +323,7 @@ export class CoreUsageDataService implements CoreService { return await this.getNonDefaultKibanaConfigs(exposedConfigsToUsage); - } + }, }; } diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index 701edce22bbdc..ac72ec49d79c8 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -126,7 +126,7 @@ export interface CoreUsageData extends CoreUsageStats { * Type describing Core's usage data payload * @internal */ -export type ConfigUsageData = Record +export type ConfigUsageData = Record; /** * Type describing Core's usage data payload diff --git a/src/core/server/index.ts b/src/core/server/index.ts index ad26ca39506fb..6b7fa994e6a97 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -414,7 +414,6 @@ export type { CoreStatus, ServiceStatus, ServiceStatusLevel, StatusServiceSetup export type { CoreUsageDataStart } from './core_usage_data'; - /** * Plugin specific context passed to a route handler. * diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts index da76a3df300a0..5c50df07dc697 100644 --- a/src/core/server/plugins/plugins_service.test.ts +++ b/src/core/server/plugins/plugins_service.test.ts @@ -538,12 +538,10 @@ describe('PluginsService', () => { plugin$: from([pluginA, pluginB, pluginC]), }); - const mockPluginConfigUsageDescriptors = new Map(); - // @ts-ignore - pluginsService['pluginConfigUsageDescriptors'] = mockPluginConfigUsageDescriptors; await pluginsService.discover({ environment: environmentSetup }); - expect(mockPluginConfigUsageDescriptors).toMatchInlineSnapshot(` + // eslint-disable-next-line dot-notation + expect(pluginsService['pluginConfigUsageDescriptors']).toMatchInlineSnapshot(` Map { "pathA" => Object { "nested.prop": true, @@ -709,14 +707,15 @@ describe('PluginsService', () => { describe('#getExposedPluginConfigsToUsage', () => { it('returns pluginConfigUsageDescriptors', () => { - expect(pluginsService.getExposedPluginConfigsToUsage()).toEqual(new Map()); - - const mockPluginConfigUsageDescriptors = Symbol('test'); - // @ts-ignore - pluginsService['pluginConfigUsageDescriptors'] = mockPluginConfigUsageDescriptors; - expect(pluginsService.getExposedPluginConfigsToUsage()).toEqual( - mockPluginConfigUsageDescriptors - ); + // eslint-disable-next-line dot-notation + pluginsService['pluginConfigUsageDescriptors'].set('test', { enabled: true }); + expect(pluginsService.getExposedPluginConfigsToUsage()).toMatchInlineSnapshot(` + Map { + "test" => Object { + "enabled": true, + }, + } + `); }); }); diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts index 73a431edcf0f0..547fe00fdb1cf 100644 --- a/src/core/server/plugins/plugins_service.ts +++ b/src/core/server/plugins/plugins_service.ts @@ -16,12 +16,7 @@ import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { discover, PluginDiscoveryError, PluginDiscoveryErrorType } from './discovery'; import { PluginWrapper } from './plugin'; -import { - DiscoveredPlugin, - PluginConfigDescriptor, - PluginName, - InternalPluginInfo, -} from './types'; +import { DiscoveredPlugin, PluginConfigDescriptor, PluginName, InternalPluginInfo } from './types'; import { PluginsConfig, PluginsConfigType } from './plugins_config'; import { PluginsSystem } from './plugins_system'; import { InternalCoreSetup, InternalCoreStart } from '../internal_types'; @@ -223,7 +218,7 @@ export class PluginsService implements CoreService { * @public */ export type MakeUsageFromSchema = { - [Key in keyof T]?: T[Key] extends object - ? MakeUsageFromSchema | boolean - : boolean; + [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; }; /** diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index e8f9dab435754..bd5725f1dd51b 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -381,6 +381,9 @@ export { ConfigPath } export { ConfigService } +// @internal +export type ConfigUsageData = Record; + // @public export interface ContextSetup { createContextContainer(): IContextContainer; @@ -551,6 +554,8 @@ export interface CoreUsageData extends CoreUsageStats { // @internal export interface CoreUsageDataStart { + // (undocumented) + getConfigsUsageData(): Promise; getCoreUsageData(): Promise; } @@ -1652,6 +1657,11 @@ export { LogMeta } export { LogRecord } +// @public +export type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; +}; + // @public export interface MetricsServiceSetup { readonly collectionInterval: number; @@ -1838,6 +1848,7 @@ export interface PluginConfigDescriptor { exposeToBrowser?: { [P in keyof T]?: boolean; }; + exposeToUsage?: MakeUsageFromSchema; schema: PluginConfigSchema; } @@ -3224,9 +3235,9 @@ export const validBodyOutput: readonly ["data", "stream"]; // // src/core/server/elasticsearch/client/types.ts:94:7 - (ae-forgotten-export) The symbol "Explanation" needs to be exported by the entry point index.d.ts // src/core/server/http/router/response.ts:297:3 - (ae-forgotten-export) The symbol "KibanaResponse" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:296:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:401:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" +// src/core/server/plugins/types.ts:317:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:317:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:320:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:425:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" ``` diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts index 7f579ffa968af..ad7f570432abf 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts @@ -13,7 +13,6 @@ export function registerConfigUsageCollector( usageCollection: UsageCollectionSetup, getCoreUsageDataService: () => CoreUsageDataStart ) { - const collector = usageCollection.makeUsageCollector({ type: 'kibana_config_usage', isReady: () => typeof getCoreUsageDataService() !== 'undefined', @@ -27,13 +26,13 @@ export function registerConfigUsageCollector( */ schema: {}, fetch: async () => { - const coreUsageDataService = getCoreUsageDataService() + const coreUsageDataService = getCoreUsageDataService(); if (!coreUsageDataService) { return; } return await coreUsageDataService.getConfigsUsageData(); - } + }, }); usageCollection.registerCollector(collector); diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index c2132b14742e7..fcba6d750d2a7 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import supertestAsPromised from 'supertest-as-promised'; +import { merge, omit } from 'lodash'; import { basicUiCounters } from './__fixtures__/ui_counters'; import { basicUsageCounters } from './__fixtures__/usage_counters'; import type { FtrProviderContext } from '../../ftr_provider_context'; @@ -15,7 +16,6 @@ import type { SavedObject } from '../../../../src/core/server'; import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; import ossPluginsTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_plugins.json'; import { assertTelemetryPayload, flatKeys } from './utils'; -import { merge } from 'lodash'; async function retrieveTelemetry( supertest: supertestAsPromised.SuperTest @@ -65,8 +65,8 @@ export default function ({ getService }: FtrProviderContext) { */ plugins: merge(ossPluginsTelemetrySchema, { properties: { - kibana_config_usage: { type: 'pass_through' } - } + kibana_config_usage: { type: 'pass_through' }, + }, }), }, stats @@ -100,14 +100,13 @@ export default function ({ getService }: FtrProviderContext) { expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.be.an('object'); // non-default kibana configs. Configs set at 'test/api_integration/config.js'. - expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.eql({ + expect(omit(stats.stack_stats.kibana.plugins.kibana_config_usage, 'server.port')).to.eql({ 'elasticsearch.username': '[redacted]', 'elasticsearch.password': '[redacted]', 'elasticsearch.hosts': '[redacted]', 'elasticsearch.healthCheck.delay': 3600000, 'plugins.paths': '[redacted]', 'logging.json': false, - 'server.port': 5620, 'server.xsrf.disableProtection': true, 'server.compression.referrerWhitelist': '[redacted]', 'server.maxPayload': 1679958, @@ -123,8 +122,11 @@ export default function ({ getService }: FtrProviderContext) { 'newsfeed.service.pathTemplate': '[redacted]', 'savedObjects.maxImportPayloadBytes': 10485760, 'savedObjects.maxImportExportSize': 10001, - 'usageCollection.usageCounters.bufferDuration': 0 + 'usageCollection.usageCounters.bufferDuration': 0, }); + expect(stats.stack_stats.kibana.plugins.kibana_config_usage['server.port']).to.be.a( + 'number' + ); // Testing stack_stats.data expect(stats.stack_stats.data).to.be.an('object'); From 57d39eb50fc0b1804c27d604138f1134dc98883a Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 16:32:17 +0300 Subject: [PATCH 05/11] update snapshots --- .../server/plugin.test.ts | 98 ++++++++++--------- .../apis/telemetry/telemetry_local.ts | 12 ++- 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/plugin.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts index 86204ed30e656..38206a9ba7388 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -52,53 +52,57 @@ describe('kibana_usage_collection', () => { }) ) ).resolves.toMatchInlineSnapshot(` - Array [ - Object { - "isReady": true, - "type": "ui_counters", - }, - Object { - "isReady": true, - "type": "usage_counters", - }, - Object { - "isReady": false, - "type": "kibana_stats", - }, - Object { - "isReady": true, - "type": "kibana", - }, - Object { - "isReady": false, - "type": "stack_management", - }, - Object { - "isReady": false, - "type": "ui_metric", - }, - Object { - "isReady": false, - "type": "application_usage", - }, - Object { - "isReady": false, - "type": "cloud_provider", - }, - Object { - "isReady": true, - "type": "csp", - }, - Object { - "isReady": false, - "type": "core", - }, - Object { - "isReady": true, - "type": "localization", - }, - ] - `); + Array [ + Object { + "isReady": true, + "type": "ui_counters", + }, + Object { + "isReady": true, + "type": "usage_counters", + }, + Object { + "isReady": false, + "type": "kibana_stats", + }, + Object { + "isReady": true, + "type": "kibana", + }, + Object { + "isReady": false, + "type": "stack_management", + }, + Object { + "isReady": false, + "type": "ui_metric", + }, + Object { + "isReady": false, + "type": "application_usage", + }, + Object { + "isReady": false, + "type": "cloud_provider", + }, + Object { + "isReady": true, + "type": "csp", + }, + Object { + "isReady": false, + "type": "core", + }, + Object { + "isReady": false, + "type": "kibana_config_usage", + }, + Object { + "isReady": true, + "type": "localization", + }, + ] + `); }); test('Runs the start method without issues', () => { diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index a85e8ef82fc8c..ca03bc27d7e47 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -51,8 +51,18 @@ export default function ({ getService }: FtrProviderContext) { }); it('should pass the schema validation', () => { + const excludedFromSchema = { + properties: { + kibana_config_usage: { type: 'pass_through' }, + }, + }; + const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); - const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); + const plugins = deepmerge( + deepmerge(ossPluginsTelemetrySchema, excludedFromSchema), + xpackPluginsTelemetrySchema + ); + try { assertTelemetryPayload({ root, plugins }, stats); } catch (err) { From 545412ff41708a3f6bf9a39c5a47b43866a47bce Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 16:34:07 +0300 Subject: [PATCH 06/11] fix spacing --- .../server/plugin.test.ts | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/plugin.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts index 38206a9ba7388..450c610afc620 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -52,57 +52,57 @@ describe('kibana_usage_collection', () => { }) ) ).resolves.toMatchInlineSnapshot(` - Array [ - Object { - "isReady": true, - "type": "ui_counters", - }, - Object { - "isReady": true, - "type": "usage_counters", - }, - Object { - "isReady": false, - "type": "kibana_stats", - }, - Object { - "isReady": true, - "type": "kibana", - }, - Object { - "isReady": false, - "type": "stack_management", - }, - Object { - "isReady": false, - "type": "ui_metric", - }, - Object { - "isReady": false, - "type": "application_usage", - }, - Object { - "isReady": false, - "type": "cloud_provider", - }, - Object { - "isReady": true, - "type": "csp", - }, - Object { - "isReady": false, - "type": "core", - }, - Object { - "isReady": false, - "type": "kibana_config_usage", - }, - Object { - "isReady": true, - "type": "localization", - }, - ] - `); + Array [ + Object { + "isReady": true, + "type": "ui_counters", + }, + Object { + "isReady": true, + "type": "usage_counters", + }, + Object { + "isReady": false, + "type": "kibana_stats", + }, + Object { + "isReady": true, + "type": "kibana", + }, + Object { + "isReady": false, + "type": "stack_management", + }, + Object { + "isReady": false, + "type": "ui_metric", + }, + Object { + "isReady": false, + "type": "application_usage", + }, + Object { + "isReady": false, + "type": "cloud_provider", + }, + Object { + "isReady": true, + "type": "csp", + }, + Object { + "isReady": false, + "type": "core", + }, + Object { + "isReady": false, + "type": "kibana_config_usage", + }, + Object { + "isReady": true, + "type": "localization", + }, + ] + `); }); test('Runs the start method without issues', () => { From bca8b7e57be9bb6e6c43e495626916d339f5f742 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 16 Apr 2021 19:22:01 +0300 Subject: [PATCH 07/11] fix x-pack tests --- .../test/api_integration/apis/telemetry/telemetry.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.ts b/x-pack/test/api_integration/apis/telemetry/telemetry.ts index fdf55fd6f4670..1cd9e5634d267 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.ts @@ -106,6 +106,12 @@ export default function ({ getService }: FtrProviderContext) { after(() => esArchiver.unload(archive)); it('should pass the schema validations', () => { + const excludedFromSchema = { + properties: { + kibana_config_usage: { type: 'pass_through' }, + }, + }; + const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); // Merging root to monitoring because `kibana` may be passed in some cases for old collection methods reporting to a newer monitoring cluster @@ -114,7 +120,10 @@ export default function ({ getService }: FtrProviderContext) { // It's nested because of the way it's collected and declared monitoringRootTelemetrySchema.properties.monitoringTelemetry.properties.stats.items ); - const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); + const plugins = deepmerge( + deepmerge(ossPluginsTelemetrySchema, excludedFromSchema), + xpackPluginsTelemetrySchema + ); try { assertTelemetryPayload({ root, plugins }, localXPack); monitoring.forEach((stats) => { From 9fd517f6a56689cd89f79ac3cfcfac3b2622bdb3 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 20 Apr 2021 13:23:16 +0300 Subject: [PATCH 08/11] core review fixes --- .../core_usage_data_service.test.ts | 327 +++++++++++++----- .../core_usage_data_service.ts | 77 +++-- src/core/server/plugins/types.ts | 11 +- .../server/collectors/config_usage/README.md | 6 +- src/plugins/telemetry/schema/oss_root.json | 4 +- src/plugins/usage_collection/server/config.ts | 5 + .../apis/telemetry/telemetry_local.ts | 13 +- .../utils/schema_to_config_schema.ts | 9 +- .../apis/telemetry/telemetry.ts | 11 +- .../apis/telemetry/telemetry_local.ts | 11 +- 10 files changed, 324 insertions(+), 150 deletions(-) diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index 0adba21fe4a2f..dc74b65c8dcfc 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -36,16 +36,27 @@ describe('CoreUsageDataService', () => { let service: CoreUsageDataService; const mockConfig = { - ui_metric: {}, + unused_config: {}, elasticsearch: { username: 'kibana_system', password: 'changeme' }, - plugins: { paths: ['some_path', 'another_path'] }, + plugins: { paths: ['pluginA', 'pluginAB', 'pluginB'] }, server: { port: 5603, basePath: '/zvt', rewriteBasePath: true }, logging: { json: false }, - usageCollection: { - uiCounters: { + pluginA: { + enabled: true, + objectConfig: { debug: true, username: 'some_user', }, + arrayOfNumbers: [1, 2, 3], + }, + pluginAB: { + enabled: false, + }, + pluginB: { + arrayOfObjects: [ + { propA: 'a', propB: 'b' }, + { propA: 'a2', propB: 'b2' }, + ], }, }; @@ -308,17 +319,135 @@ describe('CoreUsageDataService', () => { exposedConfigsToUsage = new Map(); }); + it('loops over all used configs once each', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'logging.json', + ]); + + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, + }); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + const mockGetMarkedAsSafe = jest.fn().mockReturnValue({}); + // @ts-expect-error + service.getMarkedAsSafe = mockGetMarkedAsSafe; + await getConfigsUsageData(); + + expect(mockGetMarkedAsSafe).toBeCalledTimes(2); + expect(mockGetMarkedAsSafe.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Map { + "pluginA" => Object { + "objectConfig": true, + }, + }, + "pluginA.objectConfig.debug", + "pluginA", + ], + Array [ + Map { + "pluginA" => Object { + "objectConfig": true, + }, + }, + "logging.json", + undefined, + ], + ] + `); + }); + + it('plucks pluginId from config path correctly', async () => { + exposedConfigsToUsage.set('pluginA', { + enabled: false, + }); + exposedConfigsToUsage.set('pluginAB', { + enabled: false, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginA.enabled', 'pluginAB.enabled']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.enabled": "[redacted]", + "pluginAB.enabled": "[redacted]", + } + `); + }); + + it('returns an object of plugin config usage', async () => { + exposedConfigsToUsage.set('unused_config', { never_reported: true }); + exposedConfigsToUsage.set('server', { basePath: true }); + exposedConfigsToUsage.set('pluginA', { elasticsearch: false }); + exposedConfigsToUsage.set('plugins', { paths: false }); + exposedConfigsToUsage.set('pluginA', { arrayOfNumbers: false }); + + configService.getUsedPaths.mockResolvedValue([ + 'elasticsearch.username', + 'elasticsearch.password', + 'plugins.paths', + 'server.port', + 'server.basePath', + 'server.rewriteBasePath', + 'logging.json', + 'pluginA.enabled', + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', + 'pluginA.arrayOfNumbers', + 'pluginAB.enabled', + 'pluginB.arrayOfObjects', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "elasticsearch.password": "[redacted]", + "elasticsearch.username": "[redacted]", + "logging.json": false, + "pluginA.arrayOfNumbers": "[redacted]", + "pluginA.enabled": true, + "pluginA.objectConfig.debug": true, + "pluginA.objectConfig.username": "[redacted]", + "pluginAB.enabled": false, + "pluginB.arrayOfObjects": "[redacted]", + "plugins.paths": "[redacted]", + "server.basePath": "/zvt", + "server.port": 5603, + "server.rewriteBasePath": true, + } + `); + }); + describe('config explicitly exposed to usage', () => { it('returns [redacted] on unsafe complete match', async () => { - exposedConfigsToUsage.set('usageCollection', { - 'uiCounters.debug': false, + exposedConfigsToUsage.set('pluginA', { + 'objectConfig.debug': false, }); exposedConfigsToUsage.set('server', { basePath: false, }); configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', + 'pluginA.objectConfig.debug', 'server.basePath', ]); @@ -330,8 +459,8 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { + "pluginA.objectConfig.debug": "[redacted]", "server.basePath": "[redacted]", - "usageCollection.uiCounters.debug": "[redacted]", } `); }); @@ -357,13 +486,13 @@ describe('CoreUsageDataService', () => { }); it('returns [redacted] on unsafe parent match', async () => { - exposedConfigsToUsage.set('usageCollection', { - uiCounters: false, + exposedConfigsToUsage.set('pluginA', { + objectConfig: false, }); configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', - 'usageCollection.uiCounters.username', + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', ]); const { getConfigsUsageData } = service.start({ @@ -374,20 +503,20 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { - "usageCollection.uiCounters.debug": "[redacted]", - "usageCollection.uiCounters.username": "[redacted]", + "pluginA.objectConfig.debug": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", } `); }); it('returns config value on safe parent match', async () => { - exposedConfigsToUsage.set('usageCollection', { - uiCounters: true, + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, }); configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', - 'usageCollection.uiCounters.username', + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', ]); const { getConfigsUsageData } = service.start({ @@ -398,13 +527,57 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { - "usageCollection.uiCounters.debug": true, - "usageCollection.uiCounters.username": "some_user", + "pluginA.objectConfig.debug": true, + "pluginA.objectConfig.username": "some_user", + } + `); + }); + + it('returns [redacted] on explicitly marked as safe array of objects', async () => { + exposedConfigsToUsage.set('pluginB', { + arrayOfObjects: true, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginB.arrayOfObjects": "[redacted]", } `); }); - it('returns [redacted] on implicit arrays', async () => { + it('returns values on explicitly marked as safe array of numbers', async () => { + exposedConfigsToUsage.set('pluginA', { + arrayOfNumbers: true, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.arrayOfNumbers": Array [ + 1, + 2, + 3, + ], + } + `); + }); + + it('returns values on explicitly marked as safe array of strings', async () => { exposedConfigsToUsage.set('plugins', { paths: true, }); @@ -420,8 +593,9 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { "plugins.paths": Array [ - "some_path", - "another_path", + "pluginA", + "pluginAB", + "pluginB", ], } `); @@ -430,13 +604,13 @@ describe('CoreUsageDataService', () => { describe('config not explicitly exposed to usage', () => { it('returns [redacted] for string configs', async () => { - exposedConfigsToUsage.set('usageCollection', { - uiCounters: false, + exposedConfigsToUsage.set('pluginA', { + objectConfig: false, }); configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', - 'usageCollection.uiCounters.username', + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', ]); const { getConfigsUsageData } = service.start({ @@ -447,8 +621,8 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { - "usageCollection.uiCounters.debug": "[redacted]", - "usageCollection.uiCounters.username": "[redacted]", + "pluginA.objectConfig.debug": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", } `); }); @@ -457,7 +631,7 @@ describe('CoreUsageDataService', () => { configService.getUsedPaths.mockResolvedValue([ 'elasticsearch.password', 'elasticsearch.username', - 'usageCollection.uiCounters.username', + 'pluginA.objectConfig.username', ]); const { getConfigsUsageData } = service.start({ @@ -470,12 +644,47 @@ describe('CoreUsageDataService', () => { Object { "elasticsearch.password": "[redacted]", "elasticsearch.username": "[redacted]", - "usageCollection.uiCounters.username": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", } `); }); - it('returns [redacted] on implicit arrays', async () => { + it('returns [redacted] on implicit array of objects', async () => { + configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginB.arrayOfObjects": "[redacted]", + } + `); + }); + + it('returns values on implicit array of numbers', async () => { + configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.arrayOfNumbers": Array [ + 1, + 2, + 3, + ], + } + `); + }); + it('returns [redacted] on implicit array of strings', async () => { configService.getUsedPaths.mockResolvedValue(['plugins.paths']); const { getConfigsUsageData } = service.start({ @@ -509,7 +718,7 @@ describe('CoreUsageDataService', () => { it('returns config value for booleans', async () => { configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', + 'pluginA.objectConfig.debug', 'logging.json', ]); @@ -522,14 +731,14 @@ describe('CoreUsageDataService', () => { await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` Object { "logging.json": false, - "usageCollection.uiCounters.debug": true, + "pluginA.objectConfig.debug": true, } `); }); it('ignores exposed to usage configs but not used', async () => { - exposedConfigsToUsage.set('usageCollection', { - uiCounters: true, + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, }); configService.getUsedPaths.mockResolvedValue(['logging.json']); @@ -547,52 +756,6 @@ describe('CoreUsageDataService', () => { `); }); }); - - it('loops over all used configs once each', async () => { - configService.getUsedPaths.mockResolvedValue([ - 'usageCollection.uiCounters.debug', - 'logging.json', - ]); - - exposedConfigsToUsage.set('usageCollection', { - uiCounters: true, - }); - - const { getConfigsUsageData } = service.start({ - savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), - exposedConfigsToUsage, - elasticsearch, - }); - - const mockGetMarkedAsSafe = jest.fn().mockReturnValue({}); - // @ts-expect-error - service.getMarkedAsSafe = mockGetMarkedAsSafe; - await getConfigsUsageData(); - - expect(mockGetMarkedAsSafe).toBeCalledTimes(2); - expect(mockGetMarkedAsSafe.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - Map { - "usageCollection" => Object { - "uiCounters": true, - }, - }, - "usageCollection", - "usageCollection.uiCounters.debug", - ], - Array [ - Map { - "usageCollection" => Object { - "uiCounters": true, - }, - }, - undefined, - "logging.json", - ], - ] - `); - }); }); }); diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index bf7d79a7d7035..85abdca9ea5dc 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -274,28 +274,26 @@ export class CoreUsageDataService implements CoreService { + const fullPath = `${pluginId}.${exposeKey}`; + return hasConfigPathIntersection(usedPath, fullPath); + }); - const exposeKeyDetails = Object.keys(exposeDetails).find((exposeKey) => { - const fullPath = `${pluginId}.${exposeKey}`; - const hasIntersection = hasConfigPathIntersection(usedPath, fullPath); + if (exposeKeyDetails) { + const explicitlyMarkedAsSafe = exposeDetails[exposeKeyDetails]; - return hasIntersection; - }); - if (!exposeKeyDetails) { - return { explicitlyMarked: false, isSafe: false }; - } - - const isSafe = exposeDetails[exposeKeyDetails]; - if (typeof isSafe === 'boolean') { - return { explicitlyMarked: true, isSafe }; + if (typeof explicitlyMarkedAsSafe === 'boolean') { + return { + explicitlyMarked: true, + isSafe: explicitlyMarkedAsSafe, + }; + } + } } return { explicitlyMarked: false, isSafe: false }; @@ -310,19 +308,31 @@ export class CoreUsageDataService implements CoreService { - const configValue = get(nonDefaultConfigs, usedPath); - const pluginId = exposedConfigsKeys.find((exposedConfigsKey) => - usedPath.startsWith(exposedConfigsKey) + const rawConfigValue = get(nonDefaultConfigs, usedPath); + const pluginId = exposedConfigsKeys.find( + (exposedConfigsKey) => + usedPath === exposedConfigsKey || usedPath.startsWith(`${exposedConfigsKey}.`) ); + const { explicitlyMarked, isSafe } = this.getMarkedAsSafe( exposedConfigsToUsage, - pluginId, - usedPath + usedPath, + pluginId ); // explicitly marked as safe if (explicitlyMarked && isSafe) { - acc[usedPath] = configValue; + // report array of objects as redacted even if explicitly marked as safe. + // TS typings prevent explicitly marking arrays of objects as safe + // this makes sure to report redacted even if TS was bypassed. + if ( + Array.isArray(rawConfigValue) && + rawConfigValue.some((item) => typeof item === 'object') + ) { + acc[usedPath] = '[redacted]'; + } else { + acc[usedPath] = rawConfigValue; + } } // explicitly marked as unsafe @@ -335,14 +345,27 @@ export class CoreUsageDataService implements CoreService typeof item === 'number' || typeof item === 'boolean' + ) + ) { + acc[usedPath] = rawConfigValue; + break; + } + } + } default: { acc[usedPath] = '[redacted]'; } diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index d7de7199e4802..2b5acd8ed5e94 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -18,6 +18,8 @@ import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config'; import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config'; import { CoreSetup, CoreStart } from '..'; +type Maybe = T | undefined; + /** * Dedicated type for plugin configuration schema. * @@ -93,7 +95,14 @@ export interface PluginConfigDescriptor { * @public */ export type MakeUsageFromSchema = { - [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; + [Key in keyof T]?: T[Key] extends Maybe + ? // arrays of objects are always redacted + false + : T[Key] extends Maybe + ? boolean + : T[Key] extends Maybe + ? MakeUsageFromSchema | boolean + : boolean; }; /** diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md index 02807c7060dee..b476244e5082f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md @@ -43,7 +43,7 @@ Settings the config path or its parent path to `false` will explicitly mark this The collector will send `[redacted]` for non-default configs when setting an exact config or its parent path to `false`. -Example output of the collector +### Output of the collector ```json { @@ -59,4 +59,6 @@ Example output of the collector "usageCollection.uiCounters.debug": true } } -``` \ No newline at end of file +``` + +Note that arrays of objects will be reported as `[redacted]` and cannot be explicitly marked as safe. \ No newline at end of file diff --git a/src/plugins/telemetry/schema/oss_root.json b/src/plugins/telemetry/schema/oss_root.json index 658f5ee4e66da..c4dd1096a6e98 100644 --- a/src/plugins/telemetry/schema/oss_root.json +++ b/src/plugins/telemetry/schema/oss_root.json @@ -183,8 +183,8 @@ }, "plugins": { "properties": { - "THIS_WILL_BE_REPLACED_BY_THE_PLUGINS_JSON": { - "type": "text" + "kibana_config_usage": { + "type": "pass_through" } } } diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index cd6f6b9d81396..faf8ce7535e8a 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -38,4 +38,9 @@ export const config: PluginConfigDescriptor = { exposeToBrowser: { uiCounters: true, }, + exposeToUsage: { + usageCounters: { + bufferDuration: true, + }, + }, }; diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index fcba6d750d2a7..a90c4c101abd5 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -57,18 +57,7 @@ export default function ({ getService }: FtrProviderContext) { it('should pass the schema validation', () => { try { assertTelemetryPayload( - { - root: ossRootTelemetrySchema, - /** - * set 'kibana_config_usage' type to 'pass_through' - * This collector is excluded in the telemetryrc.json file - */ - plugins: merge(ossPluginsTelemetrySchema, { - properties: { - kibana_config_usage: { type: 'pass_through' }, - }, - }), - }, + { root: ossRootTelemetrySchema, plugins: ossPluginsTelemetrySchema }, stats ); } catch (err) { diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts index b45930682e3aa..c3973d853ebf7 100644 --- a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts @@ -8,7 +8,7 @@ import type { ObjectType, Type } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; -import { get } from 'lodash'; +import { get, merge } from 'lodash'; import { set } from '@elastic/safer-lodash-set'; import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server'; @@ -125,11 +125,12 @@ export function assertTelemetryPayload( stats: unknown ): void { const fullSchema = telemetrySchema.root; - set( - fullSchema, - 'properties.stack_stats.properties.kibana.properties.plugins', + + merge( + get(fullSchema, 'properties.stack_stats.properties.kibana.properties.plugins'), telemetrySchema.plugins ); + const ossTelemetryValidationSchema = convertSchemaToConfigSchema(fullSchema); // Run @kbn/config-schema validation to the entire payload diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.ts b/x-pack/test/api_integration/apis/telemetry/telemetry.ts index 1cd9e5634d267..fdf55fd6f4670 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.ts @@ -106,12 +106,6 @@ export default function ({ getService }: FtrProviderContext) { after(() => esArchiver.unload(archive)); it('should pass the schema validations', () => { - const excludedFromSchema = { - properties: { - kibana_config_usage: { type: 'pass_through' }, - }, - }; - const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); // Merging root to monitoring because `kibana` may be passed in some cases for old collection methods reporting to a newer monitoring cluster @@ -120,10 +114,7 @@ export default function ({ getService }: FtrProviderContext) { // It's nested because of the way it's collected and declared monitoringRootTelemetrySchema.properties.monitoringTelemetry.properties.stats.items ); - const plugins = deepmerge( - deepmerge(ossPluginsTelemetrySchema, excludedFromSchema), - xpackPluginsTelemetrySchema - ); + const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); try { assertTelemetryPayload({ root, plugins }, localXPack); monitoring.forEach((stats) => { diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index ca03bc27d7e47..2412b91e6ee68 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -51,17 +51,8 @@ export default function ({ getService }: FtrProviderContext) { }); it('should pass the schema validation', () => { - const excludedFromSchema = { - properties: { - kibana_config_usage: { type: 'pass_through' }, - }, - }; - const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); - const plugins = deepmerge( - deepmerge(ossPluginsTelemetrySchema, excludedFromSchema), - xpackPluginsTelemetrySchema - ); + const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); try { assertTelemetryPayload({ root, plugins }, stats); From a324d29fcc45c238f4b48ea6ac8127fdf401ae85 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 20 Apr 2021 13:39:51 +0300 Subject: [PATCH 09/11] update docs --- .../kibana-plugin-core-server.makeusagefromschema.md | 2 +- src/core/server/server.api.md | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md index af499f9defff7..f47d01a2d09e8 100644 --- a/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md +++ b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md @@ -10,6 +10,6 @@ List of configuration values that will be exposed to usage collection. If parent ```typescript export declare type MakeUsageFromSchema = { - [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; + [Key in keyof T]?: T[Key] extends Maybe ? false : T[Key] extends Maybe ? boolean : T[Key] extends Maybe ? MakeUsageFromSchema | boolean : boolean; }; ``` diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index d8e02b483928f..ccff20458f7e6 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1667,9 +1667,11 @@ export { LogMeta } export { LogRecord } +// Warning: (ae-forgotten-export) The symbol "Maybe" needs to be exported by the entry point index.d.ts +// // @public export type MakeUsageFromSchema = { - [Key in keyof T]?: T[Key] extends object ? MakeUsageFromSchema | boolean : boolean; + [Key in keyof T]?: T[Key] extends Maybe ? false : T[Key] extends Maybe ? boolean : T[Key] extends Maybe ? MakeUsageFromSchema | boolean : boolean; }; // @public @@ -3245,9 +3247,9 @@ export const validBodyOutput: readonly ["data", "stream"]; // // src/core/server/elasticsearch/client/types.ts:94:7 - (ae-forgotten-export) The symbol "Explanation" needs to be exported by the entry point index.d.ts // src/core/server/http/router/response.ts:297:3 - (ae-forgotten-export) The symbol "KibanaResponse" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:317:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:317:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:320:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:425:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" +// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:329:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:434:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" ``` From 46e3c4320b43090b228c81c9b29bccc066e0ccdb Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 20 Apr 2021 14:49:15 +0300 Subject: [PATCH 10/11] type check --- test/api_integration/apis/telemetry/telemetry_local.ts | 2 +- .../apis/telemetry/utils/schema_to_config_schema.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index a90c4c101abd5..c14fc658f2768 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import supertestAsPromised from 'supertest-as-promised'; -import { merge, omit } from 'lodash'; +import { omit } from 'lodash'; import { basicUiCounters } from './__fixtures__/ui_counters'; import { basicUsageCounters } from './__fixtures__/usage_counters'; import type { FtrProviderContext } from '../../ftr_provider_context'; diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts index c3973d853ebf7..468ed630cd4d1 100644 --- a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts @@ -9,7 +9,6 @@ import type { ObjectType, Type } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { get, merge } from 'lodash'; -import { set } from '@elastic/safer-lodash-set'; import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server'; /** From c3a3e118062ec23353edf3be91ee29dd5247700c Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 20 Apr 2021 16:05:26 +0300 Subject: [PATCH 11/11] fix integration tests unit tests :sweat_smile: --- .../apis/telemetry/utils/schema_to_config_schema.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts index 468ed630cd4d1..ec44cec39c29a 100644 --- a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts @@ -8,6 +8,7 @@ import type { ObjectType, Type } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; +import { set } from '@elastic/safer-lodash-set'; import { get, merge } from 'lodash'; import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server'; @@ -125,11 +126,18 @@ export function assertTelemetryPayload( ): void { const fullSchema = telemetrySchema.root; - merge( + const mergedPluginsSchema = merge( + {}, get(fullSchema, 'properties.stack_stats.properties.kibana.properties.plugins'), telemetrySchema.plugins ); + set( + fullSchema, + 'properties.stack_stats.properties.kibana.properties.plugins', + mergedPluginsSchema + ); + const ossTelemetryValidationSchema = convertSchemaToConfigSchema(fullSchema); // Run @kbn/config-schema validation to the entire payload