[OTel Metrics] Add compatibility with monitoring_collection

.github/CODEOWNERS

@@ -384,7 +384,7 @@ src/platform/packages/private/kbn-ui-shared-deps-npm @elastic/kibana-operations
 src/platform/packages/private/kbn-ui-shared-deps-src @elastic/kibana-operations
 src/platform/packages/private/kbn-unsaved-changes-badge @elastic/kibana-data-discovery
 src/platform/packages/private/kbn-validate-oas @elastic/kibana-core
-src/platform/packages/private/opentelemetry/kbn-metrics @elastic/kibana-core
+src/platform/packages/private/opentelemetry/kbn-metrics @elastic/kibana-core @elastic/stack-monitoring
 src/platform/packages/private/opentelemetry/kbn-metrics-config @elastic/kibana-core
 src/platform/packages/private/shared-ux/storybook/config @elastic/appex-sharedux
 src/platform/packages/shared/chart-expressions-common @elastic/kibana-visualizations

src/platform/packages/private/kbn-apm-config-loader/src/config.ts

@@ -16,6 +16,7 @@ import type { AgentConfigOptions } from 'elastic-apm-node';
 import type { AgentConfigOptions as RUMAgentConfigOptions } from '@elastic/apm-rum';
 import { getFlattenedObject } from '@kbn/std';
 import { type TelemetryConfig, telemetryConfigSchema } from '@kbn/telemetry-config';
+import { type MonitoringCollectionConfig, monitoringCollectionSchema } from '@kbn/metrics-config';
 import type { ApmConfigSchema } from './apm_config';
 
 // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html
@@ -53,7 +54,8 @@ const CENTRALIZED_SERVICE_DIST_CONFIG: AgentConfigOptions = {
 };
 
 interface KibanaRawConfig {
-  telemetry?: TelemetryConfig;
+  monitoring_collection?: Partial<MonitoringCollectionConfig>;
+  telemetry?: Partial<TelemetryConfig>;
   elastic?: {
     apm?: ApmConfigSchema;
   };
@@ -103,6 +105,10 @@ export class ApmConfiguration {
     return telemetryConfigSchema.validate({ enabled, metrics, tracing });
   }
 
+  public getMonitoringCollectionConfig(): MonitoringCollectionConfig {
+    return monitoringCollectionSchema.validate(this.rawKibanaConfig.monitoring_collection);
+  }
+
   public isUsersRedactionEnabled(): boolean {
     const { redactUsers = true } = this.getConfigFromKibanaConfig();
     return redactUsers;

src/platform/packages/private/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts

@@ -45,4 +45,11 @@ describe('applyConfigOverrides', () => {
     expect(config.server.uuid).toEqual('from-argv');
     expect(config.path.data).toEqual('/data-path');
   });
+
+  it('overrides the `telemetry.metrics.enabled` when provided as a command line argument', () => {
+    const config: Record<string, any> = {};
+    const argv = ['--telemetry.metrics.enabled'];
+    applyConfigOverrides(config, argv);
+    expect(config.telemetry.metrics.enabled).toEqual('true');
+  });
 });

src/platform/packages/private/kbn-apm-config-loader/src/utils/apply_config_overrides.ts

@@ -8,13 +8,18 @@
  */
 
 import { set } from '@kbn/safer-lodash-set';
-import { getArgValue } from './read_argv';
+import { getArgValue, getAllArgKeysValueWithPrefix } from './read_argv';
+
+const CONFIG_PREFIXES = ['--elastic.apm', '--telemetry', '--monitoring_collection'];
 
 /**
  * Manually applies the specific configuration overrides we need to load the APM config.
  * Currently, only these are needed:
  *   - server.uuid
  *   - path.data
+ *   - elastic.apm.*
+ *   - telemetry.*
+ *   - monitoring_collection.*
  */
 export const applyConfigOverrides = (config: Record<string, any>, argv: string[]) => {
   const serverUuid = getArgValue(argv, '--server.uuid');
@@ -25,4 +30,13 @@ export const applyConfigOverrides = (config: Record<string, any>, argv: string[]
   if (dataPath) {
     set(config, 'path.data', dataPath);
   }
+
+  CONFIG_PREFIXES.forEach((prefix) => {
+    getAllArgKeysValueWithPrefix(argv, prefix).forEach(([key, value]) => {
+      if (typeof value === 'undefined') {
+        value = 'true'; // Add support to boolean flags without values (i.e.: --telemetry.enabled)
+      }
+      set(config, key, value);
+    });
+  });
 };

src/platform/packages/private/kbn-apm-config-loader/src/utils/read_argv.test.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { getArgValue, getArgValues } from './read_argv';
+import { getAllArgKeysValueWithPrefix, getArgValue, getArgValues } from './read_argv';
 
 describe('getArgValues', () => {
   it('retrieve the arg value from the provided argv arguments', () => {
@@ -41,6 +41,22 @@ describe('getArgValues', () => {
     );
     expect(argValues).toEqual(['my-config']);
   });
+
+  it('ignores the flag when no value is provided (even if followed by another flag)', () => {
+    const argValues = getArgValues(
+      ['--config', '-c', 'my-config', '--foo', '-b', 'bar'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual(['my-config']);
+  });
+
+  it('supports the format --foo=bar', () => {
+    const argValues = getArgValues(
+      ['--config', '-c', 'my-config', '--foo=bar', '-b', 'bar'],
+      ['--foo']
+    );
+    expect(argValues).toEqual(['bar']);
+  });
 });
 
 describe('getArgValue', () => {
@@ -68,3 +84,41 @@ describe('getArgValue', () => {
     expect(argValues).toBeUndefined();
   });
 });
+
+describe('getAllArgKeysValueWithPrefix', () => {
+  it('returns all the keys exact-matching the provided prefix', () => {
+    const argv = [
+      '--config',
+      'my-config',
+      '--foo',
+      '-b',
+      'bar',
+      '--baz',
+      '--unicorn',
+      '{"name": "Bob", "madeOf": "chocolate"}',
+    ];
+
+    expect(getAllArgKeysValueWithPrefix(argv, '--unicorn')).toEqual([
+      ['unicorn', '{"name": "Bob", "madeOf": "chocolate"}'],
+    ]);
+  });
+
+  it('returns all the sub-keys matching the provided prefix', () => {
+    const argv = [
+      '--config',
+      'my-config',
+      '--foo',
+      '-b',
+      'bar',
+      '--baz',
+      '--unicorn.name',
+      'Bob',
+      '--unicorn.madeOf=chocolate',
+    ];
+
+    expect(getAllArgKeysValueWithPrefix(argv, '--unicorn')).toEqual([
+      ['unicorn.name', 'Bob'],
+      ['unicorn.madeOf', 'chocolate'],
+    ]);
+  });
+});

src/platform/packages/private/kbn-apm-config-loader/src/utils/read_argv.ts

@@ -7,12 +7,37 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+/**
+ * Convert flags in format `--flag=value` to `--flag value`
+ * @param argv List of arguments
+ * @returns Array of arguments with flags in `--flag=value` format normalized to `--flag value`
+ */
+const normalizeArgFormat = (argv: string[]): string[] => {
+  return argv.flatMap((arg) => {
+    if (/^--\w+/.test(arg) && arg.includes('=')) {
+      const [key, ...valueParts] = arg.split('=');
+      return [key, valueParts.join('=')];
+    }
+    return [arg];
+  });
+};
+
 export const getArgValues = (argv: string[], flag: string | string[]): string[] => {
   const flags = typeof flag === 'string' ? [flag] : flag;
+
+  const normalizedArgv = normalizeArgFormat(argv);
+
   const values: string[] = [];
-  for (let i = 0; i < argv.length; i++) {
-    if (flags.includes(argv[i]) && argv[i + 1]) {
-      values.push(argv[++i]);
+  for (let i = 0; i < normalizedArgv.length; i++) {
+    if (flags.includes(normalizedArgv[i])) {
+      if (
+        normalizedArgv[i + 1] &&
+        // Take the next argument as the value if it's not another flag
+        !/^--\w+/.test(normalizedArgv[i + 1]) && // In the --flag format
+        !/^-\w$/.test(normalizedArgv[i + 1]) // In the -f format
+      ) {
+        values.push(normalizedArgv[++i]);
+      }
     }
   }
   return values;
@@ -24,3 +49,26 @@ export const getArgValue = (argv: string[], flag: string | string[]): string | u
     return values[0];
   }
 };
+
+/**
+ * Get all flags matching the provided prefix
+ * @param argv List of arguments
+ * @param flagPrefix Flag prefix to match (either identical or its subkeys)
+ * @returns Array of [flag, value] pairs for the matching flags. The returned flags are cleaned up from the `--` prefix.
+ */
+export const getAllArgKeysValueWithPrefix = (
+  argv: string[],
+  flagPrefix: string
+): Array<[string, string | undefined]> => {
+  const flags = normalizeArgFormat(argv).filter(
+    (arg) =>
+      typeof arg === 'string' &&
+      arg.startsWith('--') &&
+      (arg === flagPrefix || // Exact match
+        arg.startsWith(`${flagPrefix}.`)) // Subkey match
+  );
+  return flags.map((flag) => [
+    flag.slice(2), // remove the leading '--'
+    getArgValue(argv, flag),
+  ]);
+};

src/platform/packages/private/kbn-apm-config-loader/tsconfig.json

@@ -16,6 +16,7 @@
     "@kbn/config-schema",
     "@kbn/std",
     "@kbn/telemetry-config",
+    "@kbn/metrics-config",
   ],
   "exclude": [
     "target/**/*",

src/platform/packages/private/opentelemetry/kbn-metrics-config/index.ts

@@ -8,4 +8,8 @@
  */
 
 export type { MetricsExporterConfig, MetricsConfig } from './src/types';
-export { metricsConfigSchema } from './src/schema';
+export { metricsConfigSchema } from './src/metrics_schema';
+export {
+  monitoringCollectionSchema,
+  type MonitoringCollectionConfig,
+} from './src/monitoring_collection_schema';

src/platform/packages/private/opentelemetry/kbn-metrics-config/src/schema.ts → src/platform/packages/private/opentelemetry/kbn-metrics-config/src/metrics_schema.ts

@@ -16,6 +16,7 @@ export const metricsExporterConfigSchema: Type<MetricsExporterConfig> = schema.o
     grpc: schema.object({
       url: schema.string(),
       headers: schema.maybe(schema.recordOf(schema.string(), schema.string())),
+      exportIntervalMillis: schema.maybe(schema.duration()),
     }),
   }),
 
@@ -24,6 +25,7 @@ export const metricsExporterConfigSchema: Type<MetricsExporterConfig> = schema.o
     http: schema.object({
       url: schema.string(),
       headers: schema.maybe(schema.recordOf(schema.string(), schema.string())),
+      exportIntervalMillis: schema.maybe(schema.duration()),
     }),
   }),
 ]);

src/platform/packages/private/opentelemetry/kbn-metrics-config/src/monitoring_collection_schema.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { type TypeOf, schema } from '@kbn/config-schema';
+
+/**
+ * Configuration of the plugin `monitoring_collection`
+ */
+export type MonitoringCollectionConfig = TypeOf<typeof monitoringCollectionSchema>;
+
+/**
+ * Config schema of the plugin `monitoring_collection`.
+ * @privateRemarks It needs to be defined here because it declares the configuration of some Metric Exporters,
+ * and importing the config from the plugin would create a circular dependency.
+ */
+export const monitoringCollectionSchema = schema.object({
+  enabled: schema.boolean({ defaultValue: true }),
+  opentelemetry: schema.object({
+    metrics: schema.object({
+      otlp: schema.object({
+        url: schema.maybe(schema.string()),
+        headers: schema.maybe(schema.recordOf(schema.string(), schema.string())),
+        exportIntervalMillis: schema.number({ defaultValue: 10000 }),
+        logLevel: schema.string({ defaultValue: 'info' }),
+      }),
+      prometheus: schema.object({
+        enabled: schema.boolean({ defaultValue: false }),
+      }),
+    }),
+  }),
+});

src/platform/packages/private/opentelemetry/kbn-metrics-config/src/types.ts

@@ -20,6 +20,8 @@ export type MetricsExporterConfig =
         url: string;
         /** HTTP headers to send to the OTLP gRPC endpoint. Typically, the `Authorization` header is one of them */
         headers?: Record<string, string>;
+        /** Frequency in which the exporter should collect the metrics */
+        exportIntervalMillis?: number | Duration;
       };
     }
   | {
@@ -29,6 +31,8 @@ export type MetricsExporterConfig =
         url: string;
         /** HTTP headers to send to the OTLP HTTP endpoint. Typically, the `Authorization` header is one of them */
         headers?: Record<string, string>;
+        /** Frequency in which the exporter should collect the metrics */
+        exportIntervalMillis?: number | Duration;
       };
     };
 

src/platform/packages/private/opentelemetry/kbn-metrics/README.md

@@ -19,6 +19,10 @@ telemetry.metrics:
           authorization: "ApiKey [REDACTED]"
 ```
 
+#### Special case: Monitoring Collection
+
+This package also takes care of initializing the metric exporters defined in the Monitoring Collection plugin. Refer to the [plugin's docs](../../../../../../x-pack/platform/plugins/private/monitoring_collection/README.md) for more info about those exporters.
+
 ### Instrument your code with metrics
 
 Whenever you want to instrument your code with any metric, you can do this with these 4 simple steps:

src/platform/packages/private/opentelemetry/kbn-metrics/index.ts

@@ -8,3 +8,4 @@
  */
 
 export { initMetrics, type InitMetricsOptions } from './src/init_metrics';
+export { PrometheusExporter } from './src/prometheus_exporter';

src/platform/packages/private/opentelemetry/kbn-metrics/kibana.jsonc

@@ -1,7 +1,10 @@
 {
   "type": "shared-common",
   "id": "@kbn/metrics",
-  "owner": "@elastic/kibana-core",
+  "owner": [
+    "@elastic/kibana-core",
+    "@elastic/stack-monitoring"
+  ],
   "group": "platform",
   "visibility": "private"
 }