Fleet Usage telemetry extension

x-pack/plugins/fleet/server/collectors/agent_collectors.ts

@@ -8,7 +8,9 @@
 import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server';
 
 import type { FleetConfigType } from '../../common/types';
+import { AGENTS_INDEX } from '../../common';
 import * as AgentService from '../services/agents';
+import { appContextService } from '../services';
 
 export interface AgentUsage {
   total_enrolled: number;
@@ -47,3 +49,103 @@ export const getAgentUsage = async (
     updating,
   };
 };
+
+export interface AgentData {
+  agent_versions: string[];
+  agent_checkin_status: {
+    error: number;
+    degraded: number;
+  };
+  agent_checkin_status_last_1h: {
+    error: number;
+    degraded: number;
+  };
+}
+
+const DEFAULT_AGENT_DATA = {
+  agent_versions: [],
+  agent_checkin_status: { error: 0, degraded: 0 },
+  agent_checkin_status_last_1h: { error: 0, degraded: 0 },
+};
+
+export const getAgentData = async (
+  esClient: ElasticsearchClient,
+  abortController: AbortController
+): Promise<AgentData> => {
+  try {
+    const transformLastCheckinStatusBuckets = (resp: any) =>
+      ((resp?.aggregations?.last_checkin_status as any).buckets ?? []).reduce(
+        (acc: any, bucket: any) => {
+          if (acc[bucket.key] !== undefined) acc[bucket.key] = bucket.doc_count;
+          return acc;
+        },
+        { error: 0, degraded: 0 }
+      );
+    const response = await esClient.search(
+      {
+        index: AGENTS_INDEX,
+        size: 0,
+        aggs: {
+          versions: {
+            terms: { field: 'agent.version' },
+          },
+          last_checkin_status: {
+            terms: { field: 'last_checkin_status' },
+          },
+        },
+      },
+      { signal: abortController.signal }
+    );
+    const versions = ((response?.aggregations?.versions as any).buckets ?? []).map(
+      (bucket: any) => bucket.key
+    );
+    const statuses = transformLastCheckinStatusBuckets(response);
+
+    const responseLast1h = await esClient.search(
+      {
+        index: AGENTS_INDEX,
+        size: 0,
+        query: {
+          bool: {
+            filter: [
+              {
+                bool: {
+                  must: [
+                    {
+                      range: {
+                        last_checkin: {
+                          gte: 'now-1h/h',
+                          lt: 'now/h',
+                        },
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          last_checkin_status: {
+            terms: { field: 'last_checkin_status' },
+          },
+        },
+      },
+      { signal: abortController.signal }
+    );
+    const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h);
+
+    return {
+      agent_versions: versions,
+      agent_checkin_status: statuses,
+      agent_checkin_status_last_1h: statusesLast1h,
+    };
+  } catch (error) {
+    if (error.statusCode === 404) {
+      appContextService.getLogger().debug('Index .fleet-agents does not exist yet.');
+    } else {
+      throw error;
+    }
+    return DEFAULT_AGENT_DATA;
+  }
+};

x-pack/plugins/fleet/server/collectors/agent_policies.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+
+import { AGENT_POLICY_INDEX } from '../../common';
+import { ES_SEARCH_LIMIT } from '../../common/constants';
+import { appContextService } from '../services';
+
+export interface AgentPoliciesUsage {
+  count: number;
+  output_types: string[];
+}
+
+const DEFAULT_AGENT_POLICIES_USAGE = {
+  count: 0,
+  output_types: [],
+};
+
+export const getAgentPoliciesUsage = async (
+  esClient: ElasticsearchClient,
+  abortController: AbortController
+): Promise<AgentPoliciesUsage> => {
+  try {
+    const res = await esClient.search(
+      {
+        index: AGENT_POLICY_INDEX,
+        size: ES_SEARCH_LIMIT,
+        track_total_hits: true,
+        rest_total_hits_as_int: true,
+      },
+      { signal: abortController.signal }
+    );
+
+    const agentPolicies = res.hits.hits;
+
+    const outputTypes = new Set<string>();
+    agentPolicies.forEach((item) => {
+      const source = (item._source as any) ?? {};
+      Object.keys(source.data.outputs).forEach((output) => {
+        outputTypes.add(source.data.outputs[output].type);
+      });
+    });
+
+    return {
+      count: res.hits.total as number,
+      output_types: Array.from(outputTypes),
+    };
+  } catch (error) {
+    if (error.statusCode === 404) {
+      appContextService.getLogger().debug('Index .fleet-policies does not exist yet.');
+    } else {
+      throw error;
+    }
+    return DEFAULT_AGENT_POLICIES_USAGE;
+  }
+};

x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts

@@ -7,6 +7,8 @@
 
 import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server';
 
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../constants';
+
 import { packagePolicyService } from '../services';
 import { getAgentStatusForAgentPolicy } from '../services/agents';
 import { listFleetServerHosts } from '../services/fleet_server_host';
@@ -84,3 +86,25 @@ export const getFleetServerUsage = async (
     num_host_urls: numHostsUrls,
   };
 };
+
+export const getFleetServerConfig = async (soClient: SavedObjectsClient): Promise<any> => {
+  const res = await packagePolicyService.list(soClient, {
+    page: 1,
+    perPage: SO_SEARCH_LIMIT,
+    kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`,
+  });
+  const getInputConfig = (item: any) => {
+    let config = (item.inputs[0] ?? {}).compiled_input;
+    if (config.server) {
+      config = { ...config, server: { ...config.server } };
+      delete config.server.host;
+      delete config.server.port;
+    }
+    return config;
+  };
+  const policies = res.items.map((item) => ({
+    input_config: getInputConfig(item),
+  }));
+
+  return { policies };
+};

x-pack/plugins/fleet/server/collectors/register.ts

@@ -11,13 +11,14 @@ import type { CoreSetup } from '@kbn/core/server';
 import type { FleetConfigType } from '..';
 
 import { getIsAgentsEnabled } from './config_collectors';
-import { getAgentUsage } from './agent_collectors';
+import { getAgentUsage, getAgentData } from './agent_collectors';
 import type { AgentUsage } from './agent_collectors';
 import { getInternalClients } from './helpers';
 import { getPackageUsage } from './package_collectors';
 import type { PackageUsage } from './package_collectors';
-import { getFleetServerUsage } from './fleet_server_collector';
+import { getFleetServerUsage, getFleetServerConfig } from './fleet_server_collector';
 import type { FleetServerUsage } from './fleet_server_collector';
+import { getAgentPoliciesUsage } from './agent_policies';
 
 export interface Usage {
   agents_enabled: boolean;
@@ -26,7 +27,29 @@ export interface Usage {
   fleet_server: FleetServerUsage;
 }
 
-export const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => {
+export const fetchFleetUsage = async (
+  core: CoreSetup,
+  config: FleetConfigType,
+  abortController: AbortController
+) => {
+  const [soClient, esClient] = await getInternalClients(core);
+  if (!soClient || !esClient) {
+    return;
+  }
+  const usage = {
+    agents_enabled: getIsAgentsEnabled(config),
+    agents: await getAgentUsage(config, soClient, esClient),
+    fleet_server: await getFleetServerUsage(soClient, esClient),
+    packages: await getPackageUsage(soClient),
+    ...(await getAgentData(esClient, abortController)),
+    fleet_server_config: await getFleetServerConfig(soClient),
+    agent_policies: await getAgentPoliciesUsage(esClient, abortController),
+  };
+  return usage;
+};
+
+// used by kibana daily collector
+const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => {
   const [soClient, esClient] = await getInternalClients(core);
   const usage = {
     agents_enabled: getIsAgentsEnabled(config),

x-pack/plugins/fleet/server/plugin.ts

@@ -88,7 +88,11 @@ import {
   AgentServiceImpl,
   PackageServiceImpl,
 } from './services';
-import { registerFleetUsageCollector, fetchUsage, fetchAgentsUsage } from './collectors/register';
+import {
+  registerFleetUsageCollector,
+  fetchAgentsUsage,
+  fetchFleetUsage,
+} from './collectors/register';
 import { getAuthzFromRequest, makeRouterWithFleetAuthz } from './routes/security';
 import { FleetArtifactsClient } from './services/artifacts';
 import type { FleetRouter } from './types/request_context';
@@ -370,14 +374,9 @@ export class FleetPlugin
 
     // Register usage collection
     registerFleetUsageCollector(core, config, deps.usageCollection);
-    const fetch = async () => fetchUsage(core, config);
-    this.fleetUsageSender = new FleetUsageSender(
-      deps.taskManager,
-      core,
-      fetch,
-      this.kibanaVersion,
-      this.isProductionMode
-    );
+    const fetch = async (abortController: AbortController) =>
+      await fetchFleetUsage(core, config, abortController);
+    this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch);
     registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config));
 
     const router: FleetRouter = core.http.createRouter<FleetRequestHandlerContext>();