[Fleet] Only enable secret storage once all fleet servers are above 8.10.0

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1505,6 +1505,9 @@
       },
       "prerelease_integrations_enabled": {
         "type": "boolean"
+      },
+      "secret_storage_requirements_met": {
+        "type": "boolean"
       }
     }
   },

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-download-sources": "d7edc5e588d9afa61c4b831604582891c54ef1c7",
         "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0",
         "ingest-package-policies": "55816507db0134b8efbe0509e311a91ce7e1c6cc",
-        "ingest_manager_settings": "418311b03c8eda53f5d2ea6f54c1356afaa65511",
+        "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
         "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",
         "legacy-url-alias": "9b8cca3fbb2da46fd12823d3cd38fdf1c9f24bc8",

x-pack/plugins/fleet/common/constants/secrets.ts

@@ -6,3 +6,5 @@
  */
 
 export const SECRETS_ENDPOINT_PATH = '/_fleet/secret';
+
+export const SECRETS_MINIMUM_FLEET_SERVER_VERSION = '8.10.0';

x-pack/plugins/fleet/common/types/models/settings.ts

@@ -14,4 +14,5 @@ export interface BaseSettings {
 export interface Settings extends BaseSettings {
   id: string;
   preconfigured_fields?: Array<'fleet_server_hosts'>;
+  secret_storage_requirements_met?: boolean;
 }

x-pack/plugins/fleet/server/constants/index.ts

@@ -79,6 +79,7 @@ export {
   MESSAGE_SIGNING_SERVICE_API_ROUTES,
   // secrets
   SECRETS_ENDPOINT_PATH,
+  SECRETS_MINIMUM_FLEET_SERVER_VERSION,
 } from '../../common/constants';
 
 export {

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -90,6 +90,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
         fleet_server_hosts: { type: 'keyword' },
         has_seen_add_data_notice: { type: 'boolean', index: false },
         prerelease_integrations_enabled: { type: 'boolean' },
+        secret_storage_requirements_met: { type: 'boolean' },
       },
     },
     migrations: {

x-pack/plugins/fleet/server/services/agents/crud.ts

@@ -444,6 +444,66 @@ export async function getAgentsById(
   );
 }
 
+// given a list of agentPolicyIds, return a map of agent version => count of agents
+// this is used to get all fleet server versions
+export async function getAgentVersionsForAgentPolicyIds(
+  esClient: ElasticsearchClient,
+  agentPolicyIds: string[]
+): Promise<Record<string, number>> {
+  const versionCount: Record<string, number> = {};
+
+  if (!agentPolicyIds.length) {
+    return versionCount;
+  }
+
+  try {
+    const res = esClient.search<
+      FleetServerAgent,
+      Record<'agent_versions', { buckets: Array<{ key: string; doc_count: number }> }>
+    >({
+      size: 0,
+      track_total_hits: false,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {
+                terms: {
+                  policy_id: agentPolicyIds,
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          agent_versions: {
+            terms: {
+              field: 'local_metadata.elastic.agent.version.keyword',
+              size: 1000,
+            },
+          },
+        },
+      },
+      index: AGENTS_INDEX,
+      ignore_unavailable: true,
+    });
+
+    const { aggregations } = await res;
+
+    if (aggregations && aggregations.agent_versions) {
+      aggregations.agent_versions.buckets.forEach((bucket) => {
+        versionCount[bucket.key] = bucket.doc_count;
+      });
+    }
+  } catch (error) {
+    if (error.statusCode !== 404) {
+      throw error;
+    }
+  }
+
+  return versionCount;
+}
+
 export async function getAgentByAccessAPIKeyId(
   esClient: ElasticsearchClient,
   soClient: SavedObjectsClientContract,

x-pack/plugins/fleet/server/services/fleet_server/index.ts

@@ -5,10 +5,14 @@
  * 2.0.
  */
 
-import type { ElasticsearchClient } from '@kbn/core/server';
+import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import semverGte from 'semver/functions/gte';
+import semverCoerce from 'semver/functions/coerce';
 
 import { FLEET_SERVER_SERVERS_INDEX } from '../../constants';
+import { getAgentVersionsForAgentPolicyIds } from '../agents';
 
+import { packagePolicyService } from '../package_policy';
 /**
  * Check if at least one fleet server is connected
  */
@@ -23,3 +27,42 @@ export async function hasFleetServers(esClient: ElasticsearchClient) {
 
   return (res.hits.total as number) > 0;
 }
+
+export async function allFleetServerVersionsAreAtLeast(
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract,
+  version: string
+): Promise<boolean> {
+  let hasMore = true;
+  const policyIds = new Set<string>();
+  let page = 1;
+  while (hasMore) {
+    const res = await packagePolicyService.list(soClient, {
+      page: page++,
+      perPage: 20,
+      kuery: 'ingest-package-policies.package.name:fleet_server',
+    });
+
+    for (const item of res.items) {
+      policyIds.add(item.policy_id);
+    }
+
+    if (res.items.length === 0) {
+      hasMore = false;
+    }
+  }
+
+  const versionCounts = await getAgentVersionsForAgentPolicyIds(esClient, [...policyIds]);
+  const versions = Object.keys(versionCounts);
+
+  // there must be at least one fleet server agent for this check to pass
+  if (versions.length === 0) {
+    return false;
+  }
+
+  return _allVersionsAreAtLeast(version, versions);
+}
+
+function _allVersionsAreAtLeast(version: string, versions: string[]) {
+  return versions.every((v) => semverGte(semverCoerce(v)!, version));
+}

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -117,6 +117,7 @@ import {
   extractAndUpdateSecrets,
   extractAndWriteSecrets,
   deleteSecretsIfNotReferenced as deleteSecrets,
+  isSecretStorageEnabled,
 } from './secrets';
 
 export type InputsOverride = Partial<NewPackagePolicyInput> & {
@@ -243,8 +244,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       }
       validatePackagePolicyOrThrow(enrichedPackagePolicy, pkgInfo);
 
-      const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
-      if (secretsStorageEnabled) {
+      if (await isSecretStorageEnabled(esClient, soClient)) {
         const secretsRes = await extractAndWriteSecrets({
           packagePolicy: { ...enrichedPackagePolicy, inputs },
           packageInfo: pkgInfo,
@@ -747,8 +747,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       });
       validatePackagePolicyOrThrow(packagePolicy, pkgInfo);
 
-      const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
-      if (secretsStorageEnabled) {
+      if (await isSecretStorageEnabled(esClient, soClient)) {
         const secretsRes = await extractAndUpdateSecrets({
           oldPackagePolicy,
           packagePolicyUpdate: { ...restOfPackagePolicy, inputs },
@@ -913,9 +912,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
           );
           if (pkgInfo) {
             validatePackagePolicyOrThrow(packagePolicy, pkgInfo);
-            const { secretsStorage: secretsStorageEnabled } =
-              appContextService.getExperimentalFeatures();
-            if (secretsStorageEnabled) {
+            if (await isSecretStorageEnabled(esClient, soClient)) {
               const secretsRes = await extractAndUpdateSecrets({
                 oldPackagePolicy,
                 packagePolicyUpdate: { ...restOfPackagePolicy, inputs },

x-pack/plugins/fleet/server/services/secrets.ts

@@ -34,14 +34,16 @@ import type {
 } from '../types';
 
 import { FleetError } from '../errors';
-import { SECRETS_ENDPOINT_PATH } from '../constants';
+import { SECRETS_ENDPOINT_PATH, SECRETS_MINIMUM_FLEET_SERVER_VERSION } from '../constants';
 
 import { retryTransientEsErrors } from './epm/elasticsearch/retry';
 
 import { auditLoggingService } from './audit_logging';
 
 import { appContextService } from './app_context';
 import { packagePolicyService } from './package_policy';
+import { settingsService } from '.';
+import { allFleetServerVersionsAreAtLeast } from './fleet_server';
 
 export async function createSecrets(opts: {
   esClient: ElasticsearchClient;
@@ -270,10 +272,21 @@ export async function extractAndUpdateSecrets(opts: {
     ...createdSecrets.map(({ id }) => ({ id })),
   ];
 
+  const secretsToDelete: PolicySecretReference[] = [];
+
+  toDelete.forEach((secretPath) => {
+    // check if the previous secret is actually a secret refrerence
+    // it may be that secrets were not enabled at the time of creation
+    // in which case they are just stored as plain text
+    if (secretPath.value.value.isSecretRef) {
+      secretsToDelete.push({ id: secretPath.value.value.id });
+    }
+  });
+
   return {
     packagePolicyUpdate: policyWithSecretRefs,
     secretReferences,
-    secretsToDelete: toDelete.map((secretPath) => ({ id: secretPath.value.value.id })),
+    secretsToDelete,
   };
 }
 
@@ -344,6 +357,58 @@ export function getPolicySecretPaths(
   return [...packageLevelVarPaths, ...inputSecretPaths];
 }
 
+export async function isSecretStorageEnabled(
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract
+): Promise<boolean> {
+  const logger = appContextService.getLogger();
+
+  // first check if the feature flag is enabled, if not secrets are disabled
+  const { secretsStorage: secretsStorageEnabled } = appContextService.getExperimentalFeatures();
+  if (!secretsStorageEnabled) {
+    logger.debug('Secrets storage is disabled by feature flag');
+    return false;
+  }
+
+  // if serverless then secrets will always be supported
+  const isFleetServerStandalone =
+    appContextService.getConfig()?.internal?.fleetServerStandalone ?? false;
+
+  if (isFleetServerStandalone) {
+    logger.trace('Secrets storage is enabled as fleet server is standalone');
+    return true;
+  }
+
+  // now check the flag in settings to see if the fleet server requirement has already been met
+  // once the requirement has been met, secrets are always on
+  const settings = await settingsService.getSettings(soClient);
+
+  if (settings.secret_storage_requirements_met) {
+    logger.debug('Secrets storage already met, turned on is settings');
+    return true;
+  }
+
+  // otherwise check if we have the minimum fleet server version and enable secrets if so
+  if (
+    await allFleetServerVersionsAreAtLeast(esClient, soClient, SECRETS_MINIMUM_FLEET_SERVER_VERSION)
+  ) {
+    logger.debug('Enabling secrets storage as minimum fleet server version has been met');
+    try {
+      await settingsService.saveSettings(soClient, {
+        secret_storage_requirements_met: true,
+      });
+    } catch (err) {
+      // we can suppress this error as it will be retried on the next function call
+      logger.warn(`Failed to save settings after enabling secrets storage: ${err.message}`);
+    }
+
+    return true;
+  }
+
+  logger.info('Secrets storage is disabled as minimum fleet server version has not been met');
+  return false;
+}
+
 function _getPackageLevelSecretPaths(
   packagePolicy: NewPackagePolicy,
   packageInfo: PackageInfo

x-pack/plugins/fleet/server/types/so_attributes.ts

@@ -202,6 +202,7 @@ export interface SettingsSOAttributes {
   prerelease_integrations_enabled: boolean;
   has_seen_add_data_notice?: boolean;
   fleet_server_hosts?: string[];
+  secret_storage_requirements_met?: boolean;
 }
 
 export interface DownloadSourceSOAttributes {