[Fleet] Save package policy previous revision on package upgrade

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -313,12 +313,9 @@
     "enabled",
     "error",
     "filter",
-    "indexPattern",
     "integrationName",
     "managed",
     "matchers",
-    "matchers.fields",
-    "matchers.values",
     "name",
     "type"
   ],
@@ -373,6 +370,7 @@
     "latest_install_failed_attempts",
     "name",
     "package_assets",
+    "previous_version",
     "verification_key_id",
     "verification_status",
     "version"
@@ -550,6 +548,7 @@
     "enabled",
     "inputs",
     "is_managed",
+    "latest_revision",
     "name",
     "namespace",
     "output_id",
@@ -735,6 +734,7 @@
     "enabled",
     "inputs",
     "is_managed",
+    "latest_revision",
     "name",
     "namespace",
     "output_id",
@@ -841,19 +841,6 @@
     "job.job_id",
     "model_id"
   ],
-  "monitoring-entity-source": [
-    "enabled",
-    "error",
-    "filter",
-    "indexPattern",
-    "integrationName",
-    "managed",
-    "matchers",
-    "matchers.fields",
-    "matchers.values",
-    "name",
-    "type"
-  ],
   "monitoring-telemetry": [
     "reportedClusterUuids"
   ],

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1256,6 +1256,9 @@
         "dynamic": false,
         "properties": {}
       },
+      "previous_version": {
+        "type": "keyword"
+      },
       "verification_key_id": {
         "type": "keyword"
       },
@@ -1835,6 +1838,9 @@
       "is_managed": {
         "type": "boolean"
       },
+      "latest_revision": {
+        "type": "boolean"
+      },
       "name": {
         "type": "keyword"
       },
@@ -2438,6 +2444,9 @@
       "is_managed": {
         "type": "boolean"
       },
+      "latest_revision": {
+        "type": "boolean"
+      },
       "name": {
         "type": "keyword"
       },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -100,7 +100,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
-        "epm-packages": "5a9f55e38d424f5b5ebbfeac802788b5b05d867f",
+        "epm-packages": "db1a500677ffca84c2900df498f83626554cb4fb",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
         "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",
@@ -113,7 +113,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "fleet-agent-policies": "f69f7c5639f4cf9e85077c904e161f3574ac3ca2",
         "fleet-fleet-server-host": "232d98738d5321b86edc426e21a9ca2f607da999",
         "fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f",
-        "fleet-package-policies": "b1ded996118af658bc420a737ff3c4d784641fc7",
+        "fleet-package-policies": "efd05a0ed95f387cecf0fad8902c482a5732668b",
         "fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e",
         "fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d",
         "fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde",
@@ -129,7 +129,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "cfe66f4aeca8f53b26bd4ddb0e956de1637d774e",
         "ingest-download-sources": "5be99940d6b5f9121b2fd279708d14e2bc0bde26",
         "ingest-outputs": "6743521f501bd77b1523dbb1df48d7c47fdad529",
-        "ingest-package-policies": "6a80000fdf2544f2485b0c6a51ecc434b6a12987",
+        "ingest-package-policies": "cb7e5f23e0af62f2d66e194ad2b108c9645e422c",
         "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505",
         "intercept_interaction_record": "13587751af378409df5cadd08aeb0d3884b1645a",
         "intercept_trigger_record": "9223039379bf9997781ad91df120eb360c3e6b77",

x-pack/platform/plugins/shared/fleet/common/experimental_features.ts

@@ -16,6 +16,7 @@ const _allowedExperimentalValues = {
   installedIntegrationsTabularUI: true,
   enabledUpgradeAgentlessDeploymentsTask: true,
   enableAgentMigrations: false,
+  enablePackageRollback: false,
 };
 
 /**

x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts

@@ -684,6 +684,7 @@ export interface Installation {
   latest_uninstall_failed_attempts?: FailedAttempt[];
   latest_executed_state?: InstallLatestExecutedState;
   latest_custom_asset_install_failed_attempts?: { [asset: string]: CustomAssetFailedAttempt };
+  previous_version?: string;
 }
 
 export interface PackageUsageStats {

x-pack/platform/plugins/shared/fleet/server/integration_tests/fleet_usage_telemetry.test.ts

@@ -367,6 +367,7 @@ describe('fleet usage telemetry', () => {
           },
         },
       ],
+      latest_revision: true,
     });
 
     await soClient.create('ingest-package-policies', {
@@ -381,6 +382,7 @@ describe('fleet usage telemetry', () => {
       policy_id: 'policy2',
       policy_ids: ['policy2', 'policy3'],
       inputs: [],
+      latest_revision: true,
     });
 
     await soClient.create(

x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts

@@ -103,6 +103,7 @@ import {
 import { backfillAgentPolicyToV4 } from './model_versions/agent_policy_v4';
 import { backfillOutputPolicyToV7 } from './model_versions/outputs';
 import { packagePolicyV17AdvancedFieldsForEndpointV818 } from './model_versions/security_solution/v17_advanced_package_policy_fields';
+import { backfillPackagePolicyLatestRevision } from './model_versions/package_policy_latest_revision_backfill';
 
 /*
  * Saved object types and mappings
@@ -695,6 +696,7 @@ export const getSavedObjectTypes = (
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
           bump_agent_policy_revision: { type: 'boolean' },
+          latest_revision: { type: 'boolean' },
         },
       },
       modelVersions: {
@@ -875,6 +877,20 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '19': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                latest_revision: { type: 'boolean' },
+              },
+            },
+            {
+              type: 'data_backfill',
+              backfillFn: backfillPackagePolicyLatestRevision,
+            },
+          ],
+        },
       },
       migrations: {
         '7.10.0': migratePackagePolicyToV7100,
@@ -938,6 +954,7 @@ export const getSavedObjectTypes = (
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
           bump_agent_policy_revision: { type: 'boolean' },
+          latest_revision: { type: 'boolean' },
         },
       },
       modelVersions: {
@@ -977,6 +994,20 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '5': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                latest_revision: { type: 'boolean' },
+              },
+            },
+            {
+              type: 'data_backfill',
+              backfillFn: backfillPackagePolicyLatestRevision,
+            },
+          ],
+        },
       },
     },
     [PACKAGES_SAVED_OBJECT_TYPE]: {
@@ -1043,6 +1074,7 @@ export const getSavedObjectTypes = (
               },
             },
           },
+          previous_version: { type: 'keyword' },
         },
       },
       modelVersions: {
@@ -1084,6 +1116,16 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '5': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                previous_version: { type: 'keyword' },
+              },
+            },
+          ],
+        },
       },
       migrations: {
         '7.14.0': migrateInstallationToV7140,

x-pack/platform/plugins/shared/fleet/server/saved_objects/model_versions/package_policy_latest_revision_backfill.test.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  SavedObject,
+  SavedObjectModelTransformationContext,
+} from '@kbn/core-saved-objects-server';
+
+import type { PackagePolicy } from '../../../common';
+
+import { backfillPackagePolicyLatestRevision } from './package_policy_latest_revision_backfill';
+
+describe('backfillPackagePolicyLatestRevision', () => {
+  const packagePolicyDoc: SavedObject<PackagePolicy & { latest_revision?: boolean }> = {
+    id: 'policy1',
+    type: 'ingest-package-policies',
+    references: [],
+    attributes: {
+      id: 'policy1',
+      name: 'Policy 1',
+      namespace: 'default',
+      description: '',
+      policy_id: 'policy1',
+      policy_ids: ['policy1'],
+      package: { name: 'test-package', version: '1.0.0', title: 'Test Package' },
+      inputs: [],
+      revision: 1,
+      updated_at: '2021-08-17T14:00:00.000Z',
+      updated_by: 'elastic',
+      created_at: '2021-08-17T14:00:00.000Z',
+      created_by: 'elastic',
+      enabled: true,
+    },
+  };
+
+  it('should set latest_revision to true if not defined', () => {
+    const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
+      packagePolicyDoc,
+      {} as SavedObjectModelTransformationContext
+    );
+
+    expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(true);
+  });
+
+  it('should not change latest_revision if already defined', () => {
+    const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
+      {
+        ...packagePolicyDoc,
+        attributes: {
+          ...packagePolicyDoc.attributes,
+          latest_revision: false,
+        },
+      },
+      {} as SavedObjectModelTransformationContext
+    );
+
+    expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(false);
+  });
+});

x-pack/platform/plugins/shared/fleet/server/saved_objects/model_versions/package_policy_latest_revision_backfill.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server';
+
+import type { PackagePolicy } from '../../../common';
+
+export const backfillPackagePolicyLatestRevision: SavedObjectModelDataBackfillFn<
+  PackagePolicy & { latest_revision?: boolean },
+  PackagePolicy & { latest_revision?: boolean }
+> = (packagePolicyDoc) => {
+  if (packagePolicyDoc.attributes.latest_revision === undefined) {
+    packagePolicyDoc.attributes.latest_revision = true;
+  }
+
+  return packagePolicyDoc;
+};

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts

@@ -1129,8 +1129,16 @@ export async function restartInstallation(options: {
   pkgVersion: string;
   installSource: InstallSource;
   verificationResult?: PackageVerificationResult;
+  previousVersion?: string;
 }) {
-  const { savedObjectsClient, pkgVersion, pkgName, installSource, verificationResult } = options;
+  const {
+    savedObjectsClient,
+    pkgVersion,
+    pkgName,
+    installSource,
+    verificationResult,
+    previousVersion,
+  } = options;
 
   let savedObjectUpdate: Partial<Installation> = {
     install_version: pkgVersion,
@@ -1147,6 +1155,10 @@ export async function restartInstallation(options: {
     };
   }
 
+  if (previousVersion) {
+    savedObjectUpdate.previous_version = previousVersion;
+  }
+
   auditLoggingService.writeCustomSoAuditLog({
     action: 'update',
     id: pkgName,

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import semverGt from 'semver/functions/gt';
+
 import { ConcurrentInstallOperationError } from '../../../../../errors';
 import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../constants';
 
@@ -29,6 +31,9 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
 
   // if some installation already exists
   if (installedPkg) {
+    const previousVersion = semverGt(pkgVersion, installedPkg.attributes.install_version)
+      ? installedPkg.attributes.install_version
+      : undefined;
     const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
     const hasExceededTimeout =
       Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
@@ -50,6 +55,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
             pkgVersion,
             installSource,
             verificationResult,
+            previousVersion,
           })
         );
       } else {
@@ -72,6 +78,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
           pkgVersion,
           installSource,
           verificationResult,
+          previousVersion,
         })
       );
     }

x-pack/platform/plugins/shared/fleet/server/services/package_policies/utils.ts

@@ -38,7 +38,11 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({
   attributes,
   namespaces,
 }: SavedObject<PackagePolicySOAttributes>): PackagePolicy => {
-  const { bump_agent_policy_revision: bumpAgentPolicyRevision, ...restAttributes } = attributes;
+  const {
+    bump_agent_policy_revision: bumpAgentPolicyRevision,
+    latest_revision: latestRevision,
+    ...restAttributes
+  } = attributes;
   return {
     id,
     version,