[Fleet] fix index pattern recreation when removing experimental packages

x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts

@@ -86,7 +86,7 @@ export async function getPackageKeysByStatus(
   savedObjectsClient: SavedObjectsClientContract,
   status: InstallationStatus
 ) {
-  const allPackages = await getPackages({ savedObjectsClient });
+  const allPackages = await getPackages({ savedObjectsClient, experimental: true });
   return allPackages.reduce<Array<{ pkgName: string; pkgVersion: string }>>((acc, pkg) => {
     if (pkg.status === status) {
       if (pkg.status === InstallationStatus.installed) {

x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_multiple.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const kibanaServer = getService('kibanaServer');
+  const supertest = getService('supertest');
+  const dockerServers = getService('dockerServers');
+  const server = dockerServers.get('registry');
+  const pkgName = 'all_assets';
+  const pkgVersion = '0.1.0';
+  const pkgKey = `${pkgName}-${pkgVersion}`;
+  const experimentalPkgName = 'experimental';
+  const experimentalPkgKey = `${experimentalPkgName}-${pkgVersion}`;
+  const experimental2PkgName = 'experimental2';
+  const experimental2PkgKey = `${experimental2PkgName}-${pkgVersion}`;
+
+  const uninstallPackage = async (pkg: string) => {
+    await supertest.delete(`/api/fleet/epm/packages/${pkg}`).set('kbn-xsrf', 'xxxx');
+  };
+  const installPackage = async (pkg: string) => {
+    await supertest
+      .post(`/api/fleet/epm/packages/${pkg}`)
+      .set('kbn-xsrf', 'xxxx')
+      .send({ force: true });
+  };
+
+  const installPackages = async (pkgs: string[]) => {
+    const installingPackagesPromise = pkgs.map((pkg) => installPackage(pkg));
+    return Promise.all(installingPackagesPromise);
+  };
+  const uninstallPackages = async (pkgs: string[]) => {
+    const uninstallingPackagesPromise = pkgs.map((pkg) => uninstallPackage(pkg));
+    return Promise.all(uninstallingPackagesPromise);
+  };
+  const expectPkgFieldToExist = async (
+    fields: any[],
+    fieldName: string,
+    exists: boolean = true
+  ) => {
+    const fieldExists = fields.find((field: { name: string }) => field.name === fieldName);
+    if (exists) {
+      expect(fieldExists).not.to.be(undefined);
+    } else {
+      expect(fieldExists).to.be(undefined);
+    }
+  };
+  describe('installs and uninstalls multiple packages side effects', async () => {
+    skipIfNoDockerRegistry(providerContext);
+    before(async () => {
+      if (!server.enabled) return;
+      await installPackages([pkgKey, experimentalPkgKey, experimental2PkgKey]);
+    });
+    after(async () => {
+      if (!server.enabled) return;
+      await uninstallPackages([pkgKey, experimentalPkgKey, experimental2PkgKey]);
+    });
+    it('should create index patterns from all installed packages, experimental or beta', async () => {
+      const resIndexPatternLogs = await kibanaServer.savedObjects.get({
+        type: 'index-pattern',
+        id: 'logs-*',
+      });
+
+      const fieldsLogs = JSON.parse(resIndexPatternLogs.attributes.fields);
+
+      expectPkgFieldToExist(fieldsLogs, 'logs_test_name');
+      expectPkgFieldToExist(fieldsLogs, 'logs_experimental_name');
+      expectPkgFieldToExist(fieldsLogs, 'logs_experimental2_name');
+      const resIndexPatternMetrics = await kibanaServer.savedObjects.get({
+        type: 'index-pattern',
+        id: 'metrics-*',
+      });
+      const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields);
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_test_name');
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental_name');
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental2_name');
+    });
+    it('should correctly recreate index patterns when a package is uninstalled', async () => {
+      await uninstallPackage(experimental2PkgKey);
+      const resIndexPatternLogs = await kibanaServer.savedObjects.get({
+        type: 'index-pattern',
+        id: 'logs-*',
+      });
+      const fields = JSON.parse(resIndexPatternLogs.attributes.fields);
+      expectPkgFieldToExist(fields, 'logs_test_name');
+      expectPkgFieldToExist(fields, 'logs_experimental_name');
+      expectPkgFieldToExist(fields, 'logs_experimental2_name', false);
+      const resIndexPatternMetrics = await kibanaServer.savedObjects.get({
+        type: 'index-pattern',
+        id: 'metrics-*',
+      });
+      const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields);
+
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_test_name');
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental_name');
+      expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental2_name', false);
+    });
+  });
+}

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_logs/fields/ecs.yml

@@ -0,0 +1,3 @@
+- name: logs_experimental_name
+  title: logs_experimental_title
+  type: keyword

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_logs/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_logs/manifest.yml

@@ -0,0 +1,9 @@
+title: Test Dataset
+
+type: logs
+
+elasticsearch:
+  index_template.mappings:
+    dynamic: false
+  index_template.settings:
+    index.lifecycle.name: reference

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_metrics/fields/ecs.yml

@@ -0,0 +1,3 @@
+- name: metrics_experimental_name
+  title: metrics_experimental_title
+  type: keyword

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_metrics/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/data_stream/test_metrics/manifest.yml

@@ -0,0 +1,3 @@
+title: Test Dataset
+
+type: metrics

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+For testing side effects when installing and removing multiple packages 

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental/0.1.0/manifest.yml

@@ -0,0 +1,20 @@
+format_version: 1.0.0
+name: experimental 
+title: experimental integration
+description: This is a test package for testing experimental packages
+version: 0.1.0
+categories: []
+release: experimental
+type: integration
+license: basic
+
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+
+icons:
+  - src: '/img/logo_overrides_64_color.svg'
+    size: '16x16'
+    type: 'image/svg+xml'

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_logs/fields/ecs.yml

@@ -0,0 +1,3 @@
+- name: logs_experimental2_name
+  title: logs_experimental2_title
+  type: keyword

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_logs/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_logs/manifest.yml

@@ -0,0 +1,9 @@
+title: Test Dataset
+
+type: logs
+
+elasticsearch:
+  index_template.mappings:
+    dynamic: false
+  index_template.settings:
+    index.lifecycle.name: reference

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_metrics/fields/ecs.yml

@@ -0,0 +1,3 @@
+- name: metrics_experimental2_name
+  title: metrics_experimental2_title
+  type: keyword

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_metrics/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/data_stream/test_metrics/manifest.yml

@@ -0,0 +1,3 @@
+title: Test Dataset
+
+type: metrics

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+For testing side effects when installing and removing multiple packages 

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/experimental2/0.1.0/manifest.yml

@@ -0,0 +1,20 @@
+format_version: 1.0.0
+name: experimental2
+title: experimental integration
+description: This is a test package for testing experimental packages
+version: 0.1.0
+categories: []
+release: experimental
+type: integration
+license: basic
+
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+
+icons:
+  - src: '/img/logo_overrides_64_color.svg'
+    size: '16x16'
+    type: 'image/svg+xml'