[Monitoring] Metricbeat migration status logic

x-pack/plugins/monitoring/common/constants.js

@@ -161,3 +161,6 @@ export const INDEX_ALERTS = '.monitoring-alerts-6,.monitoring-alerts-7';
 export const INDEX_PATTERN_ELASTICSEARCH = '.monitoring-es-6-*,.monitoring-es-7-*';
 
 export const INDEX_PATTERN_FILEBEAT = 'filebeat-*';
+
+// This is the unique token that exists in monitoring indices collected by metricbeat
+export const METRICBEAT_INDEX_NAME_UNIQUE_TOKEN = '-mb-';

x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { prefixIndexPattern } from '../ccs_utils';
+import {
+  INDEX_PATTERN_ELASTICSEARCH,
+  INDEX_PATTERN_KIBANA,
+  INDEX_PATTERN_LOGSTASH,
+  INDEX_PATTERN_BEATS,
+  INDEX_ALERTS
+} from '../../../common/constants';
+
+export function getIndexPatterns(server, additionalPatterns = {}) {
+  // wildcard means to search _all_ clusters
+  const ccs = '*';
+  const config = server.config();
+  const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+  const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs);
+  const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs);
+  const beatsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
+  const apmIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
+  const alertsIndex = prefixIndexPattern(config, INDEX_ALERTS, ccs);
+  const indexPatterns = {
+    esIndexPattern,
+    kbnIndexPattern,
+    lsIndexPattern,
+    beatsIndexPattern,
+    apmIndexPattern,
+    alertsIndex,
+    ...Object.keys(additionalPatterns).reduce((accum, varName) => {
+      return {
+        ...accum,
+        [varName]: prefixIndexPattern(config, additionalPatterns[varName], ccs),
+      };
+    }, {})
+  };
+
+  return indexPatterns;
+}

x-pack/plugins/monitoring/server/lib/setup/collection/__test__/get_collection_status.js

@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import sinon from 'sinon';
+import { getCollectionStatus } from '../';
+import { getIndexPatterns } from '../../../cluster/get_index_patterns';
+
+const mockReq = (searchResult = {}, msearchResult = { responses: [] }) => {
+  return {
+    server: {
+      config() {
+        return {
+          get: sinon.stub()
+            .withArgs('server.uuid').returns('kibana-1234')
+        };
+      },
+      plugins: {
+        elasticsearch: {
+          getCluster() {
+            return {
+              callWithRequest(_req, type) {
+                return Promise.resolve(type === 'search' ? searchResult : msearchResult);
+              }
+            };
+          }
+        }
+      },
+    },
+  };
+};
+
+describe('getCollectionStatus', () => {
+  it('should handle all stack products with internal monitoring', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [
+                { key: 'apm_1', beat_type: { buckets: [ { key: 'apm-server' }] } },
+                { key: 'beats_1' }
+              ] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(1);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.kibana.byUuid.kibana_1.isInternalCollector).to.be(true);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.apm.totalUniqueInstanceCount).to.be(1);
+    expect(result.apm.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.apm.byUuid.apm_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.elasticsearch.byUuid.es_1.isInternalCollector).to.be(true);
+  });
+
+  it('should handle some stack products as fully migrated', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-mb-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-mb-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [{ key: 'beats_1' }] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(1);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.kibana.byUuid.kibana_1.isFullyMigrated).to.be(true);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
+  });
+
+  it('should handle some stack products as partially migrated', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-mb-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-mb-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }, { key: 'kibana_2' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1', by_timestamp: { value: 12 } }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [{ key: 'beats_1' }] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(2);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.kibana.byUuid.kibana_1.isPartiallyMigrated).to.be(true);
+    expect(result.kibana.byUuid.kibana_1.lastInternallyCollectedTimestamp).to.be(12);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
+  });
+
+  it('should detect products based on other indices', async () => {
+    const req = mockReq({}, {
+      responses: [
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } }
+      ]
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.detected.doesExist).to.be(true);
+    expect(result.elasticsearch.detected.doesExist).to.be(true);
+    expect(result.beats.detected.mightExist).to.be(true);
+    expect(result.logstash.detected.mightExist).to.be(true);
+  });
+});