Rollup support for TSVB (elastic#28762)

* Added a feature of rollup search on the UI side

Signed-off-by: Alexey Antonov <[email protected]>

* Rollup Feature - initial commit

* Revert "Added a feature of rollup search on the UI side"

This reverts commit 9568b0970b16f5102f50b748bb4d691a8612c2c2.

# Conflicts:
#	src/legacy/core_plugins/metrics/public/components/index_pattern.js

* Remove the 'label' property from the search strategies

* Changed search by strategy from the last

* add single search request

* rollup_search_strategy add base implementation of isViable method

* rollup_search_strategy add base implementation of isViable method -fix

* Changed requests due to search request type

* refactoring of import Base classes / remove '../../../../../../

* remove extra await

* rollup_search_strategy. Refactoring of isRollupJobExists method

* remove question

* Add support of annotations and table data

* skeleton for adding Search Strategy restrictions

* Add rollup search capabilities

* apply search strategy for annotations request

* set fields capabilities for rollup strategy

* add timezone, interval into SearchCapabilities

* Add fields from capabilities

* add timezone, interval into SearchCapabilities

* fix default timezone

* Merging of two Rollup Jobs was removed

* move getFieldsForWildcard to searchStrategy

* Fix TSVB search requests should have a timeout

# Conflicts:
#	src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js

* Add unit test

* apply getEsShardTimeout for annorations/get_request_params,  series/get_request_params

* rename metrics -> tsvb

* search_strategies_register refactoring: move 'add' method from class

* Add merge rollup capabilities with fields

* Add merge rollup capabilities with fields - small fixes

* Add support of 'Everything' aggregation for Rollup Search

* Return back metrics plugin

* remove 'metrics' from the X-pack\rollup require

* Fix test cases

* fix broken test: fail: "apis InfraOps GraphQL Endpoints metrics should basically work"

* rollup search - split by terms is not working

* Add count metric

* /get_bucket_size.js. Add support of 'auto' interval, Add support of gte intervals  e.g.:  >=1m

* fix build_request_body test

* [Rollup] [Phase 1] Error handling - rollup search errors should be more user friendly

* [Rollup] [Phase 1] Table View - research the query to ES - sorting is not wokring

* Merge elastic#26006 into rollup

# Conflicts:
#	src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js
#	src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js

* Add table view support

* fix broken build

* fix broken build

* [Rollup] [Phase 1] - write new tests  (rollup_search_request, rollup_search_strategy)

* [Rollup] [Phase 1] - write tests for rollup_search_capabilities

* Add test on default_search_capabilities, abstract_search_strategy, search_strategies_register

* Add test cases for search_requests folder

* [Rollup] [Phase 1] - write tests for rollup_search_strategy

* FIx broken build

* remove todo

* fix calculation of interval value for rollup search

* add unit tests

* Remove default exports

* fix PR comments

* fix calendar intervals

Author: alexwizp

src/legacy/core_plugins/metrics/index.js

@@ -21,14 +21,15 @@ import { resolve } from 'path';
 
 import fieldsRoutes from './server/routes/fields';
 import visDataRoutes from './server/routes/vis';
+import { SearchStrategiesRegister } from './server/lib/search_strategies/search_strategies_register';
 
 export default function (kibana) {
   return new kibana.Plugin({
     require: ['kibana', 'elasticsearch'],
 
     uiExports: {
       visTypes: [
-        'plugins/metrics/kbn_vis_types'
+        'plugins/metrics/kbn_vis_types',
       ],
       styleSheetPaths: resolve(__dirname, 'public/index.scss'),
     },
@@ -37,16 +38,15 @@ export default function (kibana) {
       return Joi.object({
         enabled: Joi.boolean().default(true),
         chartResolution: Joi.number().default(150),
-        minimumBucketSize: Joi.number().default(10)
+        minimumBucketSize: Joi.number().default(10),
       }).default();
     },
 
-
     init(server) {
       fieldsRoutes(server);
       visDataRoutes(server);
-    }
-
 
+      SearchStrategiesRegister.init(server);
+    },
   });
 }

src/legacy/core_plugins/metrics/public/components/_error.scss

@@ -1,15 +1,16 @@
-.tvbError__additional,
-.tvbError__stack {
-  margin-top: $euiSizeS;
-}
-
-// EUITODO: Convert to EuiCodeBlock
-.tvbError__stack {
-  padding: $euiSizeS;
-  background: $euiCodeBlockBackgroundColor;
-  color: $euiCodeBlockColor;
-  line-height: $euiLineHeight;
-  font-family: $euiCodeFontFamily;
-  font-weight: $euiFontWeightRegular;
-  white-space: pre-wrap;
-}
+.tvbError__title,
+.tvbError__additional,
+.tvbError__stack {
+  margin-top: $euiSizeS;
+}
+
+// EUITODO: Convert to EuiCodeBlock
+.tvbError__stack {
+  padding: $euiSizeS;
+  background: $euiCodeBlockBackgroundColor;
+  color: $euiCodeBlockColor;
+  line-height: $euiLineHeight;
+  font-family: $euiCodeFontFamily;
+  font-weight: $euiFontWeightRegular;
+  white-space: pre-wrap;
+}

src/legacy/core_plugins/metrics/public/components/error.js

@@ -23,17 +23,23 @@ import React from 'react';
 import _ from 'lodash';
 import { FormattedMessage } from '@kbn/i18n/react';
 
+const guidPattern = /\[[[a-f\d-\\]{36}\]/g;
+
 function ErrorComponent(props) {
   const { error } = props;
   let additionalInfo;
-  const type = _.get(error, 'error.caused_by.type');
+  const type = _.get(error, 'error.caused_by.type') || _.get(error, 'error.type');
   let reason = _.get(error, 'error.caused_by.reason');
   const title = _.get(error, 'error.caused_by.title');
 
   if (!reason) {
     reason = _.get(error, 'message');
   }
 
+  if (['runtime_exception', 'illegal_argument_exception'].includes(type)) {
+    reason = _.get(error, 'error.reason').replace(guidPattern, ``);
+  }
+
   if (type === 'script_exception') {
     const scriptStack = _.get(error, 'error.caused_by.script_stack');
     reason = _.get(error, 'error.caused_by.caused_by.reason');

src/legacy/core_plugins/metrics/public/components/visualization.js

@@ -27,7 +27,7 @@ import topN from './vis_types/top_n/vis';
 import table from './vis_types/table/vis';
 import gauge from './vis_types/gauge/vis';
 import markdown from './vis_types/markdown/vis';
-import Error from './error';
+import ErrorComponent from './error';
 import NoData from './no_data';
 
 const types = {
@@ -46,7 +46,7 @@ function Visualization(props) {
   if (error) {
     return (
       <div className={props.className}>
-        <Error error={error} />
+        <ErrorComponent error={error} />
       </div>
     );
   }

src/legacy/core_plugins/metrics/server/lib/get_fields.js

@@ -16,13 +16,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+import { SearchStrategiesRegister } from './search_strategies/search_strategies_register';
 import { uniq } from 'lodash';
 
 export async function getFields(req) {
-  const { indexPatternsService } = req.pre;
-  const index = req.query.index || '*';
-  const resp = await indexPatternsService.getFieldsForWildcard({ pattern: index });
-  const fields = resp.filter(field => field.aggregatable);
+  const indexPattern = req.query.index || '*';
+  const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern);
+
+  const fields = (await searchStrategy
+    .getFieldsForWildcard(req, indexPattern, capabilities))
+    .filter(field => field.aggregatable);
+
   return uniq(fields, field => field.name);
 }

src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { convertIntervalToUnit, parseInterval } from '../vis_data/helpers/unit_to_seconds';
+
+const getTimezoneFromRequest = request => {
+  return request.payload.timerange.timezone;
+};
+
+export class DefaultSearchCapabilities {
+  constructor(request, batchRequestsSupport, fieldsCapabilities = {}) {
+    this.request = request;
+    this.batchRequestsSupport = batchRequestsSupport;
+    this.fieldsCapabilities = fieldsCapabilities;
+  }
+
+  get defaultTimeInterval() {
+    return null;
+  }
+
+  get searchTimezone() {
+    return getTimezoneFromRequest(this.request);
+  }
+
+  parseInterval(interval) {
+    return parseInterval(interval);
+  }
+
+  convertIntervalToUnit(intervalString, unit) {
+    const parsedInterval = this.parseInterval(intervalString);
+
+    if (parsedInterval.unit !== unit) {
+      return convertIntervalToUnit(intervalString, unit);
+    }
+
+    return parsedInterval;
+  }
+
+  getValidTimeInterval(intervalString) {
+    // Default search capabilities doesn't have any restrictions for the interval string
+    return intervalString;
+  }
+}

src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js

@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { DefaultSearchCapabilities } from './default_search_capabilities';
+
+describe('DefaultSearchCapabilities', () => {
+  let defaultSearchCapabilities;
+  let batchRequestsSupport;
+  let req;
+
+  beforeEach(() => {
+    req = {};
+    batchRequestsSupport = true;
+    defaultSearchCapabilities = new DefaultSearchCapabilities(req, batchRequestsSupport);
+  });
+
+  test('should init default search capabilities', () => {
+    expect(defaultSearchCapabilities.request).toBe(req);
+    expect(defaultSearchCapabilities.batchRequestsSupport).toBe(batchRequestsSupport);
+    expect(defaultSearchCapabilities.fieldsCapabilities).toEqual({});
+  });
+
+  test('should return defaultTimeInterval', () => {
+    expect(defaultSearchCapabilities.defaultTimeInterval).toBe(null);
+  });
+
+  test('should return Search Timezone', () => {
+    defaultSearchCapabilities.request = {
+      payload: {
+        timerange: {
+          timezone: 'UTC'
+        }
+      }
+    };
+
+    expect(defaultSearchCapabilities.searchTimezone).toEqual('UTC');
+  });
+
+  test('should return a valid time interval', () => {
+    expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe('20m');
+  });
+
+  test('should parse interval', () => {
+    expect(defaultSearchCapabilities.parseInterval('120s')).toEqual({
+      value: 120,
+      unit: 's'
+    });
+
+    expect(defaultSearchCapabilities.parseInterval('20m')).toEqual({
+      value: 20,
+      unit: 'm'
+    });
+
+    expect(defaultSearchCapabilities.parseInterval('1y')).toEqual({
+      value: 1,
+      unit: 'y'
+    });
+  });
+
+  test('should convert interval string into different unit', () => {
+    expect(defaultSearchCapabilities.convertIntervalToUnit('120s', 's')).toEqual({
+      value: 120,
+      unit: 's'
+    });
+
+    expect(defaultSearchCapabilities.convertIntervalToUnit('60m', 'h')).toEqual({
+      value: 1,
+      unit: 'h'
+    });
+
+    expect(defaultSearchCapabilities.convertIntervalToUnit('4w', 'M')).toEqual({
+      value: 1,
+      unit: 'M'
+    });
+
+    expect(defaultSearchCapabilities.convertIntervalToUnit('1y', 'w')).toEqual({
+      value: 48,
+      unit: 'w'
+    });
+
+    expect(defaultSearchCapabilities.convertIntervalToUnit('60s', 'm')).toEqual({
+      value: 1,
+      unit: 'm'
+    });
+
+    expect(defaultSearchCapabilities.convertIntervalToUnit('1s', 'ms')).toEqual({
+      value: 1000,
+      unit: 'ms'
+    });
+  });
+});

src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js renamed to src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js

@@ -17,10 +17,4 @@
  * under the License.
  */
 
-export default [
-  'std_deviation',
-  'variance',
-  'sum_of_squares'
-];
-
-
+export { SearchStrategiesRegister } from './search_strategies_register';

src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { AbstractSearchStrategy } from './strategies/abstract_search_strategy';
+import { AbstractSearchRequest } from './searh_requests/abstract_request';
+import { DefaultSearchStrategy } from './strategies/default_search_strategy';
+import { DefaultSearchCapabilities } from './default_search_capabilities';
+
+const strategies = [];
+
+const addStrategy = searchStrategy => {
+  if (searchStrategy instanceof AbstractSearchStrategy) {
+    strategies.unshift(searchStrategy);
+  }
+  return strategies;
+};
+
+export class SearchStrategiesRegister {
+  static init(server) {
+    server.expose('AbstractSearchStrategy', AbstractSearchStrategy);
+    server.expose('AbstractSearchRequest', AbstractSearchRequest);
+    server.expose('DefaultSearchCapabilities', DefaultSearchCapabilities);
+    server.expose('addSearchStrategy', searchStrategy => addStrategy(searchStrategy));
+
+    addStrategy(new DefaultSearchStrategy(server));
+  }
+
+  static async getViableStrategy(req, indexPattern) {
+    for (const searchStrategy of strategies) {
+      const { isViable, capabilities } = await searchStrategy.checkForViability(req, indexPattern);
+
+      if (isViable) {
+        return {
+          searchStrategy,
+          capabilities,
+        };
+      }
+    }
+  }
+}