[Security Solutions] Fixes bug with the filter query compatibility for transforms

docs/developer/plugin-list.asciidoc

@@ -485,7 +485,7 @@ using the CURL scripts in the scripts folder.
 |{kib-repo}blob/{branch}/x-pack/plugins/metrics_entities/README.md[metricsEntities]
 |This is the metrics and entities plugin where you add can add transforms for your project
 and group those transforms into modules. You can also re-use existing transforms in your
-modules as well.
+newly created modules as well.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/ml/readme.md[ml]

x-pack/plugins/metrics_entities/README.md

@@ -2,7 +2,7 @@
 
 This is the metrics and entities plugin where you add can add transforms for your project
 and group those transforms into modules. You can also re-use existing transforms in your
-modules as well.
+newly created modules as well.
 
 ## Turn on experimental flags
 During at least phase 1 of this development, please add these to your `kibana.dev.yml` file to turn on the feature:
@@ -309,16 +309,14 @@ are notes during the phased approach. As we approach production and the feature
 left over TODO's in the code base.
 
 - Add these properties to the route which are:
-  - disable_transforms/exclude flag to exclude 1 or more transforms within a module,
-  - pipeline flag,
-   - Change the REST routes on post to change the indexes for whichever indexes you want
- - Unit tests to ensure the data of the mapping.json includes the correct fields such as
-   _meta, at least one alias, a mapping section, etc... 
- - Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time
- - Add a sort of @timestamp to the output destination indexes?
- - Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'],
- - Add schema validation choosing some schema library (io-ts or Kibana Schema or ... )
- - Add unit tests
- - Add e2e tests
- - Move ui code into this plugin from security_solutions? (maybe?)
-   - UI code could be within `kibana/packages` instead of in here directly and I think we will be better off.
+  - disable_transforms/exclude flag to exclude 1 or more transforms within a module
+  - pipeline flag
+  - Change the REST routes on post to change the indexes for whichever indexes you want
+- Unit tests to ensure the data of the mapping.json includes the correct fields such as _meta, at least one alias, a mapping section, etc... 
+- Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time
+- Add a sort of @timestamp to the output destination indexes?
+- Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'],
+- Add schema validation choosing some schema library (io-ts or Kibana Schema or ... )
+- Add unit tests
+- Add e2e tests
+- Any UI code should not end up here. There is none right now, but all UI code should be within a kbn package or security_solutions

x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.test.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { adjustTimeRange } from './adjust_timerange';
+import moment from 'moment';
+
+/** Get the return type of adjustTimeRange for TypeScript checks against expected */
+type ReturnTypeAdjustTimeRange = ReturnType<typeof adjustTimeRange>;
+
+describe('adjust_timerange', () => {
+  beforeEach(() => {
+    // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript
+    (moment as typeof moment & {
+      suppressDeprecationWarnings: boolean;
+    }).suppressDeprecationWarnings = true;
+  });
+
+  afterEach(() => {
+    // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript
+    (moment as typeof moment & {
+      suppressDeprecationWarnings: boolean;
+    }).suppressDeprecationWarnings = false;
+  });
+
+  test('it will adjust the time range from by rounding down by an hour within "from"', () => {
+    expect(
+      adjustTimeRange({
+        interval: '5m',
+        to: '2021-07-06T22:07:56.972Z',
+        from: '2021-07-06T22:07:56.972Z',
+      })
+    ).toMatchObject<Partial<ReturnTypeAdjustTimeRange>>({
+      timeRangeAdjusted: {
+        interval: '5m',
+        to: '2021-07-06T22:07:56.972Z',
+        from: '2021-07-06T22:00:00.000Z', // <-- Rounded down by an hour
+      },
+    });
+  });
+
+  test('it will compute the duration between to and and from', () => {
+    expect(
+      adjustTimeRange({
+        interval: '5m',
+        to: '2021-07-06T22:08:56.972Z',
+        from: '2021-07-06T22:07:56.972Z',
+      }).duration?.asMinutes()
+    ).toEqual(1);
+  });
+
+  test('it will return "undefined" if the to and from are invalid dateMath parsable', () => {
+    expect(
+      adjustTimeRange({
+        interval: '5m',
+        to: 'now-invalid',
+        from: 'now-invalid2',
+      })
+    ).toMatchObject<Partial<ReturnTypeAdjustTimeRange>>({
+      timeRangeAdjusted: undefined,
+      duration: undefined,
+    });
+  });
+});

x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts

@@ -9,11 +9,21 @@ import dateMath from '@elastic/datemath';
 import moment, { Duration } from 'moment';
 import type { TimerangeInput } from '../../../common/search_strategy';
 
-export type ParseTimeRange = (
-  timeRange: TimerangeInput
-) => { timeRangeAdjusted: TimerangeInput | undefined; duration: Duration | undefined };
+export interface TimeRangeAdjusted {
+  timeRangeAdjusted: TimerangeInput | undefined;
+  duration: Duration | undefined;
+}
 
-export const adjustTimeRange: ParseTimeRange = (timerange) => {
+/**
+ * Adjusts a given timerange by rounding the "from" down by an hour and returning
+ * the duration between "to" and "from". The duration is typically analyzed to determine
+ * if the adjustment should be made or not. Although we check "to" and use "to" for duration
+ * we are careful to still return "to: timerange.to", which is the original input to be careful
+ * about accidental bugs from trying to over parse or change relative date time ranges.
+ * @param timerange The timeRange to determine if we adjust or not
+ * @returns The time input adjustment and a duration
+ */
+export const adjustTimeRange = (timerange: TimerangeInput): TimeRangeAdjusted => {
   const from = dateMath.parse(timerange.from);
   const to = dateMath.parse(timerange.to);
   if (from == null || to == null) {

x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.test.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createIndicesFromPrefix } from './create_indices_from_prefix';
+
+/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */
+type ReturnTypeCreateIndicesFromPrefix = ReturnType<typeof createIndicesFromPrefix>;
+
+describe('create_indices_from_prefix', () => {
+  test('returns empty array given an empty array', () => {
+    expect(
+      createIndicesFromPrefix({
+        transformIndices: [],
+        prefix: 'prefix',
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>([]);
+  });
+
+  test('returns expected prefix given a set of indices', () => {
+    expect(
+      createIndicesFromPrefix({
+        transformIndices: ['index_1', 'index_2'],
+        prefix: 'prefix',
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(['.estc_prefix_index_1', '.estc_prefix_index_2']);
+  });
+});

x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts

@@ -7,6 +7,13 @@
 
 import { ELASTIC_NAME } from '../../../common/constants';
 
+/**
+ * Given a set of input indices and a prefix this will return the elastic name
+ * concatenated with the prefix.
+ * @param transformIndices The indices to add the prefix to
+ * @param prefix The prefix to add along with the elastic name
+ * @returns The indices with the prefix string
+ */
 export const createIndicesFromPrefix = ({
   transformIndices,
   prefix,

x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.test.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getSettingsMatch } from './get_settings_match';
+import { getTransformConfigSchemaMock } from './transform_config_schema.mock';
+
+/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */
+type ReturnTypeCreateIndicesFromPrefix = ReturnType<typeof getSettingsMatch>;
+
+describe('get_settings_match', () => {
+  test('it returns undefined given an empty array of indices', () => {
+    expect(
+      getSettingsMatch({
+        indices: [],
+        transformSettings: getTransformConfigSchemaMock(),
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(undefined);
+  });
+
+  test('it returns a setting given an index pattern that matches', () => {
+    expect(
+      getSettingsMatch({
+        indices: [
+          'auditbeat-*',
+          'endgame-*',
+          'filebeat-*',
+          'logs-*',
+          'packetbeat-*',
+          'winlogbeat-*',
+        ],
+        transformSettings: getTransformConfigSchemaMock(),
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(getTransformConfigSchemaMock().settings[0]);
+  });
+
+  test('it returns a setting given an index pattern that matches even if the indices are different order', () => {
+    expect(
+      getSettingsMatch({
+        indices: [
+          'endgame-*',
+          'filebeat-*',
+          'logs-*',
+          'auditbeat-*',
+          'packetbeat-*',
+          'winlogbeat-*',
+        ],
+        transformSettings: getTransformConfigSchemaMock(),
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(getTransformConfigSchemaMock().settings[0]);
+  });
+
+  test('it returns a setting given an index pattern that matches and removes any that have a dash in them meaning to subtract them', () => {
+    expect(
+      getSettingsMatch({
+        indices: [
+          'endgame-*',
+          'filebeat-*',
+          'logs-*',
+          'auditbeat-*',
+          'packetbeat-*',
+          'winlogbeat-*',
+          '-subtract-1', // extra dashed one that should still allow a match
+          '-subtract-2', // extra dashed one that should still allow a match
+        ],
+        transformSettings: getTransformConfigSchemaMock(),
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(getTransformConfigSchemaMock().settings[0]);
+  });
+
+  test('it returns "undefined" given a set of indices that do not match a setting', () => {
+    expect(
+      getSettingsMatch({
+        indices: ['endgame-*', 'filebeat-*', 'logs-*', 'auditbeat-*', 'packetbeat-*'],
+        transformSettings: getTransformConfigSchemaMock(),
+      })
+    ).toEqual<ReturnTypeCreateIndicesFromPrefix>(undefined);
+  });
+});

x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts

@@ -7,6 +7,12 @@
 
 import { TransformConfigSchema } from '../../../common/transforms/types';
 
+/**
+ * Given a transform setting and indices this will return either the particular setting
+ * that matches the index or it will return undefined if it is not found
+ * @param indices The indices to check against the transform
+ * @returns Either the setting if it matches or an undefined if it cannot find one
+ */
 export const getSettingsMatch = ({
   indices,
   transformSettings,