[ES|QL] [Discover] Creating where clause filters from the table, sidebar and table row viewer

packages/kbn-esql-utils/README.md

@@ -6,4 +6,6 @@ This package contains utilities for ES|QL.
 - *getIndexPatternFromESQLQuery*: Use this to retrieve the index pattern from the `from` command.
 - *getLimitFromESQLQuery*: Use this function to get the limit for a given query. The limit can be either set from the `limit` command or can be a default value set in ES.
 - *removeDropCommandsFromESQLQuery*: Use this function to remove all the occurences of the `drop` command from the query.
+- *appendToESQLQuery*: Use this function to append more pipes in an existing ES|QL query. It adds the additional commands in a new line. 
+- *appendWhereClauseToESQLQuery*: Use this function to append where clause in an existing query. 
 

packages/kbn-esql-utils/index.ts

@@ -16,6 +16,7 @@ export {
   getInitialESQLQuery,
   getESQLWithSafeLimit,
   appendToESQLQuery,
+  appendWhereClauseToESQLQuery,
   TextBasedLanguages,
 } from './src';
 

packages/kbn-esql-utils/src/index.ts

@@ -16,4 +16,4 @@ export {
   getLimitFromESQLQuery,
   removeDropCommandsFromESQLQuery,
 } from './utils/query_parsing_helpers';
-export { appendToESQLQuery } from './utils/append_to_query';
+export { appendToESQLQuery, appendWhereClauseToESQLQuery } from './utils/append_to_query';

packages/kbn-esql-utils/src/utils/append_to_query.test.ts

@@ -5,21 +5,139 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import { appendToESQLQuery } from './append_to_query';
+import { appendToESQLQuery, appendWhereClauseToESQLQuery } from './append_to_query';
 
-describe('appendToESQLQuery', () => {
-  it('append the text on a new line after the query', () => {
-    expect(appendToESQLQuery('from logstash-* // meow', '| stats var = avg(woof)')).toBe(
-      `from logstash-* // meow
+describe('appendToQuery', () => {
+  describe('appendToESQLQuery', () => {
+    it('append the text on a new line after the query', () => {
+      expect(appendToESQLQuery('from logstash-* // meow', '| stats var = avg(woof)')).toBe(
+        `from logstash-* // meow
 | stats var = avg(woof)`
-    );
-  });
+      );
+    });
 
-  it('append the text on a new line after the query for text with variables', () => {
-    const limit = 10;
-    expect(appendToESQLQuery('from logstash-*', `| limit ${limit}`)).toBe(
-      `from logstash-*
+    it('append the text on a new line after the query for text with variables', () => {
+      const limit = 10;
+      expect(appendToESQLQuery('from logstash-*', `| limit ${limit}`)).toBe(
+        `from logstash-*
 | limit 10`
-    );
+      );
+    });
+  });
+
+  describe('appendWhereClauseToESQLQuery', () => {
+    it('appends a filter in where clause in an existing query', () => {
+      expect(
+        appendWhereClauseToESQLQuery('from logstash-* // meow', 'dest', 'tada!', '+', 'string')
+      ).toBe(
+        `from logstash-* // meow
+| where \`dest\`=="tada!"`
+      );
+    });
+    it('appends a filter out where clause in an existing query', () => {
+      expect(
+        appendWhereClauseToESQLQuery('from logstash-* // meow', 'dest', 'tada!', '-', 'string')
+      ).toBe(
+        `from logstash-* // meow
+| where \`dest\`!="tada!"`
+      );
+    });
+
+    it('appends a where clause in an existing query with casting to string when the type is not string or number', () => {
+      expect(
+        appendWhereClauseToESQLQuery('from logstash-* // meow', 'dest', 'tada!', '-', 'ip')
+      ).toBe(
+        `from logstash-* // meow
+| where \`dest\`::string!="tada!"`
+      );
+    });
+
+    it('appends a where clause in an existing query with casting to string when the type is not given', () => {
+      expect(appendWhereClauseToESQLQuery('from logstash-* // meow', 'dest', 'tada!', '-')).toBe(
+        `from logstash-* // meow
+| where \`dest\`::string!="tada!"`
+      );
+    });
+
+    it('appends a where clause in an existing query checking that the value is not null if the user asks for existence', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          'from logstash-* // meow',
+          'dest',
+          undefined,
+          '_exists_',
+          'string'
+        )
+      ).toBe(
+        `from logstash-* // meow
+| where \`dest\` is not null`
+      );
+    });
+
+    it('appends an and clause in an existing query with where command as the last pipe', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          'from logstash-* | where country == "GR"',
+          'dest',
+          'Crete',
+          '+',
+          'string'
+        )
+      ).toBe(
+        `from logstash-* | where country == "GR"
+and \`dest\`=="Crete"`
+      );
+    });
+
+    it('doesnt append anything in an existing query with where command as the last pipe if the filter preexists', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          'from logstash-* | where country == "GR"',
+          'country',
+          'GR',
+          '+',
+          'string'
+        )
+      ).toBe(`from logstash-* | where country == "GR"`);
+    });
+
+    it('changes the operator in an existing query with where command as the last pipe if the filter preexists but has the opposite operator', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          'from logstash-* | where country == "GR"',
+          'country',
+          'GR',
+          '-',
+          'string'
+        )
+      ).toBe(`from logstash-* | where country != "GR"`);
+    });
+
+    it('changes the operator in an existing query with where command as the last pipe if the filter preexists but has the opposite operator, the field has backticks', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          'from logstash-* | where `country` == "GR"',
+          'country',
+          'GR',
+          '-',
+          'string'
+        )
+      ).toBe(`from logstash-* | where \`country\` != "GR"`);
+    });
+
+    it('appends an and clause in an existing query with where command as the last pipe if the filter preexists but the operator is not the correct one', () => {
+      expect(
+        appendWhereClauseToESQLQuery(
+          `from logstash-* | where CIDR_MATCH(ip1, "127.0.0.2/32", "127.0.0.3/32")`,
+          'ip',
+          '127.0.0.2/32',
+          '-',
+          'ip'
+        )
+      ).toBe(
+        `from logstash-* | where CIDR_MATCH(ip1, "127.0.0.2/32", "127.0.0.3/32")
+and \`ip\`::string!="127.0.0.2/32"`
+      );
+    });
   });
 });

packages/kbn-esql-utils/src/utils/append_to_query.ts

@@ -5,9 +5,131 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+import {
+  type ESQLFunction,
+  type ESQLSingleAstItem,
+  type ESQLColumn,
+  type ESQLLiteral,
+  getAstAndSyntaxErrors,
+} from '@kbn/esql-ast';
+
+interface FilterArguments {
+  fieldName: string;
+  value: string;
+  operator: string;
+}
 
 // Append in a new line the appended text to take care of the case where the user adds a comment at the end of the query
 // in these cases a base query such as "from index // comment" will result in errors or wrong data if we don't append in a new line
 export function appendToESQLQuery(baseESQLQuery: string, appendedText: string): string {
   return `${baseESQLQuery}\n${appendedText}`;
 }
+
+function findInAst(
+  astItem: ESQLFunction,
+  fieldName: string,
+  value: string
+): FilterArguments | undefined {
+  const singleAstItem = astItem as ESQLFunction;
+  const args = singleAstItem.args;
+  const isLastNode = args?.some((a) => 'type' in a && a.type === 'column');
+  if (isLastNode) {
+    const column = args.find((a) => {
+      const argument = a as ESQLSingleAstItem;
+      return argument.type === 'column';
+    }) as ESQLColumn;
+    const literal = args.find((a) => {
+      const argument = a as ESQLSingleAstItem;
+      return argument.type === 'literal';
+    }) as ESQLLiteral;
+    if (column?.name === fieldName && literal?.name.includes(value)) {
+      return {
+        fieldName,
+        value,
+        operator: astItem.name,
+      };
+    } else {
+      return undefined;
+    }
+  } else {
+    const functions = args?.filter((item) => 'type' in item && item.type === 'function');
+    for (const functionAstItem of functions) {
+      const item = findInAst(functionAstItem as ESQLFunction, fieldName, value);
+      if (item) {
+        return item;
+      }
+    }
+  }
+}
+
+export function appendWhereClauseToESQLQuery(
+  baseESQLQuery: string,
+  field: string,
+  value: unknown,
+  operation: '+' | '-' | '_exists_',
+  fieldType?: string
+): string {
+  let operator = operation === '+' ? '==' : '!=';
+  let filterValue = typeof value === 'string' ? `"${value.replace(/"/g, '\\"')}"` : value;
+  // Adding the backticks here are they are needed for special char fields
+  let fieldName = `\`${field}\``;
+
+  // casting to string
+  // there are some field types such as the ip that need
+  // to cast in string first otherwise ES will fail
+  if (fieldType !== 'string' && fieldType !== 'number' && fieldType !== 'boolean') {
+    fieldName = `${fieldName}::string`;
+  }
+
+  // checking that the value is not null
+  // this is the existence filter
+  if (operation === '_exists_') {
+    fieldName = `\`${String(field)}\``;
+    operator = ' is not null';
+    filterValue = '';
+  }
+
+  const { ast } = getAstAndSyntaxErrors(baseESQLQuery);
+
+  const lastCommandIsWhere = ast[ast.length - 1].name === 'where';
+  // if where command already exists in the end of the query:
+  // - we need to append with and if the filter doesnt't exist
+  // - we need to change the filter operator if the filter exists with different operator
+  // - we do nothing if the filter exists with the same operator
+  if (lastCommandIsWhere) {
+    const whereCommand = ast[ast.length - 1];
+    const whereAstText = whereCommand.text;
+    // the filter already exists in the where clause
+    if (whereAstText.includes(field) && whereAstText.includes(String(filterValue))) {
+      const pipesArray = baseESQLQuery.split('|');
+      const whereClause = pipesArray[pipesArray.length - 1];
+
+      const args = ast[ast.length - 1].args[0] as ESQLFunction;
+      const astItem = findInAst(args, field, String(value));
+
+      if (astItem) {
+        const existingOperator = astItem.operator;
+        if (!['==', '!='].includes(existingOperator.trim())) {
+          return appendToESQLQuery(baseESQLQuery, `and ${fieldName}${operator}${filterValue}`);
+        }
+        // the filter is the same
+        if (existingOperator === operator) {
+          return baseESQLQuery;
+          // the filter has different operator
+        } else {
+          const matches = whereClause.match(new RegExp(field + '(.*)' + String(filterValue)));
+          if (matches) {
+            const existingFilter = matches[0].trim();
+            const newFilter = existingFilter.replace(existingOperator, operator);
+            return baseESQLQuery.replace(existingFilter, newFilter);
+          }
+        }
+      }
+    }
+    // filter does not exist in the where clause
+    const whereClause = `and ${fieldName}${operator}${filterValue}`;
+    return appendToESQLQuery(baseESQLQuery, whereClause);
+  }
+  const whereClause = `| where ${fieldName}${operator}${filterValue}`;
+  return appendToESQLQuery(baseESQLQuery, whereClause);
+}