From f3d557ed68b8a5c5992188fbba7243342b97fb63 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Fri, 15 Dec 2017 16:26:28 -0700 Subject: [PATCH 01/29] Introduce simple kuery language --- .../public/dashboard/dashboard_app.html | 2 +- .../kibana/public/discover/index.html | 2 +- .../public/visualize/editor/editor.html | 2 +- .../public/coordinate_maps_visualization.js | 3 +- .../data_source/build_query/build_es_query.js | 9 +- .../data_source/build_query/from_kuery.js | 6 +- src/ui/public/doc_table/actions/filter.js | 2 +- .../filter_bar/filter_bar_click_handler.js | 3 +- src/ui/public/kuery/ast/ast.js | 8 +- src/ui/public/kuery/ast/simple_kuery.peg | 123 ++++++++++++++++++ .../public/query_bar/directive/query_bar.html | 17 +++ src/ui/public/query_bar/lib/queryLanguages.js | 1 + 12 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 src/ui/public/kuery/ast/simple_kuery.peg diff --git a/src/core_plugins/kibana/public/dashboard/dashboard_app.html b/src/core_plugins/kibana/public/dashboard/dashboard_app.html index ac2057236a089..1e331a2c5d141 100644 --- a/src/core_plugins/kibana/public/dashboard/dashboard_app.html +++ b/src/core_plugins/kibana/public/dashboard/dashboard_app.html @@ -39,7 +39,7 @@ ng-show="showFilterBar()" state="state" index-patterns="indexPatterns" - ng-if="model.query.language === 'lucene'" + ng-if="['lucene', 'simpleKuery'].includes(model.query.language)" >
diff --git a/src/core_plugins/kibana/public/visualize/editor/editor.html b/src/core_plugins/kibana/public/visualize/editor/editor.html index 8e337b632909d..f5fab136b3cf1 100644 --- a/src/core_plugins/kibana/public/visualize/editor/editor.html +++ b/src/core_plugins/kibana/public/visualize/editor/editor.html @@ -45,7 +45,7 @@ diff --git a/src/core_plugins/tile_map/public/coordinate_maps_visualization.js b/src/core_plugins/tile_map/public/coordinate_maps_visualization.js index 3b1951d49f071..f353dff393169 100644 --- a/src/core_plugins/tile_map/public/coordinate_maps_visualization.js +++ b/src/core_plugins/tile_map/public/coordinate_maps_visualization.js @@ -120,7 +120,7 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) { const query = this.vis.API.queryManager.getQuery(); const language = query.language; - if (language === 'lucene') { + if (['lucene', 'simpleKuery'].includes(language)) { const filter = { meta: { negate: false, index: indexPatternName } }; filter[filterName] = { ignore_unmapped: true }; filter[filterName][field] = filterData; @@ -198,4 +198,3 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) { return CoordinateMapsVisualization; } - diff --git a/src/ui/public/courier/data_source/build_query/build_es_query.js b/src/ui/public/courier/data_source/build_query/build_es_query.js index 4beeb4f91d258..ce4452e1f8ad9 100644 --- a/src/ui/public/courier/data_source/build_query/build_es_query.js +++ b/src/ui/public/courier/data_source/build_query/build_es_query.js @@ -17,15 +17,16 @@ export function BuildESQueryProvider(Private) { const queriesByLanguage = groupBy(validQueries, 'language'); const kueryQuery = buildQueryFromKuery(indexPattern, queriesByLanguage.kuery); + const simpleKueryQuery = buildQueryFromKuery(indexPattern, queriesByLanguage.simpleKuery, true); const luceneQuery = buildQueryFromLucene(queriesByLanguage.lucene, decorateQuery); const filterQuery = buildQueryFromFilters(filters, decorateQuery); return { bool: { - must: [].concat(kueryQuery.must, luceneQuery.must, filterQuery.must), - filter: [].concat(kueryQuery.filter, luceneQuery.filter, filterQuery.filter), - should: [].concat(kueryQuery.should, luceneQuery.should, filterQuery.should), - must_not: [].concat(kueryQuery.must_not, luceneQuery.must_not, filterQuery.must_not), + must: [].concat(kueryQuery.must, simpleKueryQuery.must, luceneQuery.must, filterQuery.must), + filter: [].concat(kueryQuery.filter, simpleKueryQuery.filter, luceneQuery.filter, filterQuery.filter), + should: [].concat(kueryQuery.should, simpleKueryQuery.should, luceneQuery.should, filterQuery.should), + must_not: [].concat(kueryQuery.must_not, simpleKueryQuery.must_not, luceneQuery.must_not, filterQuery.must_not), } }; } diff --git a/src/ui/public/courier/data_source/build_query/from_kuery.js b/src/ui/public/courier/data_source/build_query/from_kuery.js index 4801f7fca2956..d591aaf9f88b4 100644 --- a/src/ui/public/courier/data_source/build_query/from_kuery.js +++ b/src/ui/public/courier/data_source/build_query/from_kuery.js @@ -1,8 +1,8 @@ import _ from 'lodash'; import { fromKueryExpression, toElasticsearchQuery, nodeTypes } from '../../../kuery'; -export function buildQueryFromKuery(indexPattern, queries) { - const queryASTs = _.map(queries, query => fromKueryExpression(query.query)); +export function buildQueryFromKuery(indexPattern, queries, useSimple = false) { + const queryASTs = _.map(queries, query => fromKueryExpression(query.query, undefined, useSimple)); const compoundQueryAST = nodeTypes.function.buildNode('and', queryASTs); const kueryQuery = toElasticsearchQuery(compoundQueryAST, indexPattern); return { @@ -13,5 +13,3 @@ export function buildQueryFromKuery(indexPattern, queries) { ...kueryQuery.bool }; } - - diff --git a/src/ui/public/doc_table/actions/filter.js b/src/ui/public/doc_table/actions/filter.js index 6df9f314e2b48..1ef530dee1d44 100644 --- a/src/ui/public/doc_table/actions/filter.js +++ b/src/ui/public/doc_table/actions/filter.js @@ -8,7 +8,7 @@ export function addFilter(field, values = [], operation, index, state, filterMan values = [values]; } - if (state.query.language === 'lucene') { + if (['lucene', 'simpleKuery'].includes(state.query.language)) { filterManager.add(field, values, operation, index); } diff --git a/src/ui/public/filter_bar/filter_bar_click_handler.js b/src/ui/public/filter_bar/filter_bar_click_handler.js index 814f295ce1b63..dce06cd3879d3 100644 --- a/src/ui/public/filter_bar/filter_bar_click_handler.js +++ b/src/ui/public/filter_bar/filter_bar_click_handler.js @@ -64,7 +64,7 @@ export function FilterBarClickHandlerProvider(Notifier, Private) { filters = dedupFilters($state.filters, uniqFilters(filters), { negate: true }); if (!simulate) { - if ($state.query.language === 'lucene') { + if (['lucene', 'simpleKuery'].includes($state.query.language)) { $state.$newFilters = filters; } else if ($state.query.language === 'kuery') { @@ -81,4 +81,3 @@ export function FilterBarClickHandlerProvider(Notifier, Private) { }; }; } - diff --git a/src/ui/public/kuery/ast/ast.js b/src/ui/public/kuery/ast/ast.js index 214fb5ed58838..6a0ea4ad54945 100644 --- a/src/ui/public/kuery/ast/ast.js +++ b/src/ui/public/kuery/ast/ast.js @@ -1,21 +1,25 @@ import grammar from 'raw-loader!./kuery.peg'; +import simpleGrammar from 'raw-loader!./simple_kuery.peg'; import PEG from 'pegjs'; import _ from 'lodash'; import { nodeTypes } from '../node_types/index'; const kueryParser = PEG.buildParser(grammar); +const simpleKueryParser = PEG.buildParser(simpleGrammar); -export function fromKueryExpression(expression, parseOptions = {}) { +export function fromKueryExpression(expression, parseOptions = {}, useSimpleParser = false) { if (_.isUndefined(expression)) { throw new Error('expression must be a string, got undefined instead'); } + const parser = useSimpleParser ? simpleKueryParser : kueryParser; + parseOptions = { ...parseOptions, helpers: { nodeTypes } }; - return kueryParser.parse(expression, parseOptions); + return parser.parse(expression, parseOptions); } export function toKueryExpression(node) { diff --git a/src/ui/public/kuery/ast/simple_kuery.peg b/src/ui/public/kuery/ast/simple_kuery.peg new file mode 100644 index 0000000000000..eb1c6034a366d --- /dev/null +++ b/src/ui/public/kuery/ast/simple_kuery.peg @@ -0,0 +1,123 @@ +/* + * Kuery parser + */ + +/* + * Initialization block + */ +{ + var nodeTypes = options.helpers.nodeTypes; + + if (options.includeMetadata === undefined) { + options.includeMetadata = true; + } + + function addMeta(source, text, location) { + if (options.includeMetadata) { + return Object.assign( + {}, + source, + { + text: text, + location: simpleLocation(location), + } + ); + } + + return source; + } + + function simpleLocation(location) { + // Returns an object representing the position of the function within the expression, + // demarcated by the position of its first character and last character. We calculate these values + // using the offset because the expression could span multiple lines, and we don't want to deal + // with column and line values. + return { + min: location.start.offset, + max: location.end.offset + } + } +} + +start + = Query + / space* { + return addMeta(nodeTypes.function.buildNode('and', []), text(), location()); + } + +Query + = space? query:OrQuery space? { + if (query.type === 'literal') { + return addMeta(nodeTypes.function.buildNode('and', [query]), text(), location()); + } + return query; + } + +OrQuery + = left:AndQuery space 'or'i space right:OrQuery { + return addMeta(nodeTypes.function.buildNode('or', [left, right]), text(), location()); + } + / AndQuery + +AndQuery + = left:NotQuery space 'and'i space right:AndQuery { + return addMeta(nodeTypes.function.buildNode('and', [left, right]), text(), location()); + } + / NotQuery + +NotQuery + = 'not'i space clause:Clause { + return addMeta(nodeTypes.function.buildNode('not', clause), text(), location()); + } + / Clause + +Clause + = '(' subQuery:Query ')' { + return subQuery; + } + / Term + +Term + = field:literal_arg_type space? ':' space? value:literal_arg_type { + return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('is', [field, value]), text(), location()); + } + / field:literal_arg_type space? ':' space? '[' space? gt:literal_arg_type space 'to'i space lt:literal_arg_type space? ']' { + return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('range', [field, gt, lt]), text(), location()); + } + / !Keywords literal:literal_arg_type { return literal; } + +literal_arg_type + = literal:literal { + var result = addMeta(nodeTypes.literal.buildNode(literal), text(), location()); + return result; + } + +Keywords + = 'or'i / 'and'i / 'not'i + + /* ----- Core types ----- */ + +literal "literal" + = '"' chars:dq_char* '"' { return chars.join(''); } // double quoted string + / "'" chars:sq_char* "'" { return chars.join(''); } // single quoted string + / 'true' { return true; } // unquoted literals from here down + / 'false' { return false; } + / 'null' { return null; } + / string:[^\[\]()"',:=\ \t]+ { // this also matches numbers via Number() + var result = string.join(''); + // Sort of hacky, but PEG doesn't have backtracking so + // a number rule is hard to read, and performs worse + if (isNaN(Number(result))) return result; + return Number(result) + } + +space + = [\ \t\r\n]+ + +dq_char + = "\\" sequence:('"' / "\\") { return sequence; } + / [^"] // everything except " + +sq_char + = "\\" sequence:("'" / "\\") { return sequence; } + / [^'] // everything except ' diff --git a/src/ui/public/query_bar/directive/query_bar.html b/src/ui/public/query_bar/directive/query_bar.html index 3199f5d528857..38eb2f9642bb7 100644 --- a/src/ui/public/query_bar/directive/query_bar.html +++ b/src/ui/public/query_bar/directive/query_bar.html @@ -67,6 +67,23 @@
+ +
+ +
+ Date: Thu, 4 Jan 2018 15:46:08 -0700 Subject: [PATCH 03/29] Update KQL syntax --- src/ui/public/kuery/ast/kql.peg | 219 ++++++++++++++++++-------------- 1 file changed, 123 insertions(+), 96 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index eb1c6034a366d..f3498c536e797 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -1,123 +1,150 @@ -/* - * Kuery parser - */ - -/* - * Initialization block - */ +// Initialization block { - var nodeTypes = options.helpers.nodeTypes; + var { nodeTypes: { function: { buildNode }}} = options.helpers; +} + +start + = OrQuery + / EmptyQuery - if (options.includeMetadata === undefined) { - options.includeMetadata = true; +EmptyQuery + = Space* { + return buildNode('and', []); } - function addMeta(source, text, location) { - if (options.includeMetadata) { - return Object.assign( - {}, - source, - { - text: text, - location: simpleLocation(location), - } - ); - } +OrQuery + = head:AndQuery Or tail:OrQuery { + return buildNode('or', [head, tail]); + } + / AndQuery - return source; +AndQuery + = head:NotQuery And tail:AndQuery{ + return buildNode('and', [head, tail]); } + / NotQuery - function simpleLocation(location) { - // Returns an object representing the position of the function within the expression, - // demarcated by the position of its first character and last character. We calculate these values - // using the offset because the expression could span multiple lines, and we don't want to deal - // with column and line values. - return { - min: location.start.offset, - max: location.end.offset +NotQuery + = Not query:SubQuery { + return buildNode('not', query); + } + / SubQuery + +SubQuery + = '(' Space* query:OrQuery Space* ')' { return query; } + / Expression + +Expression + = FieldListExpression + / FieldRangeExpression + / ValueExpression + +FieldListExpression + = field:Literal Space* ':' Space* list:SubList { + function buildNodeWithField(field, list) { + if (list.hasOwnProperty('and')) { + return buildNode('and', list.and.map(node => buildNodeWithField(field, node))); + } else if (list.hasOwnProperty('or')) { + return buildNode('or', list.or.map(node => buildNodeWithField(field, node))); + } else if (list.hasOwnProperty('not')) { + return buildNode('not', buildNodeWithField(field, list.not)); + } else { + return buildNode('is', field, list); + } } + return buildNodeWithField(field, list); } -} -start - = Query - / space* { - return addMeta(nodeTypes.function.buildNode('and', []), text(), location()); +FieldRangeExpression + = field:Literal Space* operator:RangeOperator Space* value:Literal { + return buildNode('range', field, { [operator]: value }); } -Query - = space? query:OrQuery space? { - if (query.type === 'literal') { - return addMeta(nodeTypes.function.buildNode('and', [query]), text(), location()); - } - return query; +ValueExpression + = value:Literal { + return buildNode('is', '*', value); } -OrQuery - = left:AndQuery space 'or'i space right:OrQuery { - return addMeta(nodeTypes.function.buildNode('or', [left, right]), text(), location()); +OrList + = head:AndList Or tail:OrList { + return { or: [head, tail] }; } - / AndQuery + / AndList -AndQuery - = left:NotQuery space 'and'i space right:AndQuery { - return addMeta(nodeTypes.function.buildNode('and', [left, right]), text(), location()); +AndList + = head:NotList And tail:AndList{ + return { and: [head, tail] }; } - / NotQuery + / NotList -NotQuery - = 'not'i space clause:Clause { - return addMeta(nodeTypes.function.buildNode('not', clause), text(), location()); +NotList + = Not list:SubList { + return { not: list }; } - / Clause + / SubList -Clause - = '(' subQuery:Query ')' { - return subQuery; - } - / Term +SubList + = '(' Space* list:OrList Space* ')' { return list; } + / Literal + +Or + = Space+ 'or'i Space+ + +And + = Space+ 'and'i Space+ + +Not + = 'not'i Space+ -Term - = field:literal_arg_type space? ':' space? value:literal_arg_type { - return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('is', [field, value]), text(), location()); +Literal + = SingleQuotedString + / DoubleQuotedString + / UnquotedLiteral + +SingleQuotedString + = "'" chars:(EscapedQuote / [^'])* "'" { + return chars.join(''); } - / field:literal_arg_type space? ':' space? '[' space? gt:literal_arg_type space 'to'i space lt:literal_arg_type space? ']' { - return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('range', [field, gt, lt]), text(), location()); + +DoubleQuotedString + = '"' chars:(EscapedQuote / [^"])* '"' { + return chars.join(''); } - / !Keywords literal:literal_arg_type { return literal; } - -literal_arg_type - = literal:literal { - var result = addMeta(nodeTypes.literal.buildNode(literal), text(), location()); - return result; - } - -Keywords - = 'or'i / 'and'i / 'not'i - - /* ----- Core types ----- */ - -literal "literal" - = '"' chars:dq_char* '"' { return chars.join(''); } // double quoted string - / "'" chars:sq_char* "'" { return chars.join(''); } // single quoted string - / 'true' { return true; } // unquoted literals from here down - / 'false' { return false; } - / 'null' { return null; } - / string:[^\[\]()"',:=\ \t]+ { // this also matches numbers via Number() - var result = string.join(''); - // Sort of hacky, but PEG doesn't have backtracking so - // a number rule is hard to read, and performs worse - if (isNaN(Number(result))) return result; - return Number(result) + +UnquotedLiteral + = chars:UnquotedCharacter+ { + const sequence = chars.join('').trim(); + if (sequence === 'null') return null; + if (sequence === 'true') return true; + if (sequence === 'false') return false; + const number = Number(sequence); + return isNaN(number) ? sequence : number; } -space - = [\ \t\r\n]+ +EscapedQuote + = '\\' quote:('"' / '"') { return quote; } + +UnquotedCharacter + = EscapedSpecialCharacter + / !Separator char:. { return char; } + +EscapedSpecialCharacter + = '\\' char:SpecialCharacter { return char; } + +Separator + = Keyword / SpecialCharacter + +Keyword + = Or / And / Not + +SpecialCharacter + = [():<>] -dq_char - = "\\" sequence:('"' / "\\") { return sequence; } - / [^"] // everything except " +RangeOperator + = '<=' { return 'lte'; } + / '>=' { return 'gte'; } + / '<' { return 'lt'; } + / '>' { return 'gt'; } -sq_char - = "\\" sequence:("'" / "\\") { return sequence; } - / [^'] // everything except ' +Space + = [\ \t\r\n] From 162a49749d98cf46cc0c323966f0f6677fe066f1 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 5 Feb 2018 15:39:44 -0700 Subject: [PATCH 04/29] Update terminology to be clearer --- src/ui/public/kuery/ast/kql.peg | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index f3498c536e797..36ce26903701f 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -13,14 +13,14 @@ EmptyQuery } OrQuery - = head:AndQuery Or tail:OrQuery { - return buildNode('or', [head, tail]); + = left:AndQuery Or right:OrQuery { + return buildNode('or', [left, right]); } / AndQuery AndQuery - = head:NotQuery And tail:AndQuery{ - return buildNode('and', [head, tail]); + = left:NotQuery And right:AndQuery{ + return buildNode('and', [left, right]); } / NotQuery @@ -66,14 +66,14 @@ ValueExpression } OrList - = head:AndList Or tail:OrList { - return { or: [head, tail] }; + = left:AndList Or right:OrList { + return { or: [left, right] }; } / AndList AndList - = head:NotList And tail:AndList{ - return { and: [head, tail] }; + = left:NotList And right:AndList{ + return { and: [left, right] }; } / NotList From 2d86135990f7ccd641bd56b0ca3c737f327eea73 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 5 Feb 2018 15:42:43 -0700 Subject: [PATCH 05/29] Fix typo --- src/ui/public/kuery/ast/kql.peg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index 36ce26903701f..ba3e60a57584f 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -122,7 +122,7 @@ UnquotedLiteral } EscapedQuote - = '\\' quote:('"' / '"') { return quote; } + = '\\' quote:('"' / "'") { return quote; } UnquotedCharacter = EscapedSpecialCharacter From f7c9a54a2818ee938d37004445a1f746d8279744 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 5 Feb 2018 16:00:14 -0700 Subject: [PATCH 06/29] Simplify building of nodes --- src/ui/public/kuery/ast/kql.peg | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index ba3e60a57584f..0a96319fc0924 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -41,18 +41,15 @@ Expression FieldListExpression = field:Literal Space* ':' Space* list:SubList { - function buildNodeWithField(field, list) { - if (list.hasOwnProperty('and')) { - return buildNode('and', list.and.map(node => buildNodeWithField(field, node))); - } else if (list.hasOwnProperty('or')) { - return buildNode('or', list.or.map(node => buildNodeWithField(field, node))); - } else if (list.hasOwnProperty('not')) { - return buildNode('not', buildNodeWithField(field, list.not)); + function buildNodeWithField(node) { + if (node.hasOwnProperty('arguments')) { + const args = node.arguments.map(buildNodeWithField); + return { ...list, arguments: args }; } else { - return buildNode('is', field, list); + return buildNode('is', field, node); } } - return buildNodeWithField(field, list); + return buildNodeWithField(list); } FieldRangeExpression @@ -67,19 +64,19 @@ ValueExpression OrList = left:AndList Or right:OrList { - return { or: [left, right] }; + return buildNode('or', [left, right]); } / AndList AndList = left:NotList And right:AndList{ - return { and: [left, right] }; + return buildNode('and', [left, right]); } / NotList NotList = Not list:SubList { - return { not: list }; + return buildNode('not', query);; } / SubList From 9abab5d188ec99fa940372f5023e708e3a8e1c42 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Fri, 9 Feb 2018 11:22:21 -0700 Subject: [PATCH 07/29] Fix typos --- src/ui/public/kuery/ast/kql.peg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index 0a96319fc0924..e484cef434df5 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -59,7 +59,7 @@ FieldRangeExpression ValueExpression = value:Literal { - return buildNode('is', '*', value); + return buildNode('is', null, value); } OrList @@ -76,7 +76,7 @@ AndList NotList = Not list:SubList { - return buildNode('not', query);; + return buildNode('not', list); } / SubList From 53db30acb5b1b79684d69b7b4cb4c34382afc047 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Fri, 9 Feb 2018 17:23:03 -0700 Subject: [PATCH 08/29] Build up AST for sublist by returning functions that take a field name --- src/ui/public/kuery/ast/kql.peg | 62 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index e484cef434df5..8ecb7f617726d 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -35,54 +35,52 @@ SubQuery / Expression Expression - = FieldListExpression - / FieldRangeExpression + = FieldRangeExpression + / FieldValueExpression / ValueExpression -FieldListExpression - = field:Literal Space* ':' Space* list:SubList { - function buildNodeWithField(node) { - if (node.hasOwnProperty('arguments')) { - const args = node.arguments.map(buildNodeWithField); - return { ...list, arguments: args }; - } else { - return buildNode('is', field, node); - } - } - return buildNodeWithField(list); - } - FieldRangeExpression = field:Literal Space* operator:RangeOperator Space* value:Literal { return buildNode('range', field, { [operator]: value }); } +FieldValueExpression + = field:Literal Space* ':' Space* partial:ListOfValues { + return partial(field); + } + ValueExpression - = value:Literal { - return buildNode('is', null, value); + = partial:Value { + const field = null; + return partial(field); } -OrList - = left:AndList Or right:OrList { - return buildNode('or', [left, right]); +ListOfValues + = '(' Space* partial:OrListOfValues Space* ')' { return partial; } + / Value + +OrListOfValues + = partialLeft:AndListOfValues Or partialRight:OrListOfValues { + return (field) => buildNode('or', [partialLeft(field), partialRight(field)]); } - / AndList + / AndListOfValues -AndList - = left:NotList And right:AndList{ - return buildNode('and', [left, right]); +AndListOfValues + = partialLeft:NotListOfValues And partialRight:AndListOfValues { + return (field) => buildNode('and', [partialLeft(field), partialRight(field)]); } - / NotList + / NotListOfValues -NotList - = Not list:SubList { - return buildNode('not', list); +NotListOfValues + = Not partial:ListOfValues { + return (field) => buildNode('not', partial(field)); } - / SubList + / ListOfValues -SubList - = '(' Space* list:OrList Space* ')' { return list; } - / Literal +Value + = value:Literal { + return (field) => buildNode('is', field, value); + } Or = Space+ 'or'i Space+ From 6ee08ca80edc5aff4eb2eec827fbf6ec45c7793b Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 12 Feb 2018 10:17:37 -0700 Subject: [PATCH 09/29] Remove single quoted strings and add double quote to special characters --- src/ui/public/kuery/ast/kql.peg | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index 8ecb7f617726d..51f11dbf987ca 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -92,17 +92,11 @@ Not = 'not'i Space+ Literal - = SingleQuotedString - / DoubleQuotedString + = QuotedString / UnquotedLiteral -SingleQuotedString - = "'" chars:(EscapedQuote / [^'])* "'" { - return chars.join(''); - } - -DoubleQuotedString - = '"' chars:(EscapedQuote / [^"])* '"' { +QuotedString + = '"' chars:(EscapedSpecialCharacter / [^"])* '"' { return chars.join(''); } @@ -116,9 +110,6 @@ UnquotedLiteral return isNaN(number) ? sequence : number; } -EscapedQuote - = '\\' quote:('"' / "'") { return quote; } - UnquotedCharacter = EscapedSpecialCharacter / !Separator char:. { return char; } @@ -133,7 +124,7 @@ Keyword = Or / And / Not SpecialCharacter - = [():<>] + = [():<>"] RangeOperator = '<=' { return 'lte'; } From 668c87ae8df4b42ec21ee735cdd82cbf87e6caa4 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 12 Feb 2018 10:45:16 -0700 Subject: [PATCH 10/29] Build nodes with arg nodes instead of args themselves --- src/ui/public/kuery/ast/kql.peg | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index 51f11dbf987ca..f92e7f3278b7c 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -1,6 +1,9 @@ // Initialization block { - var { nodeTypes: { function: { buildNode }}} = options.helpers; + const { nodeTypes } = options.helpers; + const buildFunctionNode = nodeTypes.function.buildNodeWithArgumentNodes; + const buildLiteralNode = nodeTypes.literal.buildNode; + const buildNamedArgNode = nodeTypes.namedArg.buildNode; } start @@ -9,24 +12,24 @@ start EmptyQuery = Space* { - return buildNode('and', []); + return buildFunctionNode('and', []); } OrQuery = left:AndQuery Or right:OrQuery { - return buildNode('or', [left, right]); + return buildFunctionNode('or', [left, right]); } / AndQuery AndQuery = left:NotQuery And right:AndQuery{ - return buildNode('and', [left, right]); + return buildFunctionNode('and', [left, right]); } / NotQuery NotQuery = Not query:SubQuery { - return buildNode('not', query); + return buildFunctionNode('not', [query]); } / SubQuery @@ -41,7 +44,8 @@ Expression FieldRangeExpression = field:Literal Space* operator:RangeOperator Space* value:Literal { - return buildNode('range', field, { [operator]: value }); + const range = buildNamedArgNode(operator, value); + return buildFunctionNode('range', [field, range]); } FieldValueExpression @@ -51,7 +55,7 @@ FieldValueExpression ValueExpression = partial:Value { - const field = null; + const field = buildLiteralNode(null); return partial(field); } @@ -61,25 +65,25 @@ ListOfValues OrListOfValues = partialLeft:AndListOfValues Or partialRight:OrListOfValues { - return (field) => buildNode('or', [partialLeft(field), partialRight(field)]); + return (field) => buildFunctionNode('or', [partialLeft(field), partialRight(field)]); } / AndListOfValues AndListOfValues = partialLeft:NotListOfValues And partialRight:AndListOfValues { - return (field) => buildNode('and', [partialLeft(field), partialRight(field)]); + return (field) => buildFunctionNode('and', [partialLeft(field), partialRight(field)]); } / NotListOfValues NotListOfValues = Not partial:ListOfValues { - return (field) => buildNode('not', partial(field)); + return (field) => buildFunctionNode('not', [partial(field)]); } / ListOfValues Value = value:Literal { - return (field) => buildNode('is', field, value); + return (field) => buildFunctionNode('is', [field, value]); } Or @@ -92,8 +96,9 @@ Not = 'not'i Space+ Literal - = QuotedString - / UnquotedLiteral + = value:(QuotedString / UnquotedLiteral) { + return buildLiteralNode(value); + } QuotedString = '"' chars:(EscapedSpecialCharacter / [^"])* '"' { From 6086784ddb0c3b4b7520ca31064b7e401a2072be Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 12 Feb 2018 13:00:08 -0700 Subject: [PATCH 11/29] Add support for exact phrase search for quoted values --- src/ui/public/kuery/ast/__tests__/ast.js | 2 +- src/ui/public/kuery/ast/kql.peg | 24 +++++++++++-------- src/ui/public/kuery/functions/__tests__/is.js | 6 ++--- src/ui/public/kuery/functions/is.js | 14 ++++++----- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/ui/public/kuery/ast/__tests__/ast.js b/src/ui/public/kuery/ast/__tests__/ast.js index fb782fa8af772..8841cafaccaf8 100644 --- a/src/ui/public/kuery/ast/__tests__/ast.js +++ b/src/ui/public/kuery/ast/__tests__/ast.js @@ -141,7 +141,7 @@ describe('kuery AST API', function () { }); it('should support a shorthand operator syntax for "is" functions', function () { - const expected = nodeTypes.function.buildNode('is', 'foo', 'bar', 'operator'); + const expected = nodeTypes.function.buildNode('is', 'foo', 'bar', true, 'operator'); const actual = fromKueryExpressionNoMeta('foo:bar'); expectDeepEqual(actual, expected); }); diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index f92e7f3278b7c..a1c64d155f24e 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -82,8 +82,13 @@ NotListOfValues / ListOfValues Value - = value:Literal { - return (field) => buildFunctionNode('is', [field, value]); + = value:QuotedString { + const isPhrase = buildLiteralNode(true); + return (field) => buildFunctionNode('is', [field, value, isPhrase]); + } + / value:UnquotedLiteral { + const isPhrase = buildLiteralNode(false); + return (field) => buildFunctionNode('is', [field, value, isPhrase]); } Or @@ -96,23 +101,22 @@ Not = 'not'i Space+ Literal - = value:(QuotedString / UnquotedLiteral) { - return buildLiteralNode(value); - } + = QuotedString / UnquotedLiteral QuotedString = '"' chars:(EscapedSpecialCharacter / [^"])* '"' { - return chars.join(''); + return buildLiteralNode(chars.join('')); } UnquotedLiteral = chars:UnquotedCharacter+ { const sequence = chars.join('').trim(); - if (sequence === 'null') return null; - if (sequence === 'true') return true; - if (sequence === 'false') return false; + if (sequence === 'null') return buildLiteralNode(null); + if (sequence === 'true') return buildLiteralNode(true); + if (sequence === 'false') return buildLiteralNode(false); const number = Number(sequence); - return isNaN(number) ? sequence : number; + const value = isNaN(number) ? sequence : number; + return buildLiteralNode(value); } UnquotedCharacter diff --git a/src/ui/public/kuery/functions/__tests__/is.js b/src/ui/public/kuery/functions/__tests__/is.js index 11684bedb951b..f1113652d9c9a 100644 --- a/src/ui/public/kuery/functions/__tests__/is.js +++ b/src/ui/public/kuery/functions/__tests__/is.js @@ -103,7 +103,7 @@ describe('kuery functions', function () { } }; - const node = nodeTypes.function.buildNode('is', 'response', 200); + const node = nodeTypes.function.buildNode('is', 'response', 200, true); const result = is.toElasticsearchQuery(node, indexPattern); expectDeepEqual(result, expected); }); @@ -119,13 +119,13 @@ describe('kuery functions', function () { describe('toKueryExpression', function () { it('should serialize "is" nodes with an operator syntax', function () { - const node = nodeTypes.function.buildNode('is', 'response', 200, 'operator'); + const node = nodeTypes.function.buildNode('is', 'response', 200, false, 'operator'); const result = is.toKueryExpression(node); expect(result).to.be('"response":200'); }); it('should throw an error for nodes with unknown or undefined serialize styles', function () { - const node = nodeTypes.function.buildNode('is', 'response', 200, 'notValid'); + const node = nodeTypes.function.buildNode('is', 'response', 200, false, 'notValid'); expect(is.toKueryExpression) .withArgs(node).to.throwException(/Cannot serialize "is" function as "notValid"/); }); diff --git a/src/ui/public/kuery/functions/is.js b/src/ui/public/kuery/functions/is.js index 32ef1d6ac2d1d..a2a376453d2d3 100644 --- a/src/ui/public/kuery/functions/is.js +++ b/src/ui/public/kuery/functions/is.js @@ -2,7 +2,7 @@ import _ from 'lodash'; import * as literal from '../node_types/literal'; import { getPhraseScript } from 'ui/filter_manager/lib/phrase'; -export function buildNodeParams(fieldName, value, serializeStyle = 'operator') { +export function buildNodeParams(fieldName, value, isPhrase = false, serializeStyle = 'operator') { if (_.isUndefined(fieldName)) { throw new Error('fieldName is a required argument'); } @@ -11,16 +11,17 @@ export function buildNodeParams(fieldName, value, serializeStyle = 'operator') { } return { - arguments: [literal.buildNode(fieldName), literal.buildNode(value)], + arguments: [literal.buildNode(fieldName), literal.buildNode(value), literal.buildNode(isPhrase)], serializeStyle }; } export function toElasticsearchQuery(node, indexPattern) { - const { arguments: [ fieldNameArg, valueArg ] } = node; + const { arguments: [ fieldNameArg, valueArg, isPhraseArg ] } = node; const fieldName = literal.toElasticsearchQuery(fieldNameArg); const field = indexPattern.fields.byName[fieldName]; const value = !_.isUndefined(valueArg) ? literal.toElasticsearchQuery(valueArg) : valueArg; + const type = isPhraseArg.value ? 'phrase' : 'best_fields'; if (field && field.scripted) { return { @@ -32,8 +33,8 @@ export function toElasticsearchQuery(node, indexPattern) { else if (fieldName === null) { return { multi_match: { + type, query: value, - type: 'phrase', lenient: true, } }; @@ -44,9 +45,9 @@ export function toElasticsearchQuery(node, indexPattern) { else if (fieldName === '*' && value !== '*') { return { multi_match: { + type, query: value, fields: ['*'], - type: 'phrase', lenient: true, } }; @@ -57,8 +58,9 @@ export function toElasticsearchQuery(node, indexPattern) { }; } else { + const queryType = type === 'phrase' ? 'match_phrase' : 'match'; return { - match_phrase: { + [queryType]: { [fieldName]: value } }; From 85e3213569c01051badf6d1eba1ff16588577914 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 13 Feb 2018 15:01:08 -0700 Subject: [PATCH 12/29] This commit makes Bargs very happy cuz it does a lot --- src/ui/public/kuery/ast/kql.peg | 22 ++++- src/ui/public/kuery/functions/is.js | 88 ++++++++++++------- .../kuery/functions/utils/get_fields.js | 19 ++++ src/ui/public/kuery/node_types/index.js | 3 +- src/ui/public/kuery/node_types/wildcard.js | 51 +++++++++++ 5 files changed, 146 insertions(+), 37 deletions(-) create mode 100644 src/ui/public/kuery/functions/utils/get_fields.js create mode 100644 src/ui/public/kuery/node_types/wildcard.js diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index a1c64d155f24e..c9d2e54d0dff3 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -3,6 +3,7 @@ const { nodeTypes } = options.helpers; const buildFunctionNode = nodeTypes.function.buildNodeWithArgumentNodes; const buildLiteralNode = nodeTypes.literal.buildNode; + const buildWildcardNode = nodeTypes.wildcard.buildNode; const buildNamedArgNode = nodeTypes.namedArg.buildNode; } @@ -86,6 +87,10 @@ Value const isPhrase = buildLiteralNode(true); return (field) => buildFunctionNode('is', [field, value, isPhrase]); } + / value:WildcardString { + const isPhrase = buildLiteralNode(false); + return (field) => buildFunctionNode('is', [field, value, isPhrase]); + } / value:UnquotedLiteral { const isPhrase = buildLiteralNode(false); return (field) => buildFunctionNode('is', [field, value, isPhrase]); @@ -101,13 +106,26 @@ Not = 'not'i Space+ Literal - = QuotedString / UnquotedLiteral + = QuotedString / WildcardString / UnquotedLiteral QuotedString = '"' chars:(EscapedSpecialCharacter / [^"])* '"' { return buildLiteralNode(chars.join('')); } +WildcardString + = sequences:WildcardSequence+ { + return buildWildcardNode(sequences.reduce((acc, arr) => { + const compacted = arr.filter(value => value !== ''); + return [...acc, ...compacted]; + }, [])); + } + +WildcardSequence + = left:UnquotedCharacter* '*' right:UnquotedCharacter* { + return [left.join(''), Symbol('*'), right.join('')]; + } + UnquotedLiteral = chars:UnquotedCharacter+ { const sequence = chars.join('').trim(); @@ -133,7 +151,7 @@ Keyword = Or / And / Not SpecialCharacter - = [():<>"] + = [\\():<>"*] RangeOperator = '<=' { return 'lte'; } diff --git a/src/ui/public/kuery/functions/is.js b/src/ui/public/kuery/functions/is.js index a2a376453d2d3..ae4397a5f5b8f 100644 --- a/src/ui/public/kuery/functions/is.js +++ b/src/ui/public/kuery/functions/is.js @@ -1,6 +1,9 @@ import _ from 'lodash'; +import * as ast from '../ast'; import * as literal from '../node_types/literal'; +import * as wildcard from '../node_types/wildcard'; import { getPhraseScript } from 'ui/filter_manager/lib/phrase'; +import { getFields } from './utils/get_fields'; export function buildNodeParams(fieldName, value, isPhrase = false, serializeStyle = 'operator') { if (_.isUndefined(fieldName)) { @@ -18,19 +21,11 @@ export function buildNodeParams(fieldName, value, isPhrase = false, serializeSty export function toElasticsearchQuery(node, indexPattern) { const { arguments: [ fieldNameArg, valueArg, isPhraseArg ] } = node; - const fieldName = literal.toElasticsearchQuery(fieldNameArg); - const field = indexPattern.fields.byName[fieldName]; - const value = !_.isUndefined(valueArg) ? literal.toElasticsearchQuery(valueArg) : valueArg; + + const value = !_.isUndefined(valueArg) ? ast.toElasticsearchQuery(valueArg) : valueArg; const type = isPhraseArg.value ? 'phrase' : 'best_fields'; - if (field && field.scripted) { - return { - script: { - ...getPhraseScript(field, value) - } - }; - } - else if (fieldName === null) { + if (fieldNameArg.value === null) { return { multi_match: { type, @@ -39,32 +34,57 @@ export function toElasticsearchQuery(node, indexPattern) { } }; } - else if (fieldName === '*' && value === '*') { + + const fields = getFields(fieldNameArg, indexPattern); + const isExistsQuery = valueArg.type === 'wildcard' && value === '*'; + const isMatchAllQuery = isExistsQuery && fields && fields.length === indexPattern.fields.length; + + if (isMatchAllQuery) { return { match_all: {} }; } - else if (fieldName === '*' && value !== '*') { - return { - multi_match: { - type, - query: value, - fields: ['*'], - lenient: true, - } - }; - } - else if (fieldName !== '*' && value === '*') { - return { - exists: { field: fieldName } - }; - } - else { - const queryType = type === 'phrase' ? 'match_phrase' : 'match'; - return { - [queryType]: { - [fieldName]: value + + const queries = fields.reduce((accumulator, field) => { + if (field.scripted) { + // Exists queries don't make sense for scripted fields + if (!isExistsQuery) { + return [...accumulator, { + script: { + ...getPhraseScript(field, value) + } + }]; } - }; - } + } + else if (isExistsQuery) { + return [...accumulator, { + exists: { + field: field.name + } + }]; + } + else if (valueArg.type === 'wildcard') { + return [...accumulator, { + query_string: { + fields: [field.name], + query: wildcard.toQueryStringQuery(valueArg), + } + }]; + } + else { + const queryType = type === 'phrase' ? 'match_phrase' : 'match'; + return [...accumulator, { + [queryType]: { + [field.name]: value + } + }]; + } + }, []); + + return { + bool: { + should: queries, + minimum_should_match: 1 + } + }; } export function toKueryExpression(node) { diff --git a/src/ui/public/kuery/functions/utils/get_fields.js b/src/ui/public/kuery/functions/utils/get_fields.js new file mode 100644 index 0000000000000..9a8ff16315f26 --- /dev/null +++ b/src/ui/public/kuery/functions/utils/get_fields.js @@ -0,0 +1,19 @@ +import * as literal from '../../node_types/literal'; +import * as wildcard from '../../node_types/wildcard'; + +export function getFields(node, indexPattern) { + if (node.type === 'literal') { + const fieldName = literal.toElasticsearchQuery(node); + const field = indexPattern.fields.byName[fieldName]; + if (!field) { + throw new Error(`Field ${fieldName} does not exist in index pattern ${indexPattern.title}`); + } + return [field]; + } else if (node.type === 'wildcard') { + const fields = indexPattern.fields.filter(field => wildcard.test(node, field.name)); + if (fields.length === 0) { + throw new Error(`No fields match the pattern ${wildcard.toElasticsearchQuery(node)} in index pattern ${indexPattern.title}`); + } + return fields; + } +} diff --git a/src/ui/public/kuery/node_types/index.js b/src/ui/public/kuery/node_types/index.js index 8a9e00a3bd8a5..26249b0ed8e16 100644 --- a/src/ui/public/kuery/node_types/index.js +++ b/src/ui/public/kuery/node_types/index.js @@ -1,10 +1,11 @@ import * as functionType from './function'; import * as literal from './literal'; import * as namedArg from './named_arg'; +import * as wildcard from './wildcard'; export const nodeTypes = { function: functionType, literal, namedArg, + wildcard, }; - diff --git a/src/ui/public/kuery/node_types/wildcard.js b/src/ui/public/kuery/node_types/wildcard.js new file mode 100644 index 0000000000000..aae20b7e277aa --- /dev/null +++ b/src/ui/public/kuery/node_types/wildcard.js @@ -0,0 +1,51 @@ +// Copied from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions +function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string +} + +// See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters +function escapeQueryString(string) { + return string.replace(/[+-=&|> { + if (typeof sequence === 'symbol') { + return '.*'; + } else { + return escapeRegExp(sequence); + } + }).join(''); + const regexp = new RegExp(`^${regex}$`); + return regexp.test(string); +} + +export function toElasticsearchQuery(node) { + const { value } = node; + return value.map(sequence => { + if (typeof sequence === 'symbol') { + return '*'; + } else { + return sequence; + } + }).join(''); +} + +export function toQueryStringQuery(node) { + const { value } = node; + return value.map(sequence => { + if (typeof sequence === 'symbol') { + return '*'; + } else { + return escapeQueryString(sequence); + } + }).join(''); +} From 0a71bcd587e2e247c72134c79edd430eb0effa3d Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 13 Feb 2018 15:24:14 -0700 Subject: [PATCH 13/29] Add wildcard field support to range query --- src/ui/public/kuery/ast/kql.peg | 20 ++++++++++++++++++-- src/ui/public/kuery/functions/range.js | 25 +++++++++++++++++-------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index c9d2e54d0dff3..e7e3990d448c4 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -5,6 +5,14 @@ const buildLiteralNode = nodeTypes.literal.buildNode; const buildWildcardNode = nodeTypes.wildcard.buildNode; const buildNamedArgNode = nodeTypes.namedArg.buildNode; + + function trimLeft(string) { + return string.replace(/^[\s\uFEFF\xA0]+/g, ''); + } + + function trimRight(string) { + return string.replace(/[\s\uFEFF\xA0]+$/g, ''); + } } start @@ -115,10 +123,18 @@ QuotedString WildcardString = sequences:WildcardSequence+ { - return buildWildcardNode(sequences.reduce((acc, arr) => { + const compactedSequences = sequences.reduce((acc, arr, i) => { const compacted = arr.filter(value => value !== ''); return [...acc, ...compacted]; - }, [])); + }, []); + if (typeof compactedSequences[0] === 'string') { + compactedSequences[0] = trimLeft(compactedSequences[0]); + } + const lastIndex = compactedSequences.length - 1; + if (typeof compactedSequences[lastIndex] === 'string') { + compactedSequences[lastIndex] = trimRight(compactedSequences[lastIndex]); + } + return buildWildcardNode(compactedSequences); } WildcardSequence diff --git a/src/ui/public/kuery/functions/range.js b/src/ui/public/kuery/functions/range.js index d673f6d8fe8de..dc894c667cbf2 100644 --- a/src/ui/public/kuery/functions/range.js +++ b/src/ui/public/kuery/functions/range.js @@ -2,6 +2,7 @@ import _ from 'lodash'; import { nodeTypes } from '../node_types'; import * as ast from '../ast'; import { getRangeScript } from 'ui/filter_manager/lib/range'; +import { getFields } from './utils/get_fields'; export function buildNodeParams(fieldName, params, serializeStyle = 'operator') { params = _.pick(params, 'gt', 'lt', 'gte', 'lte', 'format'); @@ -23,22 +24,30 @@ export function buildNodeParams(fieldName, params, serializeStyle = 'operator') export function toElasticsearchQuery(node, indexPattern) { const [ fieldNameArg, ...args ] = node.arguments; - const fieldName = nodeTypes.literal.toElasticsearchQuery(fieldNameArg); - const field = indexPattern.fields.byName[fieldName]; + const fields = getFields(fieldNameArg, indexPattern); const namedArgs = extractArguments(args); const queryParams = _.mapValues(namedArgs, ast.toElasticsearchQuery); - if (field && field.scripted) { + const queries = fields.map((field) => { + if (field.scripted) { + return { + script: { + ...getRangeScript(field, queryParams) + } + }; + } + return { - script: { - ...getRangeScript(field, queryParams) + range: { + [field.name]: queryParams } }; - } + }); return { - range: { - [fieldName]: queryParams + bool: { + should: queries, + minimum_should_match: 1 } }; } From ae4fa1cefecdf43f608fa3910f69cfa1259c1ec9 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 13 Feb 2018 15:27:14 -0700 Subject: [PATCH 14/29] Remove range support for wildcard values --- src/ui/public/kuery/ast/kql.peg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/public/kuery/ast/kql.peg b/src/ui/public/kuery/ast/kql.peg index e7e3990d448c4..1087f62f3630b 100644 --- a/src/ui/public/kuery/ast/kql.peg +++ b/src/ui/public/kuery/ast/kql.peg @@ -52,7 +52,7 @@ Expression / ValueExpression FieldRangeExpression - = field:Literal Space* operator:RangeOperator Space* value:Literal { + = field:Literal Space* operator:RangeOperator Space* value:(QuotedString / UnquotedLiteral) { const range = buildNamedArgNode(operator, value); return buildFunctionNode('range', [field, range]); } From 8b6bb8e72d173eb173508d3ca728158324131bea Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Thu, 15 Feb 2018 17:07:22 -0500 Subject: [PATCH 15/29] Remove KQL as a separate language Updates kuery to use KQL's grammar. This will lead to a smoother transition for both us and Kuery users. We mainly added KQL as a separate language so that we could notify Kuery users that the syntax had changed. I realized we could do the same by trying to parse their query strings with the old grammar if the new grammar fails, and if the old grammar parses successfully we can display an error message with a link to the docs describing the syntax changes. Since Kuery now uses the more simple KQL syntax, I've also re-enabled the filter bar when Kuery is selected. --- docs/discover/kuery.asciidoc | 117 ++++++------------ .../public/dashboard/dashboard_app.html | 1 - .../kibana/public/discover/index.html | 1 - .../public/visualize/editor/editor.html | 2 +- .../public/coordinate_maps_visualization.js | 36 +----- .../data_source/build_query/build_es_query.js | 11 +- .../data_source/build_query/from_kuery.js | 26 ++-- src/ui/public/doc_table/actions/filter.js | 31 +---- .../filter_bar/filter_bar_click_handler.js | 16 +-- .../public/query_bar/directive/query_bar.html | 16 --- src/ui/public/query_bar/lib/queryLanguages.js | 1 - 11 files changed, 73 insertions(+), 185 deletions(-) diff --git a/docs/discover/kuery.asciidoc b/docs/discover/kuery.asciidoc index d25d1a71c9738..e82575ada1fc4 100644 --- a/docs/discover/kuery.asciidoc +++ b/docs/discover/kuery.asciidoc @@ -3,22 +3,32 @@ experimental[This functionality is experimental and may be changed or removed completely in a future release.] +[NOTE] +============ +Breaking changes were made to Kuery's experimental syntax in 6.3. Read on for details of the new syntax. +============ + Kuery is a new query language built specifically for Kibana. It aims to simplify the search experience in Kibana and enable the creation of helpful features like auto-complete, seamless migration of saved searches, additional query types, and more. Kuery is a basic experience today but we're hard at work building these additional features on top of the foundation Kuery provides. -Kueries are built with functions. Many functions take a field name as their first argument. Extremely common functions have shorthand notations. +If you're familiar with Kibana's old lucene query syntax, you should feel right at home with Kuery. Both languages +are very similar, but there are some differences we'll note along the way. -`is("response", 200)` will match documents where the response field matches the value 200. -`response:200` does the same thing. `:` is an alias for the `is` function. +`response:200` will match documents where the response field matches the value 200. -Multiple search terms are separated by whitespace. +Quotes around a search term will initiate a phrase search. For example, `message:"Quick brown fox"` will search +for the phrase "quick brown fox" in the message field. Without the quotes, your query will get broken down into tokens via +the message field's configured analyzer and will match documents that contain those tokens, regardless of the order in which +they appear. This means documents with "quick brown fox" will match, but so will "quick fox brown". Remember to use quotes if you want +to search for a phrase. -`response:200 extension:php` will match documents where response matches 200 and extension matches php. +Unlike lucene, Kuery will not split on whitespace. Multiple search terms must be separated by explicit +boolean operators. Note that boolean operators in Kuery are not case sensitive. -*All terms must match by default*. The language supports boolean logic with and/or operators. The above query is equivalent to `response:200 and extension:php`. -This is a departure from the Lucene query syntax where all terms are optional by default. +`response:200 extension:php` in lucene would become `response:200 and extension:php`. + This will match documents where response matches 200 and extension matches php. We can make terms optional by using `or`. @@ -32,85 +42,40 @@ We can override the default precedence with grouping. `response:200 and (extension:php or extension:css)` will match documents where response is 200 and extension is either php or css. -Terms can be inverted by prefixing them with `!`. +A shorthand exists that allows us to easily search a single field for multiple values. + +`response:(200 or 404)` searches for docs where the `response` field matches 200 or 404. We can also search for docs +with multi-value fields that contain a list of terms, for example: `tags:(success and info and security)` -`!response:200` will match all documents where response is not 200. +Terms can be inverted by prefixing them with `not`. + +`not response:200` will match all documents where response is not 200. Entire groups can also be inverted. -`response:200 and !(extension:php or extension:css)` +`response:200 and not (extension:php or extension:css)` + +Ranges in Kuery are similar to lucene with a small syntactical difference. + +Instead of `bytes:>1000`, Kuery omits the colon: `bytes > 1000`. + +`>, >=, <, <=` are all valid range operators. + +Exist queries are simple and do not require a special operator. `response:*` will find all docs where the response +field exists. -Some query functions have named arguments. +Wildcard queries are available. `machine.os:win*` would match docs where the machine.os field starts with "win", which +would match values like "windows 7" and "windows 10". -`range("bytes", gt=1000, lt=8000)` will match documents where the bytes field is greater than 1000 and less than 8000. +Wildcards also allow us to search multiple fields at once. This can come in handy when you have both `text` and `keyword` +versions of a field. Let's say we have `machine.os` and `machine.os.keyword` fields and we want to check both for the term +"windows 10". We can do it like this: `machine.os*:windows 10". -Quotes are generally optional if your terms don't have whitespace or special characters. `range(bytes, gt=1000, lt=8000)` -would also be a valid query. [NOTE] ============ -Terms without fields will be matched against all fields. For example, a query for `response:200` will search for the value 200 +Terms without fields will be matched against the default field in your index settings. If a default field is not +set these terms will be matched against all fields. For example, a query for `response:200` will search for the value 200 in the response field, but a query for just `200` will search for 200 across all fields in your index. ============ -==== Function Reference - -[horizontal] -Function Name:: Description - -and:: -Purpose::: Match all given sub-queries -Alias::: `and` as a binary operator -Examples::: -* `and(response:200, extension:php)` -* `response:200 and extension:php` - -or:: -Purpose::: Match one or more sub-queries -Alias::: `or` as a binary operator -Examples::: -* `or(extension:css, extension:php)` -* `extension:css or extension:php` - -not:: -Purpose::: Negates a sub-query -Alias::: `!` as a prefix operator -Examples::: -* `not(response:200)` -* `!response:200` - -is:: -Purpose::: Matches a field with a given term -Alias::: `:` -Examples::: -* `is("response", 200)` -* `response:200` - -range:: -Purpose::: Match a field against a range of values. -Alias::: `:[]` -Examples::: -* `range("bytes", gt=1000, lt=8000)` -* `bytes:[1000 to 8000]` -Named arguments::: -* `gt` - greater than -* `gte` - greater than or equal to -* `lt` - less than -* `lte` - less than or equal to - -exists:: -Purpose::: Match documents where a given field exists -Examples::: `exists("response")` - -geoBoundingBox:: -Purpose::: Creates a geo_bounding_box query -Examples::: -* `geoBoundingBox("coordinates", topLeft="40.73, -74.1", bottomRight="40.01, -71.12")` (whitespace between lat and lon is ignored) -Named arguments::: -* `topLeft` - the top left corner of the bounding box as a "lat, lon" string -* `bottomRight` - the bottom right corner of the bounding box as a "lat, lon" string - -geoPolygon:: -Purpose::: Creates a geo_polygon query given 3 or more points as "lat, lon" -Examples::: -* `geoPolygon("geo.coordinates", "40.97, -127.26", "24.20, -84.375", "40.44, -66.09")` \ No newline at end of file diff --git a/src/core_plugins/kibana/public/dashboard/dashboard_app.html b/src/core_plugins/kibana/public/dashboard/dashboard_app.html index 7f124c25a979c..ad4180aea3fb5 100644 --- a/src/core_plugins/kibana/public/dashboard/dashboard_app.html +++ b/src/core_plugins/kibana/public/dashboard/dashboard_app.html @@ -39,7 +39,6 @@ ng-show="showFilterBar()" state="state" index-patterns="indexPatterns" - ng-if="['lucene', 'kql'].includes(model.query.language)" >
diff --git a/src/core_plugins/kibana/public/visualize/editor/editor.html b/src/core_plugins/kibana/public/visualize/editor/editor.html index c885e06d2fb07..d792a8c10e417 100644 --- a/src/core_plugins/kibana/public/visualize/editor/editor.html +++ b/src/core_plugins/kibana/public/visualize/editor/editor.html @@ -50,7 +50,7 @@ diff --git a/src/core_plugins/tile_map/public/coordinate_maps_visualization.js b/src/core_plugins/tile_map/public/coordinate_maps_visualization.js index 05a78410a2be2..1b4dfe08a5b5e 100644 --- a/src/core_plugins/tile_map/public/coordinate_maps_visualization.js +++ b/src/core_plugins/tile_map/public/coordinate_maps_visualization.js @@ -117,39 +117,11 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) { const indexPatternName = agg.vis.indexPattern.id; const field = agg.fieldName(); - const query = this.vis.API.queryManager.getQuery(); - const language = query.language; + const filter = { meta: { negate: false, index: indexPatternName } }; + filter[filterName] = { ignore_unmapped: true }; + filter[filterName][field] = filterData; - if (['lucene', 'kql'].includes(language)) { - const filter = { meta: { negate: false, index: indexPatternName } }; - filter[filterName] = { ignore_unmapped: true }; - filter[filterName][field] = filterData; - - this.vis.API.queryFilter.addFilters([filter]); - } - else if (language === 'kuery') { - const { fromKueryExpression, toKueryExpression, nodeTypes } = this.vis.API.kuery; - let newQuery; - - if (filterName === 'geo_bounding_box') { - newQuery = nodeTypes.function.buildNode('geoBoundingBox', field, _.mapKeys(filterData, (value, key) => _.camelCase(key))); - } - else if (filterName === 'geo_polygon') { - newQuery = nodeTypes.function.buildNode('geoPolygon', field, filterData.points); - } - else { - throw new Error(`Kuery does not support ${filterName} queries`); - } - - const allQueries = _.isEmpty(query.query) - ? [newQuery] - : [fromKueryExpression(query.query), newQuery]; - - this.vis.API.queryManager.setQuery({ - query: toKueryExpression(nodeTypes.function.buildNode('and', allQueries, 'implicit')), - language: 'kuery' - }); - } + this.vis.API.queryFilter.addFilters([filter]); this.vis.updateState(); } diff --git a/src/ui/public/courier/data_source/build_query/build_es_query.js b/src/ui/public/courier/data_source/build_query/build_es_query.js index 1026bed66221d..00e133268b3e7 100644 --- a/src/ui/public/courier/data_source/build_query/build_es_query.js +++ b/src/ui/public/courier/data_source/build_query/build_es_query.js @@ -1,6 +1,6 @@ import { groupBy, has } from 'lodash'; import { DecorateQueryProvider } from '../_decorate_query'; -import { buildQueryFromKuery, buildQueryFromKql } from './from_kuery'; +import { buildQueryFromKuery } from './from_kuery'; import { buildQueryFromFilters } from './from_filters'; import { buildQueryFromLucene } from './from_lucene'; @@ -17,16 +17,15 @@ export function BuildESQueryProvider(Private) { const queriesByLanguage = groupBy(validQueries, 'language'); const kueryQuery = buildQueryFromKuery(indexPattern, queriesByLanguage.kuery); - const kqlQuery = buildQueryFromKql(indexPattern, queriesByLanguage.kql); const luceneQuery = buildQueryFromLucene(queriesByLanguage.lucene, decorateQuery); const filterQuery = buildQueryFromFilters(filters, decorateQuery, indexPattern); return { bool: { - must: [].concat(kueryQuery.must, kqlQuery.must, luceneQuery.must, filterQuery.must), - filter: [].concat(kueryQuery.filter, kqlQuery.filter, luceneQuery.filter, filterQuery.filter), - should: [].concat(kueryQuery.should, kqlQuery.should, luceneQuery.should, filterQuery.should), - must_not: [].concat(kueryQuery.must_not, kqlQuery.must_not, luceneQuery.must_not, filterQuery.must_not), + must: [].concat(kueryQuery.must, luceneQuery.must, filterQuery.must), + filter: [].concat(kueryQuery.filter, luceneQuery.filter, filterQuery.filter), + should: [].concat(kueryQuery.should, luceneQuery.should, filterQuery.should), + must_not: [].concat(kueryQuery.must_not, luceneQuery.must_not, filterQuery.must_not), } }; } diff --git a/src/ui/public/courier/data_source/build_query/from_kuery.js b/src/ui/public/courier/data_source/build_query/from_kuery.js index b04f3d36d5c92..c2e153b563795 100644 --- a/src/ui/public/courier/data_source/build_query/from_kuery.js +++ b/src/ui/public/courier/data_source/build_query/from_kuery.js @@ -1,13 +1,25 @@ -import _ from 'lodash'; import { fromKueryExpression, fromKqlExpression, toElasticsearchQuery, nodeTypes } from '../../../kuery'; +import { documentationLinks } from '../../../documentation_links'; -export function buildQueryFromKuery(indexPattern, queries) { - const queryASTs = _.map(queries, query => fromKueryExpression(query.query)); - return buildQuery(indexPattern, queryASTs); -} +const queryDocs = documentationLinks.query; -export function buildQueryFromKql(indexPattern, queries) { - const queryASTs = _.map(queries, query => fromKqlExpression(query.query)); +export function buildQueryFromKuery(indexPattern, queries = []) { + const queryASTs = queries.map((query) => { + try { + return fromKqlExpression(query.query); + } + catch (parseError) { + try { + fromKueryExpression(query.query); + } + catch (legacyParseError) { + throw parseError; + } + throw new Error( + `It looks like you're using an outdated Kuery syntax. See what changed in the [docs](${queryDocs.kueryQuerySyntax})!` + ); + } + }); return buildQuery(indexPattern, queryASTs); } diff --git a/src/ui/public/doc_table/actions/filter.js b/src/ui/public/doc_table/actions/filter.js index 2101f6d2d2431..b9049519e3767 100644 --- a/src/ui/public/doc_table/actions/filter.js +++ b/src/ui/public/doc_table/actions/filter.js @@ -1,36 +1,7 @@ -import _ from 'lodash'; -import { toKueryExpression, fromKueryExpression, nodeTypes } from 'ui/kuery'; - export function addFilter(field, values = [], operation, index, state, filterManager) { - const fieldName = _.isObject(field) ? field.name : field; - if (!Array.isArray(values)) { values = [values]; } - if (['lucene', 'kql'].includes(state.query.language)) { - filterManager.add(field, values, operation, index); - } - - if (state.query.language === 'kuery') { - const negate = operation === '-'; - const isExistsQuery = fieldName === '_exists_'; - - const newQueries = values.map((value) => { - const newQuery = isExistsQuery - ? nodeTypes.function.buildNode('exists', value) - : nodeTypes.function.buildNode('is', fieldName, value); - - return negate ? nodeTypes.function.buildNode('not', newQuery) : newQuery; - }); - - const allQueries = _.isEmpty(state.query.query) - ? newQueries - : [fromKueryExpression(state.query.query), ...newQueries]; - - state.query = { - query: toKueryExpression(nodeTypes.function.buildNode('and', allQueries, 'implicit')), - language: 'kuery' - }; - } + filterManager.add(field, values, operation, index); } diff --git a/src/ui/public/filter_bar/filter_bar_click_handler.js b/src/ui/public/filter_bar/filter_bar_click_handler.js index 84bde607c96bb..a1af664f1f02e 100644 --- a/src/ui/public/filter_bar/filter_bar_click_handler.js +++ b/src/ui/public/filter_bar/filter_bar_click_handler.js @@ -3,10 +3,8 @@ import { dedupFilters } from './lib/dedup_filters'; import { uniqFilters } from './lib/uniq_filters'; import { findByParam } from 'ui/utils/find_by_param'; import { toastNotifications } from 'ui/notify'; -import { AddFiltersToKueryProvider } from './lib/add_filters_to_kuery'; -export function FilterBarClickHandlerProvider(Private) { - const addFiltersToKuery = Private(AddFiltersToKueryProvider); +export function FilterBarClickHandlerProvider() { return function ($state) { return function (event, simulate) { @@ -63,17 +61,7 @@ export function FilterBarClickHandlerProvider(Private) { filters = dedupFilters($state.filters, uniqFilters(filters), { negate: true }); if (!simulate) { - if (['lucene', 'kql'].includes($state.query.language)) { - $state.$newFilters = filters; - } - else if ($state.query.language === 'kuery') { - addFiltersToKuery($state, filters) - .then(() => { - if (_.isFunction($state.save)) { - $state.save(); - } - }); - } + $state.$newFilters = filters; } return filters; } diff --git a/src/ui/public/query_bar/directive/query_bar.html b/src/ui/public/query_bar/directive/query_bar.html index 209c67b00f900..9f9eca6d03138 100644 --- a/src/ui/public/query_bar/directive/query_bar.html +++ b/src/ui/public/query_bar/directive/query_bar.html @@ -67,22 +67,6 @@
- -
- -