diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts index cf6b282404f85..1316a00e53c0a 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts @@ -7,6 +7,8 @@ */ import { i18n } from '@kbn/i18n'; +import { isColumnItem } from '../shared/helpers'; +import { ESQLColumn, ESQLCommand, ESQLMessage } from '../types'; import { appendSeparatorOption, asOption, @@ -42,7 +44,7 @@ export const commandDefinitions: CommandDefinition[] = [ options: [metadataOption], signature: { multipleParams: true, - params: [{ name: 'index', type: 'source' }], + params: [{ name: 'index', type: 'source', wildcards: true }], }, }, { @@ -122,7 +124,7 @@ export const commandDefinitions: CommandDefinition[] = [ options: [], signature: { multipleParams: true, - params: [{ name: 'column', type: 'column' }], + params: [{ name: 'column', type: 'column', wildcards: true }], }, }, { @@ -134,7 +136,35 @@ export const commandDefinitions: CommandDefinition[] = [ options: [], signature: { multipleParams: true, - params: [{ name: 'column', type: 'column' }], + params: [{ name: 'column', type: 'column', wildcards: true }], + }, + validate: (command: ESQLCommand) => { + const messages: ESQLMessage[] = []; + const wildcardItems = command.args.filter((arg) => isColumnItem(arg) && arg.name === '*'); + if (wildcardItems.length) { + messages.push( + ...wildcardItems.map((column) => ({ + location: (column as ESQLColumn).location, + text: i18n.translate('monaco.esql.validation.dropAllColumnsError', { + defaultMessage: 'Removing all fields is not allowed [*]', + }), + type: 'error' as const, + })) + ); + } + const droppingTimestamp = command.args.find( + (arg) => isColumnItem(arg) && arg.name === '@timestamp' + ); + if (droppingTimestamp) { + messages.push({ + location: (droppingTimestamp as ESQLColumn).location, + text: i18n.translate('monaco.esql.validation.dropTimestampWarning', { + defaultMessage: 'Drop [@timestamp] will remove all time filters to the search results', + }), + type: 'warning', + }); + } + return messages; }, }, { diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts index 302394baafaac..8916a850653ff 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts @@ -80,6 +80,7 @@ export const appendSeparatorOption: CommandOptionsDefinition = { params: [{ name: 'separator', type: 'string' }], }, optional: true, + skipCommonValidation: true, // tell the validation engine to use only the validate function here validate: (option: ESQLCommandOption) => { const messages: ESQLMessage[] = []; const [firstArg] = option.args; diff --git a/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts b/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts index e0000628c820f..c1a0f047d4e32 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { ESQLCommandOption, ESQLMessage, ESQLSingleAstItem } from '../types'; +import type { ESQLCommand, ESQLCommandOption, ESQLMessage, ESQLSingleAstItem } from '../types'; export interface FunctionDefinition { name: string; @@ -42,6 +42,7 @@ export interface CommandBaseDefinition { optional?: boolean; innerType?: string; values?: string[]; + wildcards?: boolean; }>; }; } @@ -49,12 +50,14 @@ export interface CommandBaseDefinition { export interface CommandOptionsDefinition extends CommandBaseDefinition { wrapped?: string[]; optional: boolean; + skipCommonValidation?: boolean; validate?: (option: ESQLCommandOption) => ESQLMessage[]; } export interface CommandDefinition extends CommandBaseDefinition { options: CommandOptionsDefinition[]; examples: string[]; + validate?: (option: ESQLCommand) => ESQLMessage[]; } export interface Literals { diff --git a/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts b/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts index a4273b698e297..d799d934f4dfc 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts @@ -363,20 +363,61 @@ export function getDurationItemsWithQuantifier(quantifier: number = 1) { })); } +function fuzzySearch(fuzzyName: string, resources: IterableIterator) { + const wildCardPosition = getWildcardPosition(fuzzyName); + if (wildCardPosition !== 'none') { + const matcher = getMatcher(fuzzyName, wildCardPosition); + for (const resourceName of resources) { + if (matcher(resourceName)) { + return true; + } + } + } +} + +function getMatcher(name: string, position: 'start' | 'end' | 'middle') { + if (position === 'start') { + const prefix = name.substring(1); + return (resource: string) => resource.endsWith(prefix); + } + if (position === 'end') { + const prefix = name.substring(0, name.length - 1); + return (resource: string) => resource.startsWith(prefix); + } + const [prefix, postFix] = name.split('*'); + return (resource: string) => resource.startsWith(prefix) && resource.endsWith(postFix); +} + +function getWildcardPosition(name: string) { + if (!hasWildcard(name)) { + return 'none'; + } + if (name.startsWith('*')) { + return 'start'; + } + if (name.endsWith('*')) { + return 'end'; + } + return 'middle'; +} + +export function hasWildcard(name: string) { + return name.includes('*'); +} + +export function columnExists( + column: string, + { fields, variables }: Pick +) { + if (fields.has(column) || variables.has(column)) { + return true; + } + return Boolean(fuzzySearch(column, fields.keys()) || fuzzySearch(column, variables.keys())); +} + export function sourceExists(index: string, sources: Set) { if (sources.has(index)) { return true; } - // it is a fuzzy match - if (index[index.length - 1] === '*') { - const prefix = index.substring(0, index.length - 1); - for (const sourceName of sources.keys()) { - if (sourceName.includes(prefix)) { - // just to be sure that there's not an exact match here - // i.e. index-* should not match index- - return sourceName.length > prefix.length; - } - } - } - return false; + return Boolean(fuzzySearch(index, sources.keys())); } diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts index 31f1b416d6b5e..a7c4ed484311b 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/errors.ts @@ -153,6 +153,16 @@ function getMessageAndTypeFromId({ }, }), }; + case 'wildcardNotSupportedForCommand': + return { + message: i18n.translate('monaco.esql.validation.wildcardNotSupportedForCommand', { + defaultMessage: 'Using wildcards (*) in {command} is not allowed [{value}]', + values: { + command: out.command, + value: out.value, + }, + }), + }; } return { message: '' }; } diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/types.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/types.ts index 7d9743ec5737d..d7e5d0cb64f8e 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/types.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/types.ts @@ -99,6 +99,10 @@ export interface ValidationErrors { message: string; type: { command: string; value: string }; }; + wildcardNotSupportedForCommand: { + mesage: string; + type: { command: string; value: string }; + }; } export type ErrorTypes = keyof ValidationErrors; diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.test.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.test.ts index d1a181cb48f02..1554b7d2cb288 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.test.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.test.ts @@ -36,6 +36,7 @@ function getCallbackMocks() { name: `listField`, type: `list`, }, + { name: '@timestamp', type: 'date' }, ] : [ { name: 'otherField', type: 'string' }, @@ -224,7 +225,11 @@ describe('validation logic', () => { 'SyntaxError: expected {, PIPE, COMMA, OPENING_BRACKET} but found "(metadata"', ]); testErrorsAndWarnings(`from ind*, other*`, []); - testErrorsAndWarnings(`from index*`, ['Unknown index [index*]']); + testErrorsAndWarnings(`from index*`, []); + testErrorsAndWarnings(`from *ex`, []); + testErrorsAndWarnings(`from in*ex`, []); + testErrorsAndWarnings(`from ind*ex`, []); + testErrorsAndWarnings(`from indexes*`, ['Unknown index [indexes*]']); }); describe('row', () => { @@ -470,6 +475,14 @@ describe('validation logic', () => { testErrorsAndWarnings('from index | project missingField, numberField, dateField', [ 'Unknown column [missingField]', ]); + testErrorsAndWarnings('from index | keep s*', []); + testErrorsAndWarnings('from index | keep *Field', []); + testErrorsAndWarnings('from index | keep s*Field', []); + testErrorsAndWarnings('from index | keep string*Field', []); + testErrorsAndWarnings('from index | keep s*, n*', []); + testErrorsAndWarnings('from index | keep m*', ['Unknown column [m*]']); + testErrorsAndWarnings('from index | keep *m', ['Unknown column [*m]']); + testErrorsAndWarnings('from index | keep d*m', ['Unknown column [d*m]']); }); describe('drop', () => { @@ -489,6 +502,28 @@ describe('validation logic', () => { testErrorsAndWarnings('from index | project missingField, numberField, dateField', [ 'Unknown column [missingField]', ]); + testErrorsAndWarnings('from index | drop s*', []); + testErrorsAndWarnings('from index | drop *Field', []); + testErrorsAndWarnings('from index | drop s*Field', []); + testErrorsAndWarnings('from index | drop string*Field', []); + testErrorsAndWarnings('from index | drop s*, n*', []); + testErrorsAndWarnings('from index | drop m*', ['Unknown column [m*]']); + testErrorsAndWarnings('from index | drop *m', ['Unknown column [*m]']); + testErrorsAndWarnings('from index | drop d*m', ['Unknown column [d*m]']); + testErrorsAndWarnings('from index | drop *', ['Removing all fields is not allowed [*]']); + testErrorsAndWarnings('from index | drop stringField, *', [ + 'Removing all fields is not allowed [*]', + ]); + testErrorsAndWarnings( + 'from index | drop @timestamp', + [], + ['Drop [@timestamp] will remove all time filters to the search results'] + ); + testErrorsAndWarnings( + 'from index | drop stringField, @timestamp', + [], + ['Drop [@timestamp] will remove all time filters to the search results'] + ); }); describe('mv_expand', () => { @@ -535,6 +570,10 @@ describe('validation logic', () => { testErrorsAndWarnings('from a | eval numberField + 1 | rename `numberField + 1` as ', [ "SyntaxError: missing {SRC_UNQUOTED_IDENTIFIER, SRC_QUOTED_IDENTIFIER} at ''", ]); + testErrorsAndWarnings('from a | rename s* as strings', [ + 'Using wildcards (*) in rename is not allowed [s*]', + 'Unknown column [strings]', + ]); }); describe('dissect', () => { @@ -576,6 +615,9 @@ describe('validation logic', () => { testErrorsAndWarnings('from a | dissect stringField "%{a}" append_separator = true', [ 'Invalid value for dissect append_separator: expected a string, but was [true]', ]); + // testErrorsAndWarnings('from a | dissect s* "%{a}"', [ + // 'Using wildcards (*) in dissect is not allowed [s*]', + // ]); }); describe('grok', () => { @@ -596,6 +638,9 @@ describe('validation logic', () => { testErrorsAndWarnings('from a | grok numberField "%{a}"', [ 'Grok only supports string type values, found [numberField] of type number', ]); + // testErrorsAndWarnings('from a | grok s* "%{a}"', [ + // 'Using wildcards (*) in grok is not allowed [s*]', + // ]); }); describe('where', () => { @@ -1185,6 +1230,9 @@ describe('validation logic', () => { testErrorsAndWarnings(`from a | enrich policy with otherField`, []); testErrorsAndWarnings(`from a | enrich policy | eval otherField`, []); testErrorsAndWarnings(`from a | enrich policy with var0 = otherField | eval var0`, []); + testErrorsAndWarnings('from a | enrich my-pol*', [ + 'Using wildcards (*) in enrich is not allowed [my-pol*]', + ]); }); describe('shadowing', () => { diff --git a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts index 108408cd28820..67b1722cabf43 100644 --- a/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts +++ b/packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts @@ -33,6 +33,8 @@ import { inKnownTimeInterval, printFunctionSignature, sourceExists, + columnExists, + hasWildcard, } from '../shared/helpers'; import { collectVariables } from '../shared/variables'; import type { @@ -383,7 +385,8 @@ function validateOption( // use dedicate validate fn if provided if (optionDef.validate) { messages.push(...optionDef.validate(option)); - } else { + } + if (!optionDef.skipCommonValidation) { option.args.forEach((arg, index) => { if (!Array.isArray(arg)) { if (!optionDef.signature.multipleParams) { @@ -461,22 +464,36 @@ function validateSource( locations: source.location, }) ); - } else if (source.sourceType === 'index' && !sourceExists(source.name, sources)) { - messages.push( - getMessageFromId({ - messageId: 'unknownIndex', - values: { name: source.name }, - locations: source.location, - }) - ); - } else if (source.sourceType === 'policy' && !policies.has(source.name)) { - messages.push( - getMessageFromId({ - messageId: 'unknownPolicy', - values: { name: source.name }, - locations: source.location, - }) - ); + } else { + const isWildcardAndNotSupported = + hasWildcard(source.name) && !commandDef.signature.params.some(({ wildcards }) => wildcards); + if (isWildcardAndNotSupported) { + messages.push( + getMessageFromId({ + messageId: 'wildcardNotSupportedForCommand', + values: { command: commandName, value: source.name }, + locations: source.location, + }) + ); + } else { + if (source.sourceType === 'index' && !sourceExists(source.name, sources)) { + messages.push( + getMessageFromId({ + messageId: 'unknownIndex', + values: { name: source.name }, + locations: source.location, + }) + ); + } else if (source.sourceType === 'policy' && !policies.has(source.name)) { + messages.push( + getMessageFromId({ + messageId: 'unknownPolicy', + values: { name: source.name }, + locations: source.location, + }) + ); + } + } } return messages; } @@ -504,30 +521,48 @@ function validateColumnForCommand( ); } } else { - const commandDef = getCommandDefinition(commandName); - const columnRef = getColumnHit(column.name, references); - if (columnRef) { + const columnCheck = columnExists(column.name, references); + if (columnCheck) { + const commandDef = getCommandDefinition(commandName); const columnParamsWithInnerTypes = commandDef.signature.params.filter( ({ type, innerType }) => type === 'column' && innerType ); + if (columnParamsWithInnerTypes.length) { + // this should be guaranteed by the columnCheck above + const columnRef = getColumnHit(column.name, references)!; + if ( + columnParamsWithInnerTypes.every(({ innerType }) => { + return innerType !== columnRef.type; + }) + ) { + const supportedTypes = columnParamsWithInnerTypes.map(({ innerType }) => innerType); + + messages.push( + getMessageFromId({ + messageId: 'unsupportedColumnTypeForCommand', + values: { + command: capitalize(commandName), + type: supportedTypes.join(', '), + typeCount: supportedTypes.length, + givenType: columnRef.type, + column: column.name, + }, + locations: column.location, + }) + ); + } + } if ( - columnParamsWithInnerTypes.every(({ innerType }) => { - return innerType !== columnRef.type; - }) && - columnParamsWithInnerTypes.length + hasWildcard(column.name) && + !commandDef.signature.params.some(({ type, wildcards }) => type === 'column' && wildcards) ) { - const supportedTypes = columnParamsWithInnerTypes.map(({ innerType }) => innerType); - messages.push( getMessageFromId({ - messageId: 'unsupportedColumnTypeForCommand', + messageId: 'wildcardNotSupportedForCommand', values: { - command: capitalize(commandName), - type: supportedTypes.join(', '), - typeCount: supportedTypes.length, - givenType: columnRef.type, - column: column.name, + command: commandName, + value: column.name, }, locations: column.location, }) @@ -558,6 +593,10 @@ function validateCommand(command: ESQLCommand, references: ReferenceMaps): ESQLM // do not check the command exists, the grammar is already picking that up const commandDef = getCommandDefinition(command.name); + if (commandDef.validate) { + messages.push(...commandDef.validate(command)); + } + // Now validate arguments for (const commandArg of command.args) { const wrappedArg = Array.isArray(commandArg) ? commandArg : [commandArg]; diff --git a/packages/kbn-text-based-editor/src/editor_footer.tsx b/packages/kbn-text-based-editor/src/editor_footer.tsx index d524fd75ccfb6..98d1aad506cc9 100644 --- a/packages/kbn-text-based-editor/src/editor_footer.tsx +++ b/packages/kbn-text-based-editor/src/editor_footer.tsx @@ -148,7 +148,7 @@ interface EditorFooterProps { lines: number; containerCSS: Interpolation; errors?: MonacoMessage[]; - warning?: MonacoMessage[]; + warnings?: MonacoMessage[]; detectTimestamp: boolean; onErrorClick: (error: MonacoMessage) => void; refreshErrors: () => void; @@ -159,7 +159,7 @@ export const EditorFooter = memo(function EditorFooter({ lines, containerCSS, errors, - warning, + warnings, detectTimestamp, onErrorClick, refreshErrors, @@ -191,10 +191,10 @@ export const EditorFooter = memo(function EditorFooter({ onErrorClick={onErrorClick} /> )} - {warning && warning.length > 0 && ( + {warnings && warnings.length > 0 && ( { if (isOpen) {