diff --git a/x-pack/platform/plugins/shared/inference/common/output/create_output_api.ts b/x-pack/platform/plugins/shared/inference/common/output/create_output_api.ts index 8b8bd2d008d2d..aeafce2f004bf 100644 --- a/x-pack/platform/plugins/shared/inference/common/output/create_output_api.ts +++ b/x-pack/platform/plugins/shared/inference/common/output/create_output_api.ts @@ -92,8 +92,8 @@ export function createOutputApi(chatCompleteApi: ChatCompleteAPI) { return { id, output: - event.toolCalls.length && 'arguments' in event.toolCalls[0].function - ? event.toolCalls[0].function.arguments + event?.toolCalls?.length && 'arguments' in event?.toolCalls[0]?.function + ? event.toolCalls[0]?.function?.arguments : undefined, content: event.content, type: OutputEventType.OutputComplete, @@ -107,8 +107,8 @@ export function createOutputApi(chatCompleteApi: ChatCompleteAPI) { id, content: chatResponse.content, output: - chatResponse.toolCalls.length && 'arguments' in chatResponse.toolCalls[0].function - ? chatResponse.toolCalls[0].function.arguments + chatResponse?.toolCalls?.length && 'arguments' in chatResponse?.toolCalls[0]?.function + ? chatResponse?.toolCalls[0]?.function?.arguments : undefined, }; }, diff --git a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/README.md b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/README.md index 06b6cf22aa49d..d44c19d5b0262 100644 --- a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/README.md +++ b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/README.md @@ -8,3 +8,20 @@ The generated documentation is validated and will emit warnings when invalid que - checked out `built-docs` repo in the same folder as the `kibana` repository - a running Kibana instance - an installed Generative AI connector + +### Run script to generate ES|QL docs and verify syntax + +``` +node x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/index.js +``` + +The script will also generate a report of syntax errors found during the generation process, located at +`x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/__tmp__/syntax-errors.json`. This file will not be checked into git. + +### Checking syntax errors for generated files + +After making modifications to fix the syntax error, and you just need to check if there any remaining errors left,you can run a script that reports. + +``` +node x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/index.js +``` diff --git a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/extract_doc_entries.ts b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/extract_doc_entries.ts index 4de5752c8e6b1..c188c44f72e5f 100644 --- a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/extract_doc_entries.ts +++ b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/extract_doc_entries.ts @@ -60,9 +60,10 @@ export async function extractDocEntries({ log: ToolingLog; inferenceClient: ScriptInferenceClient; }): Promise { - const files = await fastGlob(`${builtDocsDir}/html/en/elasticsearch/reference/master/esql*.html`); + const path = `${builtDocsDir}/html/en/elasticsearch/reference/current/esql*.html`; + const files = await fastGlob(path); if (!files.length) { - throw new Error('No files found'); + throw new Error(`No files found at path: ${path}`); } const output: ExtractionOutput = { diff --git a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/load_esql_docs.ts b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/load_esql_docs.ts index c54e7f8655295..93c69c7cea333 100644 --- a/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/load_esql_docs.ts +++ b/x-pack/platform/plugins/shared/inference/scripts/load_esql_docs/load_esql_docs.ts @@ -6,8 +6,6 @@ */ import { run } from '@kbn/dev-cli-runner'; -import { ESQLMessage, EditorError } from '@kbn/esql-ast'; -import { validateQuery } from '@kbn/esql-validation-autocomplete'; import Fs from 'fs/promises'; import Path from 'path'; import yargs, { Argv } from 'yargs'; @@ -20,7 +18,8 @@ import { KibanaClient } from '../util/kibana_client'; import { selectConnector } from '../util/select_connector'; import { syncBuiltDocs } from './sync_built_docs_repo'; import { extractDocEntries } from './extract_doc_entries'; -import { generateDoc, FileToWrite } from './generate_doc'; +import { generateDoc } from './generate_doc'; +import { reportSyntaxErrors } from '../report_syntax_errors/report_syntax_errors'; yargs(process.argv.slice(2)) .command( @@ -128,49 +127,10 @@ yargs(process.argv.slice(2)) ); } - log.info(`Checking syntax...`); - const syntaxErrors = ( - await Promise.all(docFiles.map(async (file) => await findEsqlSyntaxError(file))) - ).flat(); - - log.warning( - `Please verify the following queries that had syntax errors\n${JSON.stringify( - syntaxErrors, - null, - 2 - )}` - ); + await reportSyntaxErrors(outDir, log, docFiles); }, { log: { defaultLevel: argv.logLevel as any }, flags: { allowUnexpected: true } } ); } ) .parse(); - -interface SyntaxError { - query: string; - errors: Array; -} - -const findEsqlSyntaxError = async (doc: FileToWrite): Promise => { - return Array.from(doc.content.matchAll(INLINE_ESQL_QUERY_REGEX)).reduce( - async (listP, [match, query]) => { - const list = await listP; - const { errors, warnings } = await validateQuery(query, { - // setting this to true, we don't want to validate the index / fields existence - ignoreOnMissingCallbacks: true, - }); - - const all = [...errors, ...warnings]; - if (all.length) { - list.push({ - errors: all, - query, - }); - } - - return list; - }, - Promise.resolve([] as SyntaxError[]) - ); -}; diff --git a/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/index.js b/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/index.js new file mode 100644 index 0000000000000..1624214a8b4e9 --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/index.js @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('@kbn/babel-register').install(); + +require('./report_syntax_errors'); diff --git a/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/report_syntax_errors.ts b/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/report_syntax_errors.ts new file mode 100644 index 0000000000000..1f1ecd9a7540c --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/scripts/report_syntax_errors/report_syntax_errors.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQLMessage, EditorError } from '@kbn/esql-ast'; +import { validateQuery } from '@kbn/esql-validation-autocomplete'; +import Fs from 'fs/promises'; +import Path from 'path'; +import yargs, { Argv } from 'yargs'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { run } from '@kbn/dev-cli-runner'; +import { INLINE_ESQL_QUERY_REGEX } from '../../common/tasks/nl_to_esql/constants'; +import type { FileToWrite } from '../load_esql_docs/generate_doc'; + +interface SyntaxError { + query: string; + errors: Array; +} + +/** + * Log out syntax errors and also write them to a {outDir}/__tmp__/syntax-errors.json + * If docsToCheck is provided, they will be used instead of reading the files from the outDir. + * @param docFiles - The files to check for syntax errors. + * @param outDir - The directory to write the syntax errors to. + * @param log - The logger to use. + */ +export const reportSyntaxErrors = async ( + outDir: string, + log: ToolingLog, + docsToCheck?: FileToWrite[] +) => { + let docFiles: FileToWrite[] | undefined = docsToCheck; + if (docsToCheck) { + log.info(`Checking syntax for ${docsToCheck.length} provided files`); + } else { + log.info(`Checking syntax for files in ${outDir}`); + docFiles = await Fs.readdir(outDir).then(async (files) => { + return await Promise.all( + files + .filter((file) => file.endsWith('.txt')) + .map(async (file) => { + const content = await Fs.readFile(Path.join(outDir, file), 'utf8'); + return { + name: file, + content, + }; + }) + ); + }); + log.info(`Found ${(docFiles ?? []).length} files to check in ${outDir}`); + } + + if (!docFiles) return; + + const syntaxErrors = ( + await Promise.all(docFiles.map(async (file) => await findEsqlSyntaxError(file))) + ).flat(); + + log.warning( + `Please verify the following queries that had syntax errors\n${JSON.stringify( + syntaxErrors, + null, + 2 + )}` + ); + if (syntaxErrors.length > 0) { + const tmpDir = Path.join(outDir, '__tmp__'); + await Fs.mkdir(tmpDir).catch((error) => (error.code === 'EEXIST' ? Promise.resolve() : error)); + const syntaxErrorsFile = Path.join(tmpDir, 'syntax-errors.json'); + await Fs.writeFile(syntaxErrorsFile, JSON.stringify(syntaxErrors, null, 2)); + log.info(`Syntax errors written to ${syntaxErrorsFile}`); + } +}; +const findEsqlSyntaxError = async (doc: FileToWrite): Promise => { + return Array.from(doc.content.matchAll(INLINE_ESQL_QUERY_REGEX)).reduce( + async (listP, [match, query]) => { + const list = await listP; + const { errors, warnings } = await validateQuery(query, { + // setting this to true, we don't want to validate the index / fields existence + ignoreOnMissingCallbacks: true, + }); + + const all = [...errors, ...warnings]; + if (all.length) { + list.push({ + errors: all, + query, + }); + } + + return list; + }, + Promise.resolve([] as SyntaxError[]) + ); +}; + +yargs(process.argv.slice(2)) + .command( + '*', + 'Extract ES|QL documentation', + (y: Argv) => + y + .option('logLevel', { + describe: 'Log level', + string: true, + default: process.env.LOG_LEVEL || 'info', + choices: ['info', 'debug', 'silent', 'verbose'], + }) + .option('outDir', { + describe: 'The directory to write the syntax errors to.', + default: Path.join(__dirname, '../../server/tasks/nl_to_esql/esql_docs'), + }) + .parse(), + (argv) => { + run( + async ({ log }) => { + const outDir = argv.outDir as string; + await reportSyntaxErrors(outDir, log); + }, + { log: { defaultLevel: argv.logLevel as any }, flags: { allowUnexpected: true } } + ); + } + ) + .parse(); diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt index 0700d970972a4..05781981d1cf2 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-abs.txt @@ -1,6 +1,6 @@ # ABS -The ABS function returns the absolute value of a given number. +The `ABS` function returns the absolute value of a numeric expression. ## Syntax @@ -8,23 +8,23 @@ The ABS function returns the absolute value of a given number. ### Parameters -#### number +#### `number` -A numeric expression. If the parameter is `null`, the function will also return `null`. +A numeric expression. If the value is `null`, the function returns `null`. ## Examples -In this example, the ABS function is used to calculate the absolute value of -1.0: - ```esql ROW number = -1.0 | EVAL abs_number = ABS(number) ``` -In the following example, the ABS function is used to calculate the absolute value of the height of employees: +Calculate the absolute value of a negative number. ```esql FROM employees | KEEP first_name, last_name, height | EVAL abs_height = ABS(0.0 - height) -``` \ No newline at end of file +``` + +Calculate the absolute value of the difference between `0.0` and the `height` column. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt index 370e43bcf850f..5288762efe78c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-acos.txt @@ -1,6 +1,6 @@ # ACOS -The ACOS function returns the arccosine of a given number, expressed in radians. +Returns the arccosine of a number as an angle, expressed in radians. ## Syntax @@ -8,20 +8,16 @@ The ACOS function returns the arccosine of a given number, expressed in radians. ### Parameters -#### number +#### `number` -This is a number between -1 and 1. If the parameter is `null`, the function will also return `null`. +- A number between -1 and 1. +- If `null`, the function returns `null`. ## Examples -In this example, the ACOS function calculates the arccosine of 0.9. - ```esql -ROW a=.9 -| EVAL acos=ACOS(a) +ROW a = .9 +| EVAL acos = ACOS(a) ``` -```esql -ROW b = -0.5 -| EVAL acos_b = ACOS(b) -``` +Calculate the arccosine of the value `0.9` and store the result in a new column named `acos`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt index a7901b95b8931..92f2a18cd43bf 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-asin.txt @@ -1,6 +1,6 @@ # ASIN -The ASIN function returns the arcsine of a given numeric expression as an angle, expressed in radians. +Returns the arcsine of the input numeric expression as an angle, expressed in radians. ## Syntax @@ -8,22 +8,16 @@ The ASIN function returns the arcsine of a given numeric expression as an angle, ### Parameters -#### number +#### `number` -This is a numeric value ranging between -1 and 1. If the parameter is `null`, the function will also return `null`. +- A number between -1 and 1. +- If `null`, the function returns `null`. ## Examples -In this example, the ASIN function calculates the arcsine of 0.9: - -```esql -ROW a=.9 -| EVAL asin=ASIN(a) -``` - -In this example, the ASIN function calculates the arcsine of -0.5: - ```esql -ROW a = -.5 +ROW a = .9 | EVAL asin = ASIN(a) ``` + +Calculate the arcsine of the value `0.9` and return the result in radians. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt index a8b6f3dfa547c..fa64d90163c4d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-atan.txt @@ -1,6 +1,6 @@ # ATAN -The ATAN function returns the arctangent of a given numeric expression, expressed in radians. +Returns the arctangent of the input numeric expression as an angle, expressed in radians. ## Syntax @@ -8,9 +8,9 @@ The ATAN function returns the arctangent of a given numeric expression, expresse ### Parameters -#### number +#### `number` -This is a numeric expression. If the parameter is `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples @@ -19,6 +19,8 @@ ROW a=12.9 | EVAL atan = ATAN(a) ``` +Calculate the arctangent of the value `12.9` and store the result in a new column named `atan`. + ```esql ROW x=5.0, y=3.0 | EVAL atan_yx = ATAN(y / x) diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt index b9d209b4ef3a1..cfedb5c2e0295 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-avg.txt @@ -1,6 +1,6 @@ # AVG -The AVG function calculates the average of a numeric field. +The `AVG` function calculates the average of a numeric field. ## Syntax @@ -8,22 +8,26 @@ The AVG function calculates the average of a numeric field. ### Parameters -#### number +#### `number` -The numeric field for which the average is calculated. +A numeric field to calculate the average. ## Examples -Calculate the average height of employees: +Basic Usage ```esql FROM employees | STATS AVG(height) ``` -The AVG function can be used with inline functions. For example: +Calculate the average height of employees. + +Using Inline Functions ```esql FROM employees | STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10) ``` + +Calculate the average salary change by first averaging multiple values per row using `MV_AVG`, and then applying the `AVG` function with rounding to 10 decimal places. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bit_length.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bit_length.txt index 11a5b8f1728ae..4cae922384db9 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bit_length.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bit_length.txt @@ -1,6 +1,8 @@ # BIT_LENGTH -This function calculates the bit length of a given string. +Returns the bit length of a string. + +**Note:** All strings are in UTF-8, so a single character can use multiple bytes. ## Syntax @@ -8,11 +10,9 @@ This function calculates the bit length of a given string. ### Parameters -#### string - -This is the string whose bit length you want to calculate. If `null` is provided, the function will return `null`. +#### `string` -**Note**: Strings are in UTF-8 format, which means a single character may occupy multiple bytes. +String expression. If `null`, the function returns `null`. ## Examples @@ -22,3 +22,5 @@ FROM airports | KEEP city | EVAL fn_length = LENGTH(city), fn_bit_length = BIT_LENGTH(city) ``` + +This example calculates both the character length and the bit length of city names in airports located in India. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt index 53d889332f414..d8a109f6a3ec1 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt @@ -1,28 +1,28 @@ # BUCKET -The BUCKET function allows you to create groups of values, known as buckets, from a datetime or numeric input. The size of the buckets can be specified directly or determined based on a recommended count and values range. +The `BUCKET` function creates groups of values—buckets—out of a datetime or numeric input. The size of the buckets can either be provided directly or chosen based on a recommended count and values range. ## Syntax -`BUCKET(field, buckets [, from, to])` +`BUCKET(field, buckets, from, to)` ### Parameters -#### field +#### `field` A numeric or date expression from which to derive buckets. -#### buckets +#### `buckets` -The target number of buckets, or the desired bucket size if `from` and `to` parameters are omitted. +The target number of buckets or the desired bucket size if the `from` and `to` parameters are omitted. -#### from +#### `from` (optional) -(optional) The start of the range. This can be a number, a date, or a date expressed as a string. +The start of the range. Can be a number, a date, or a date expressed as a string. -#### to +#### `to` (optional) -(optional) The end of the range. This can be a number, a date, or a date expressed as a string. +The end of the range. Can be a number, a date, or a date expressed as a string. ## Important notes: @@ -36,7 +36,7 @@ When the bucket size is provided directly for time interval, it is expressed as ## Examples -For instance, asking for at most 20 buckets over a year results in monthly buckets: +Using a target number of buckets, a start of a range, and an end of a range ```esql FROM employees @@ -45,35 +45,53 @@ FROM employees | SORT hire_date ``` -If the desired bucket size is known in advance, simply provide it as the second argument, leaving the range out: +This example creates buckets for hire dates in 1985, aiming for 20 buckets. The actual number of buckets may vary depending on the range. + +Combine BUCKET with an aggregation to create a histogram ```esql FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" -| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week) +| STATS hires_per_month = COUNT(*) BY month = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") +| SORT month +``` + +This example calculates the number of hires per month in 1985. + +Asking for more buckets can result in a smaller range + +```esql +FROM employees +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 100, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") | SORT week ``` -BUCKET can also operate on numeric fields. For example, to create a salary histogram: +This example creates weekly buckets for hire dates in 1985, aiming for 100 buckets. + +Providing the bucket size directly ```esql FROM employees -| STATS COUNT(*) BY bs = BUCKET(salary, 20, 25324, 74999) -| SORT bs +| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" +| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week) +| SORT week ``` -BUCKET may be used in both the aggregating and grouping part of the STATS ... BY ... command provided that in the aggregating part the function is referenced by an alias defined in the grouping part, or that it is invoked with the exact same expression: +This example creates weekly buckets for hire dates in 1985 by directly specifying the bucket size. + +Creating a salary histogram ```esql FROM employees -| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.) -| SORT b1, b2 -| KEEP s1, b1, s2, b2 +| STATS COUNT(*) BY bs = BUCKET(salary, 20, 25324, 74999) +| SORT bs ``` -More examples: +This example creates a histogram of salaries, dividing the range into 20 buckets. + +Omitting the range when the bucket size is known -*Regrouping employees in buckets based on salary and counting them* ```esql FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" @@ -81,21 +99,20 @@ FROM employees | SORT b ``` -*Group data emitted over the last 24h into 25 buckets* +This example creates salary buckets with a fixed size of 5000. + +Create hourly buckets for the last 24 hours + ```esql FROM sample_data -| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW() +| WHERE @timestamp >= NOW() - 1 day AND @timestamp < NOW() | STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW()) ``` -*Similar to previous example but with fixed 1 hour bucket size* -```esql -FROM sample_data -| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW() -| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 1 hour) -``` +This example creates hourly buckets for the last 24 hours. + +Create monthly buckets for the year 1985 -*Group employees in 20 buckets based on their hire_date and then calculate the average salary for each bucket* ```esql FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" @@ -103,10 +120,27 @@ FROM employees | SORT bucket ``` -*Similar to previous example but using fixed 1 month buckets size* +This example calculates the average salary for each month in 1985. + +Using BUCKET in both aggregating and grouping parts of STATS + ```esql FROM employees -| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" -| STATS AVG(salary) BY bucket = BUCKET(hire_date, 1 month) -| SORT bucket +| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.) +| SORT b1, b2 +| KEEP s1, b1, s2, b2 +``` + +This example demonstrates advanced usage of `BUCKET` in both aggregation and grouping. + +Adjusting bucket start value with an offset + +```esql +FROM employees +| STATS dates = MV_SORT(VALUES(birth_date)) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR +| EVAL d_count = MV_COUNT(dates) +| SORT d_count, b +| LIMIT 3 ``` + +This example adjusts the bucket start value by adding and subtracting an offset. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-byte_length.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-byte_length.txt index b233190aa4cc2..77f1f07c20ee3 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-byte_length.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-byte_length.txt @@ -1,6 +1,6 @@ # BYTE_LENGTH -This function calculates the byte length of a given string. +Returns the byte length of a string. Since all strings are in UTF-8, a single character may use multiple bytes. ## Syntax @@ -8,9 +8,9 @@ This function calculates the byte length of a given string. ### Parameters -#### string +#### `string` -The text string for which the byte length is to be determined. If `null` is provided, the function will return `null`. +String expression. If `null`, the function returns `null`. ## Examples @@ -20,3 +20,5 @@ FROM airports | KEEP city | EVAL fn_length = LENGTH(city), fn_byte_length = BYTE_LENGTH(city) ``` + +This example calculates both the character length and the byte length of city names in airports located in India. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt index 5484a8706391c..c881010f905b4 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-case.txt @@ -1,28 +1,30 @@ -# CASE +## CASE -The CASE function accepts pairs of conditions and values. It returns the value that corresponds to the first condition that evaluates to `true`. If no condition matches, the function returns a default value or `null` if the number of arguments is even. +The `CASE` function evaluates a series of conditions and returns a value corresponding to the first condition that evaluates to `true`. If no conditions match, a default value or `null` is returned. ## Syntax -`CASE(condition, trueValue, elseValue)` +`CASE (condition, trueValue, elseValue)` ### Parameters -#### condition +#### `condition` A condition to evaluate. -#### trueValue +#### `trueValue` -The value that is returned when the corresponding condition is the first to evaluate to `true`. If no condition matches, the default value is returned. +The value returned when the corresponding condition is the first to evaluate to `true`. If no condition matches, the default value is returned. -#### elseValue +#### `elseValue` -The value that will be returned when no condition evaluates to `true`. +The value returned when no condition evaluates to `true`. ## Examples -In this example, employees are categorized as monolingual, bilingual, or polyglot depending on how many languages they speak: +### Determine whether employees are monolingual, bilingual, or polyglot + +Classify employees based on the number of languages they speak: ```esql FROM employees @@ -33,7 +35,9 @@ FROM employees | KEEP emp_no, languages, type ``` -Calculate the total connection success rate based on log messages: +### Calculate the total connection success rate based on log messages + +Determine the success rate of connections by analyzing log messages: ```esql FROM sample_data @@ -44,7 +48,9 @@ FROM sample_data | STATS success_rate = AVG(successful) ``` -Calculate an hourly error rate as a percentage of the total number of log messages: +### Calculate an hourly error rate as a percentage of the total number of log messages + +Compute the error rate for each hour based on log messages: ```esql FROM sample_data diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-categorize.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-categorize.txt index 1625aaa738448..4e5a3eeeea642 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-categorize.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-categorize.txt @@ -1,8 +1,6 @@ # CATEGORIZE -The `CATEGORIZE` function organizes textual data into groups of similar format. - -> **Note:** The `CATEGORIZE` function is currently in technical preview and may undergo changes or be removed in future releases. +Groups text messages into categories of similarly formatted text values. ## Syntax @@ -10,13 +8,15 @@ The `CATEGORIZE` function organizes textual data into groups of similar format. ### Parameters -#### field +#### `field` -The expression that is to be categorized. +Expression to categorize. ## Examples -The following example demonstrates how to use `CATEGORIZE` to group server log messages into categories and then aggregate their counts. +Categorizing server log messages + +Categorizes server log messages into categories and aggregates their counts. ```esql FROM sample_data @@ -25,6 +25,10 @@ FROM sample_data ## Limitations -- `CATEGORIZE` can't be used within other expressions -- `CATEGORIZE` can't be used with multiple groupings -- `CATEGORIZE` can't be used or referenced within aggregate functions +- Cannot be used within other expressions. +- Cannot be used with multiple groupings. +- Cannot be used or referenced within aggregate functions. + +## Additional Notes + +- The `CATEGORIZE` function requires a platinum license. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt index 6bbff254f2b16..cff1384c1b45b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cbrt.txt @@ -1,6 +1,6 @@ # CBRT -The CBRT function calculates the cube root of a given number. +Returns the cube root of a number. The input can be any numeric value, and the return value is always a double. Cube roots of infinities are `null`. ## Syntax @@ -8,9 +8,9 @@ The CBRT function calculates the cube root of a given number. ### Parameters -#### number +#### `number` -This is a numeric expression. If the parameter is `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples @@ -18,3 +18,5 @@ This is a numeric expression. If the parameter is `null`, the function will also ROW d = 1000.0 | EVAL c = cbrt(d) ``` + +Calculate the cube root of the value `1000.0`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt index 438d02cb6646d..076ae9a7231fb 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ceil.txt @@ -1,6 +1,6 @@ # CEIL -The CEIL function rounds a number up to the nearest integer. +Rounds a number up to the nearest integer. ## Syntax @@ -8,17 +8,21 @@ The CEIL function rounds a number up to the nearest integer. ### Parameters -#### number +#### `number` -This is a numeric expression. If the parameter is `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples +Rounding up a decimal number + ```esql ROW a=1.8 -| EVAL a=CEIL(a) +| EVAL a = CEIL(a) ``` +This example rounds the value `1.8` up to the nearest integer, resulting in `2`. + ## Limitations -- the CEIL function does not perform any operation for `long` (including unsigned) and `integer` types. For `double` type, it picks the closest `double` value to the integer, similar to the Math.ceil function in other programming languages. +- This function is a no-op for `long` (including unsigned) and `integer` types. For `double`, it selects the closest `double` value to the integer, similar to `Math.ceil`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt index 2dcb0ec9b6824..7b2167b1f0724 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cidr_match.txt @@ -1,6 +1,6 @@ -# CIDR_MATCH +## CIDR_MATCH -The CIDR_MATCH function checks if a given IP address falls within one or more specified CIDR blocks. +The `CIDR_MATCH` function checks if a given IP address is contained within one or more specified CIDR blocks. ## Syntax @@ -8,28 +8,31 @@ The CIDR_MATCH function checks if a given IP address falls within one or more sp ### Parameters -#### ip +#### `ip` -The IP address to be checked. This function supports both IPv4 and IPv6 addresses. +The IP address to test. Must be of type `ip` (supports both IPv4 and IPv6). -#### blockX +#### `blockX` -The CIDR block(s) against which the IP address is to be checked. +One or more CIDR blocks to test the IP address against. ## Examples -The following example checks if the IP address 'ip1' falls within the CIDR blocks "127.0.0.2/32": +Filtering IP addresses ```esql FROM hosts | WHERE CIDR_MATCH(ip1, "127.0.0.2/32") -| KEEP card, host, ip0, ip1 +| KEEP host, ip1 ``` -The function also supports passing multiple blockX: +Filtering IP addresses within specific CIDR blocks + ```esql -FROM network_logs -| WHERE CIDR_MATCH(source_ip, "192.168.1.0/24", "10.0.0.0/8") -| KEEP timestamp, source_ip, destination_ip, action +FROM hosts +| WHERE CIDR_MATCH(ip1, "127.0.0.2/32", "127.0.0.3/32") +| KEEP host, ip1 ``` + +This example filters rows where the `ip1` column contains an IP address that falls within the specified CIDR blocks (`127.0.0.2/32` or `127.0.0.3/32`). It then keeps the `card`, `host`, `ip0`, and `ip1` columns in the output. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt index ac0162fae33e6..94e8eb2b38f56 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-coalesce.txt @@ -1,6 +1,6 @@ # COALESCE -The COALESCE function returns the first non-null argument from the list of provided arguments. +Returns the first argument that is not null. If all arguments are null, it returns `null`. ## Syntax @@ -8,27 +8,29 @@ The COALESCE function returns the first non-null argument from the list of provi ### Parameters -#### first +#### `first` -The first expression to evaluate. +Expression to evaluate. -#### rest +#### `rest` -The subsequent expressions to evaluate. - -### Description - -The COALESCE function evaluates the provided expressions in order and returns the first non-null value it encounters. If all the expressions evaluate to null, the function returns null. +Other expressions to evaluate. ## Examples -In the following example, the COALESCE function evaluates the expressions 'a' and 'b'. Since 'a' is null, the function returns the value of 'b'. +Returning the first non-null value ```esql ROW a=null, b="b" | EVAL COALESCE(a, b) ``` +#### Result + +| a | b | EVAL_COALESCE_a_b | +|------|-----|-------------------| +| null | "b" | "b" | + COALESCE supports any number of rest parameters: ```esql diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt index ca464bc74d0c6..26b99dd19cdc8 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-concat.txt @@ -1,32 +1,29 @@ # CONCAT -The CONCAT function combines two or more strings into one. +Concatenates two or more strings. ## Syntax -`CONCAT(string1, string2, [...stringN])` +`CONCAT(string1, string2)` ### Parameters -#### string1 +#### `string1` The first string to concatenate. -#### string2 +#### `string2` The second string to concatenate. ## Examples -The following example concatenates the `street_1` and `street_2` fields: - ```esql FROM address | KEEP street_1, street_2 | EVAL fullstreet = CONCAT(street_1, street_2) ``` - CONCAT supports any number of string parameters. The following example concatenates the `first_name` and `last_name` fields with a space in between: ```esql diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt index 519af24229150..5eb5eb5d0348a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cos.txt @@ -1,6 +1,6 @@ # COS -The COS function calculates the cosine of a given angle. +The `COS` function returns the cosine of a given angle. ## Syntax @@ -8,18 +8,15 @@ The COS function calculates the cosine of a given angle. ### Parameters -#### angle +#### `angle` -The angle for which the cosine is to be calculated, expressed in radians. If the parameter is `null`, the function will return `null`. +An angle, in radians. If `null`, the function returns `null`. ## Examples ```esql ROW a=1.8 -| EVAL cos=COS(a) +| EVAL cos = COS(a) ``` -```esql -ROW angle=0.5 -| EVAL cosine_value = COS(angle) -``` +Calculate the cosine of the angle `1.8` radians. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt index ec9e8906a9467..b2b1974b6b5f4 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-cosh.txt @@ -1,25 +1,22 @@ # COSH -The COSH function calculates the hyperbolic cosine of a given angle. +Returns the hyperbolic cosine of a number. ## Syntax -`COSH(angle)` +`COSH(number)` ### Parameters -#### angle +#### number -The angle in radians for which the hyperbolic cosine is to be calculated. If the angle is null, the function will return null. +Numeric expression. If `null`, the function returns `null`. ## Examples ```esql ROW a=1.8 -| EVAL cosh=COSH(a) +| EVAL cosh = COSH(a) ``` -```esql -ROW angle=0.5 -| EVAL hyperbolic_cosine = COSH(angle) -``` +Calculate the hyperbolic cosine of the value `1.8`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt index dace14b709204..8b6f16e243ea9 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count.txt @@ -1,6 +1,6 @@ -# COUNT +## COUNT -The COUNT function returns the total number of input values. +The `COUNT` function returns the total number of input values. If no field is specified, it counts the number of rows. ## Syntax @@ -8,20 +8,22 @@ The COUNT function returns the total number of input values. ### Parameters -#### field +#### `field` -This is an expression that outputs values to be counted. If it's omitted, it's equivalent to `COUNT(*)`, which counts the number of rows. +An expression that outputs values to be counted. If omitted, the function is equivalent to `COUNT(*)`, which counts the number of rows. ## Examples -Count the number of specific field values: +### Count specific field values ```esql FROM employees | STATS COUNT(height) ``` -Count the number of rows using `COUNT()` or `COUNT(*)`: +Count the number of non-null values in the `height` field. + +### Count the number of rows ```esql FROM employees @@ -29,14 +31,18 @@ FROM employees | SORT languages DESC ``` -The expression can use inline functions. In this example, a string is split into multiple values using the `SPLIT` function, and the values are counted: +Count the total number of rows grouped by the `languages` field and sort the results in descending order. + +### Count values using inline functions ```esql ROW words="foo;bar;baz;qux;quux;foo" | STATS word_count = COUNT(SPLIT(words, ";")) ``` -To count the number of times an expression returns `TRUE`, use a `WHERE` command to remove rows that shouldn’t be included: +Count the number of elements in a string split by the `;` delimiter. + +### Count values based on a condition ```esql ROW n=1 @@ -44,9 +50,13 @@ ROW n=1 | STATS COUNT(n) ``` -To count the same stream of data based on two different expressions, use the pattern `COUNT( OR NULL)`: +Count the number of rows where the value of `n` is less than 0. + +### Count based on two different expressions ```esql ROW n=1 | STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL) -``` \ No newline at end of file +``` + +Count the number of rows where `n > 0` and `n < 0` separately. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt index f6918b6651562..965faa5327f9e 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-count_distinct.txt @@ -1,6 +1,6 @@ -# COUNT_DISTINCT +## COUNT_DISTINCT -The COUNT_DISTINCT function calculates the approximate number of distinct values in a specified field. +The `COUNT_DISTINCT` function returns the approximate number of distinct values in a column or expression. ## Syntax @@ -8,37 +8,43 @@ The COUNT_DISTINCT function calculates the approximate number of distinct values ### Parameters -#### field +#### `field` The column or literal for which to count the number of distinct values. -#### precision +#### `precision` -(Optional) The precision threshold. The counts are approximate. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000. +(Optional) The precision threshold. The maximum supported value is 40,000. Thresholds above this value will behave as if set to 40,000. The default value is 3,000. Higher precision thresholds may increase memory usage and processing time. ## Examples -The following example calculates the number of distinct values in the `ip0` and `ip1` fields: +Counting distinct values in multiple columns ```esql FROM hosts | STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1) ``` -You can also specify a precision threshold. In the following example, the precision threshold for `ip0` is set to 80000 and for `ip1` to 5: +This example calculates the approximate number of distinct values in the `ip0` and `ip1` columns. + +Configuring the precision threshold ```esql FROM hosts | STATS COUNT_DISTINCT(ip0, 80000), COUNT_DISTINCT(ip1, 5) ``` -The COUNT_DISTINCT function can also be used with inline functions. This example splits a string into multiple values using the `SPLIT` function and counts the unique values: +This example demonstrates how to specify a precision threshold for each column. The `ip0` column uses a high precision threshold of 80,000, while the `ip1` column uses a low threshold of 5. + +Counting distinct values from a split string ```esql ROW words="foo;bar;baz;qux;quux;foo" | STATS distinct_word_count = COUNT_DISTINCT(SPLIT(words, ";")) ``` +This example splits the `words` string into multiple values using the `SPLIT` function and counts the unique values. The result is the number of distinct words in the string. + ### Notes - Computing exact counts requires loading values into a set and returning its size, which doesn't scale well for high-cardinality sets or large values due to memory usage and communication overhead. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt index 7c0652aa4c067..84943f18bb487 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_diff.txt @@ -1,6 +1,6 @@ -# DATE_DIFF +## DATE_DIFF -The DATE_DIFF function calculates the difference between two timestamps and returns the difference in multiples of the specified `unit`. +Calculates the difference between two timestamps in multiples of a specified unit. If the start timestamp is later than the end timestamp, the result will be negative. ## Syntax @@ -8,32 +8,43 @@ The DATE_DIFF function calculates the difference between two timestamps and retu ### Parameters -#### unit +#### `unit` -The unit of time in which the difference will be calculated. +The unit of time for the difference calculation. -#### startTimestamp +#### `startTimestamp` -The starting timestamp for the calculation. +A string representing the starting timestamp. -#### endTimestamp +#### `endTimestamp` -The ending timestamp for the calculation. +A string representing the ending timestamp. ## Examples -The following example demonstrates how to use the DATE_DIFF function to calculate the difference between two timestamps in microseconds: +Calculate the difference in microseconds between two timestamps: ```esql ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z"), date2 = TO_DATETIME("2023-12-02T11:00:00.001Z") | EVAL dd_ms = DATE_DIFF("microseconds", date1, date2) ``` +Calculate the difference in calendar units (e.g., years) between timestamps. Only fully elapsed units are counted. To include remainders, switch to a smaller unit and perform additional calculations: + ```esql -ROW date1 = TO_DATETIME("2023-01-01T00:00:00.000Z"), date2 = TO_DATETIME("2023-12-31T23:59:59.999Z") -| EVAL dd_days = DATE_DIFF("days", date1, date2) +ROW end_23=TO_DATETIME("2023-12-31T23:59:59.999Z"), + start_24=TO_DATETIME("2024-01-01T00:00:00.000Z"), + end_24=TO_DATETIME("2024-12-31T23:59:59.999") +| EVAL end23_to_start24 = DATE_DIFF("year", end_23, start_24) +| EVAL end23_to_end24 = DATE_DIFF("year", end_23, end_24) +| EVAL start_to_end_24 = DATE_DIFF("year", start_24, end_24) ``` +## Limitations + +- The function’s supported units and ES|QL’s time span literals are distinct and not interchangeable. +- Supported abbreviations align with other established implementations but may differ from Elasticsearch’s date-time nomenclature. + ## Notes - If the `startTimestamp` is later than the `endTimestamp`, the function will return a negative value. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt index 555ea5b1f3cc5..30647f30e905a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_extract.txt @@ -1,6 +1,6 @@ # DATE_EXTRACT -The DATE_EXTRACT function is used to extract specific parts of a date. +Extracts specific parts of a date, such as the year, month, day, or hour. ## Syntax @@ -8,26 +8,63 @@ The DATE_EXTRACT function is used to extract specific parts of a date. ### Parameters -#### datePart +#### `datePart` -This is the part of the date you want to extract, such as "year", "month" or "hour_of_day". +The part of the date to extract. Supported values include: -#### date +- `aligned_day_of_week_in_month` +- `aligned_day_of_week_in_year` +- `aligned_week_of_month` +- `aligned_week_of_year` +- `ampm_of_day` +- `clock_hour_of_ampm` +- `clock_hour_of_day` +- `day_of_month` +- `day_of_week` +- `day_of_year` +- `epoch_day` +- `era` +- `hour_of_ampm` +- `hour_of_day` +- `instant_seconds` +- `micro_of_day` +- `micro_of_second` +- `milli_of_day` +- `milli_of_second` +- `minute_of_day` +- `minute_of_hour` +- `month_of_year` +- `nano_of_day` +- `nano_of_second` +- `offset_seconds` +- `proleptic_month` +- `second_of_day` +- `second_of_minute` +- `year` +- `year_of_era` -This is the date expression. +Refer to `java.time.temporal.ChronoField` for detailed descriptions of these values. If `null`, the function returns `null`. + +#### `date` + +The date expression from which to extract the specified part. If `null`, the function returns `null`. ## Examples -To extract the year from a date: +### Extracting the Year from a Date + +Extract the year from a given date: ```esql ROW date = DATE_PARSE("yyyy-MM-dd", "2022-05-06") | EVAL year = DATE_EXTRACT("year", date) ``` -To find all events that occurred outside of business hours (before 9 AM or after 5PM), on any given date: +### Filtering Events Outside Business Hours + +Retrieve all events that occurred outside of business hours (before 9 AM or after 5 PM): ```esql FROM sample_data | WHERE DATE_EXTRACT("hour_of_day", @timestamp) < 9 AND DATE_EXTRACT("hour_of_day", @timestamp) >= 17 -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt index 4b8a8a174ab80..42b0ea4e2672b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_format.txt @@ -1,6 +1,6 @@ # DATE_FORMAT -The DATE_FORMAT function returns a string representation of a date, formatted according to the provided format. +The `DATE_FORMAT` function returns a string representation of a date in the specified format. ## Syntax @@ -8,21 +8,25 @@ The DATE_FORMAT function returns a string representation of a date, formatted ac ### Parameters -#### dateFormat +#### `dateFormat` -This is an optional parameter that specifies the desired date format. -If no format is provided, the function defaults to the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format. +- **Optional** +- Specifies the date format. If no format is provided, the default format `yyyy-MM-dd'T'HH:mm:ss.SSSZ` is used. +- If `null`, the function returns `null`. -#### date +#### `date` -This is the date expression that you want to format. +- A date expression. +- If `null`, the function returns `null`. ## Examples -In this example, the `hire_date` field is formatted according to the "YYYY-MM-dd" format, and the result is stored in the `hired` field: +Formatting a date to `yyyy-MM-dd` ```esql FROM employees | KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +| EVAL hired = DATE_FORMAT("yyyy-MM-dd", hire_date) ``` + +This example formats the `hire_date` field into the `yyyy-MM-dd` format and stores the result in a new column named `hired`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt index d949c92156ba4..ae615fad8c9ab 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_parse.txt @@ -1,6 +1,6 @@ # DATE_PARSE -The DATE_PARSE function is used to convert a date string into a date based on the provided format pattern. +Parses a date string into a date object using the specified format. ## Syntax @@ -8,22 +8,21 @@ The DATE_PARSE function is used to convert a date string into a date based on th ### Parameters -#### datePattern +#### `datePattern` -This is the format of the date. If `null` is provided, the function will return `null`. +The date format. Refer to the `DateTimeFormatter` documentation for the syntax. If `null`, the function returns `null`. -#### dateString +#### `dateString` -This is the date expression in string format. +A date expression as a string. If `null` or an empty string, the function returns `null`. ## Examples +Parsing a date string + ```esql ROW date_string = "2022-05-06" | EVAL date = DATE_PARSE("yyyy-MM-dd", date_string) ``` -```esql -FROM logs -| EVAL date = DATE_PARSE("yyyy-MM-dd", date_string) -``` +This example parses the string `"2022-05-06"` into a date object using the format `"yyyy-MM-dd"`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt index 7df5249933821..78aaa51d10956 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt @@ -1,6 +1,6 @@ -# DATE_TRUNC +## DATE_TRUNC -The DATE_TRUNC function rounds down a date to the nearest specified interval. +The `DATE_TRUNC` function rounds down a date to the closest specified interval. ## Syntax @@ -8,25 +8,17 @@ The DATE_TRUNC function rounds down a date to the nearest specified interval. ### Parameters -#### interval +#### `interval` -This is the interval to which the date will be rounded down. It is expressed using the timespan literal syntax. +The interval to which the date is rounded down, expressed using the timespan literal syntax. -#### date +#### `date` -This is the date expression that will be rounded down. - -## Important notes - -The *interval* parameter of DATE_TRUNC is a timespan literal, NOT a string. - - GOOD: `DATE_TRUNC(1 year, date)` - - BAD: `DATE_TRUNC("year", date)` - -When grouping data by time interval, it is recommended to use BUCKET instead of DATE_TRUNC. +The date expression to be truncated. ## Examples -The following example rounds down the hire_date to the nearest year: +Truncate hire dates to the year ```esql FROM employees @@ -34,7 +26,9 @@ FROM employees | EVAL year_hired = DATE_TRUNC(1 year, hire_date) ``` -You can combine DATE_TRUNC with STATS ... BY to create date histograms. For example, the number of hires per year: +This example truncates the `hire_date` field to the beginning of the year and stores the result in a new column named `year_hired`. + +Number of hires per year ```esql FROM employees @@ -43,7 +37,9 @@ FROM employees | SORT year ``` -Or, you can calculate an hourly error rate: +This example calculates the number of hires per year by truncating the `hire_date` field to the year and grouping the results. + +Hourly error rate ```esql FROM sample_data @@ -52,3 +48,5 @@ FROM sample_data | STATS error_rate = AVG(error) BY hour | SORT hour ``` + +This example calculates the hourly error rate by truncating the `@timestamp` field to the hour and averaging the `error` values for each hour. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt index 8f4a822e52f07..56a1885ca1341 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-dissect.txt @@ -1,6 +1,6 @@ -# DISSECT +## DISSECT -The DISSECT command is used to extract structured data from a string. It matches the string against a delimiter-based pattern and extracts the specified keys as columns. +The `DISSECT` command is used to extract structured data from a string. It matches the string against a delimiter-based pattern and extracts the specified keys as columns. ### Use Cases - **Log Parsing**: Extracting timestamps, log levels, and messages from log entries. @@ -13,21 +13,23 @@ The DISSECT command is used to extract structured data from a string. It matches ### Parameters -#### input +#### `input` -The column containing the string you want to structure. If the column has multiple values, DISSECT will process each value. +The column containing the string you want to structure. If the column has multiple values, `DISSECT` will process each value. -#### pattern +#### `pattern` A dissect pattern. If a field name conflicts with an existing column, the existing column is dropped. If a field name is used more than once, only the rightmost duplicate creates a column. -#### +#### `` A string used as the separator between appended values, when using the append modifier. ## Examples -The following example parses a string that contains a timestamp, some text, and an IP address: +Parsing a string with a timestamp, text, and IP address + +Extracts the `date`, `msg`, and `ip` fields from a structured string. ```esql ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" @@ -35,7 +37,9 @@ ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" | KEEP date, msg, ip ``` -By default, DISSECT outputs keyword string columns. To convert to another type, use Type conversion functions: +Converting output to another type + +Converts the `date` field from a string to a datetime type after extracting it. ```esql ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" @@ -43,7 +47,6 @@ ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" | KEEP date, msg, ip | EVAL date = TO_DATETIME(date) ``` - In this example, we use the `APPEND_SEPARATOR` to concatenate values with a custom separator: ```esql diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt index 2e36f20474aef..ad2cf4bd6b1cd 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-drop.txt @@ -1,6 +1,6 @@ # DROP -The DROP command is used to eliminate one or more columns from the data. +The `DROP` command removes one or more columns from the result set. ## Syntax @@ -10,31 +10,17 @@ The DROP command is used to eliminate one or more columns from the data. #### columns -This is a list of columns, separated by commas, that you want to remove. Wildcards are supported. +A comma-separated list of columns to remove. Supports wildcards. ## Examples -In the following example, the 'height' column is removed from the data: +Remove a specific column: ```esql FROM employees | DROP height ``` -You can also use wildcards to remove all columns that match a certain pattern. In the following example, all columns that start with 'height' are removed: - -```esql -FROM employees -| DROP height* -``` - -This example demonstrates how to drop multiple specific columns by listing them in a comma-separated format. - -```esql -FROM employees -| DROP height, weight, age -``` - This example shows how to drop columns that match a more complex pattern using wildcards. ```esql diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt index 2ab4a7e3449da..08f9aba8a3456 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-e.txt @@ -1,6 +1,6 @@ # E -The E function returns Euler's number. +Returns Euler’s number. ## Syntax @@ -8,7 +8,7 @@ The E function returns Euler's number. ### Parameters -This function does not require any parameters. +This function does not take any parameters. ## Examples @@ -16,8 +16,4 @@ This function does not require any parameters. ROW E() ``` -```esql -FROM employees -| EVAL euler_number = E() -| KEEP euler_number -``` +This example returns Euler’s number. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt index 0ceafe99f528d..dbbbbe2f07584 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ends_with.txt @@ -1,6 +1,6 @@ # ENDS_WITH -The ENDS_WITH function checks if a given string ends with a specified suffix. +Determines whether a keyword string ends with a specified suffix and returns a boolean value. ## Syntax @@ -8,25 +8,20 @@ The ENDS_WITH function checks if a given string ends with a specified suffix. ### Parameters -#### str +#### `str` -This is the string expression that you want to check. +String expression. If `null`, the function returns `null`. -#### suffix +#### `suffix` -The string expression that will be checked if it is the ending of the first string. +String expression. If `null`, the function returns `null`. ## Examples - ```esql FROM employees | KEEP last_name | EVAL ln_E = ENDS_WITH(last_name, "d") ``` -```esql -FROM employees -| KEEP first_name -| EVAL fn_E = ENDS_WITH(first_name, "a") -``` +This example checks if the `last_name` column values end with the letter "d" and stores the result in a new column `ln_E`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt index 9587732048639..3a6dfef829a7c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-enrich.txt @@ -1,6 +1,6 @@ -# ENRICH +## ENRICH -The ENRICH command allows you to add data from existing indices as new columns using an enrich policy. +The `ENRICH` command allows you to add data from existing indices as new columns using an enrich policy. ## Syntax @@ -8,53 +8,71 @@ The ENRICH command allows you to add data from existing indices as new columns u ### Parameters -#### policy +#### `policy` -The name of the enrich policy. You need to create and execute the enrich policy first. +The name of the enrich policy. You must create and execute the enrich policy before using it. -#### match_field +#### `mode` -The match field. ENRICH uses its value to look for records in the enrich index. If not specified, the match will be performed on the column with the same name as the `match_field` defined in the enrich policy. +(Optional) The mode of the enrich command in cross-cluster queries. Refer to enrich across clusters for more details. -#### new_nameX +#### `match_field` -Allows you to change the name of the column that’s added for each of the enrich fields. Defaults to the enrich field name. If a column has the same name as the new name, it will be discarded. If a name (new or original) occurs more than once, only the rightmost duplicate creates a new column. +(Optional) The field used to match records in the enrich index. If not specified, the match is performed on the column with the same name as the `match_field` defined in the enrich policy. -#### fieldX +#### `fieldX` -The enrich fields from the enrich index that are added to the result as new columns. If a column with the same name as the enrich field already exists, the existing column will be replaced by the new column. If not specified, each of the enrich fields defined in the policy is added. A column with the same name as the enrich field will be dropped unless the enrich field is renamed. +(Optional) The enrich fields from the enrich index to be added as new columns. If a column with the same name as the enrich field already exists, it will be replaced. If not specified, all enrich fields defined in the policy are added. Columns with the same name as the enrich fields will be dropped unless renamed. + +#### `new_nameX` + +(Optional) Allows you to rename the columns added for each enrich field. Defaults to the enrich field name. If a column with the same name as the new name already exists, it will be discarded. If a name (new or original) occurs more than once, only the rightmost duplicate creates a column. ## Examples -The following example uses the `languages_policy` enrich policy to add a new column for each enrich field defined in the policy. The match is performed using the `match_field` defined in the enrich policy and requires that the input table has a column with the same name (`language_code` in this example). ENRICH will look for records in the enrich index based on the match field value. +Basic usage + +Add a new column for each enrich field defined in the `languages_policy` enrich policy. The match is performed using the `match_field` defined in the policy, requiring the input table to have a column with the same name (`language_code` in this case). ```esql ROW language_code = "1" | ENRICH languages_policy ``` -To use a column with a different name than the `match_field` defined in the policy as the match field, use `ON `: +Using a different match field + +Use a column with a different name than the `match_field` defined in the policy as the match field. ```esql ROW a = "1" | ENRICH languages_policy ON a ``` -By default, each of the enrich fields defined in the policy is added as a column. To explicitly select the enrich fields that are added, use `WITH , , ...`: +Selecting specific enrich fields + +Explicitly select the enrich fields to be added as columns. ```esql ROW a = "1" | ENRICH languages_policy ON a WITH language_name ``` -You can rename the columns that are added using `WITH new_name=`: +Renaming added columns + +Rename the columns added using the `WITH` clause. ```esql ROW a = "1" | ENRICH languages_policy ON a WITH name = language_name ``` -### Limitations -- In case of name collisions, the newly created columns will override existing columns. -- The ENRICH command only supports enrich policies of type `match`. -- ENRICH only supports enriching on a column of type `keyword`. +In case of name collisions, the newly created columns will override existing columns. + +## Limitations + +- The `ENRICH` command requires an existing enrich policy to be created and executed beforehand. +- The `match_field` in the `ENRICH` command must match the type defined in the enrich policy. For example: + - A `geo_match` policy requires a `match_field` of type `geo_point` or `geo_shape`. + - A `range` policy requires a `match_field` of type `integer`, `long`, `date`, or `ip`, depending on the range field type in the enrich index. + - For `range` policies, if the `match_field` is of type `KEYWORD`, field values are parsed during query execution. If parsing fails, the output values for that row are set to `null`, and a warning is produced. +- The `geo_match` enrich policy type only supports the `intersects` spatial relation. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt index 5cf834f5e9618..24666bfa2878f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-eval.txt @@ -1,6 +1,6 @@ # EVAL -The EVAL command allows you to append new columns with calculated values to your data. +The `EVAL` command allows you to add new columns with calculated values to your dataset. ## Syntax @@ -8,13 +8,16 @@ The EVAL command allows you to append new columns with calculated values to your ### Parameters -#### {columnX} +#### `columnX` -This is the name of the column. If a column with the same name already exists, it will be replaced. If a column name is used more than once, only the rightmost duplicate will create a column. +- The name of the column to be added or updated. +- If a column with the same name already exists, it will be replaced by the new column. +- If a column name is used multiple times, only the rightmost definition is applied. -#### {valueX} +#### `valueX` -This is the value for the column. It can be a literal, an expression, or a function. Columns defined to the left of this one can be used. +- The value to assign to the column. This can be a literal, an expression, or a function. +- You can reference columns defined earlier in the same `EVAL` command. ## Notes @@ -30,7 +33,9 @@ Aggregation functions are NOT supported for EVAL. ## Examples -The following example multiplies the `height` column by 3.281 and 100 to create new columns `height_feet` and `height_cm`: +### Adding new calculated columns + +Add two new columns, `height_feet` and `height_cm`, by performing calculations on the `height` column: ```esql FROM employees @@ -39,7 +44,9 @@ FROM employees | EVAL height_feet = height * 3.281, height_cm = height * 100 ``` -If the specified column already exists, the existing column will be replaced, and the new column will be appended to the table: +### Overwriting an existing column + +Replace the `height` column with a new value calculated by converting it to feet: ```esql FROM employees @@ -48,7 +55,9 @@ FROM employees | EVAL height = height * 3.281 ``` -Specifying the output column name is optional. If not specified, the new column name is equal to the expression. The following query adds a column named `height*3.281`: +### Adding a column without specifying a name + +If no column name is provided, the new column will be named after the expression itself. For example, this query adds a column named `height*3.281`: ```esql FROM employees @@ -57,7 +66,9 @@ FROM employees | EVAL height * 3.281 ``` -Because this name contains special characters, it needs to be quoted with backticks (`) when using it in subsequent commands: +### Using a column with special characters in subsequent commands + +When a column name contains special characters, enclose it in backticks (`) to reference it in later commands: ```esql FROM employees diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt index ec8374e9a6a7b..571a678aadd63 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-exp.txt @@ -1,6 +1,6 @@ # EXP -The EXP function calculates the value of Euler's number (e) raised to the power of a given number. +Returns the value of e raised to the power of the given number. ## Syntax @@ -8,19 +8,15 @@ The EXP function calculates the value of Euler's number (e) raised to the power ### Parameters -#### number +#### `number` -A numeric expression. If the parameter is `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples - ```esql ROW d = 5.0 | EVAL s = EXP(d) ``` -```esql -FROM geo -| EVAL exp = EXP(x) -``` +Calculate e raised to the power of 5.0. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt index eac92ffc434b5..2375a19cf645c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-floor.txt @@ -1,6 +1,6 @@ # FLOOR -The FLOOR function rounds a number down to the nearest integer. +Rounds a number down to the nearest integer. For `double` values, it selects the closest `double` representation of the integer, similar to `Math.floor`. For `long` (including unsigned) and `integer`, this operation has no effect. ## Syntax @@ -8,22 +8,25 @@ The FLOOR function rounds a number down to the nearest integer. ### Parameters -#### number +#### `number` -This is a numeric expression. If the parameter is `null`, the function will return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples ```esql ROW a=1.8 -| EVAL a=FLOOR(a) +| EVAL a = FLOOR(a) ``` +Rounds the value `1.8` down to `1`. + ```esql FROM employees | KEEP first_name, last_name, height | EVAL height_floor = FLOOR(height) ``` +Rounds all values in the column `height` down to nearest integer ## Notes diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt index 2f3618dce2412..f2351aec5d203 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt @@ -1,6 +1,6 @@ -# FROM +## FROM -The `FROM` command retrieves a table of data from a specified data stream, index, or alias. +The `FROM` command retrieves data from a data stream, index, or alias and returns it as a table. Each row in the table represents a document, and each column corresponds to a field that can be accessed by its name. ## Syntax @@ -8,54 +8,73 @@ The `FROM` command retrieves a table of data from a specified data stream, index ### Parameters -#### index_pattern +#### `index_pattern` -This parameter represents a list of indices, data streams, or aliases. It supports the use of wildcards and date math. +A list of indices, data streams, or aliases. Supports wildcards and date math. -#### fields +#### `fields` -This is a comma-separated list of metadata fields to be retrieved. +A comma-separated list of metadata fields to retrieve. -## Description +## Examples -The `FROM` command retrieves a table of data from a specified data stream, index, or alias. Each row in the resulting table represents a document, and each column corresponds to a field. The field can be accessed using its name. +### Basic Example + +Retrieve all documents from the `employees` index: -#### Basic Data Retrieval ```esql FROM employees ``` -#### Time Series Data -Use date math to refer to indices, aliases, and data streams. This can be useful for time series data, for example, to access today’s index: +### Using Date Math + +Access indices, aliases, or data streams using date math. For example, retrieve today’s index for time series data: + ```esql FROM ``` -#### Multiple Indices -Use comma-separated lists or wildcards to query multiple data streams, indices, or aliases: +### Querying Multiple Data Streams, Indices, or Aliases + +Query multiple data streams, indices, or aliases using a comma-separated list or wildcards: + ```esql FROM employees-00001,other-employees-* ``` -#### Remote Clusters -Use the format `:` to query data streams and indices on remote clusters: +### Querying Across Clusters + +Query data streams and indices on remote clusters using the format `:`: + ```esql FROM cluster_one:employees-00001,cluster_two:other-employees-* ``` -#### Metadata Retrieval -Use the optional `METADATA` directive to enable metadata fields: +### Using the `METADATA` Directive + +Enable metadata fields by using the optional `METADATA` directive: + ```esql FROM employees METADATA _id ``` -#### Escaping Special Characters -Use enclosing double quotes (") or three enclosing double quotes (""") to escape index names that contain special characters: +### Escaping Index Names + +Escape index names containing special characters using double quotes (`"`) or triple double quotes (`"""`): + ```esql FROM "this=that","""this[that""" ``` -### Limitations +## Limitations -- By default, an ES|QL query without an explicit `LIMIT` uses an implicit limit of 1000 rows. This applies to the `FROM` command as well. -- Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. +- By default, the `FROM` command applies an implicit limit of 1,000 rows if no explicit `LIMIT` is specified. For example: + ```esql +FROM employees +``` + is equivalent to: + ```esql +FROM employees +| LIMIT 1000 +``` +- Queries cannot return more than 10,000 rows, even if a higher `LIMIT` is specified. This is a configurable upper limit. For more details, refer to the [LIMIT command](#LIMIT). \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt index feb119185c72b..2ec2f1bee371d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-greatest.txt @@ -1,6 +1,6 @@ # GREATEST -The GREATEST function returns the maximum value from multiple columns. +Returns the maximum value from multiple columns. This function is similar to `MV_MAX` but is designed to operate on multiple columns simultaneously. ## Syntax @@ -8,27 +8,26 @@ The GREATEST function returns the maximum value from multiple columns. ### Parameters -#### first +#### `first` The first column to evaluate. -#### rest +#### `rest` The remaining columns to evaluate. ## Examples +Finding the maximum value between two columns + ```esql ROW a = 10, b = 20 | EVAL g = GREATEST(a, b) ``` -```esql -ROW x = "apple", y = "banana", z = "cherry" -| EVAL max_fruit = GREATEST(x, y, z) -``` +This example evaluates the maximum value between columns `a` and `b`, resulting in `g = 20`. ## Notes -- When applied to `keyword` or `text` fields, the GREATEST function returns the last string in alphabetical order. -- When applied to `boolean` columns, it returns `true` if any values are `true`. +- When applied to `keyword` or `text` fields, the function returns the last string in alphabetical order. +- When applied to `boolean` columns, the function returns `true` if any of the values are `true`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt index 2f7fa48df693f..8948d9c64aab6 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-grok.txt @@ -1,6 +1,6 @@ -# GROK +## GROK -The GROK command is used to extract structured data from a string. It matches the string against patterns based on regular expressions and extracts the specified patterns as columns. +The `GROK` command is used to extract structured data from a string. It matches the string against patterns based on regular expressions and extracts the specified patterns as columns. ## Syntax @@ -8,47 +8,57 @@ The GROK command is used to extract structured data from a string. It matches th ### Parameters -#### input +#### `input` -The column containing the string you want to structure. If the column has multiple values, GROK will process each value. +The column containing the string you want to structure. If the column has multiple values, `GROK` will process each value. -#### pattern +#### `pattern` -A grok pattern. If a field name conflicts with an existing column, the existing column is dropped. If a field name is used more than once, a multi-valued column is created with one value per each occurrence of the field name. +A grok pattern. +- If a field name conflicts with an existing column, the existing column is discarded. +- If a field name is used more than once, a multi-valued column will be created with one value for each occurrence of the field name. ## Examples -The following example parses a string that contains a timestamp, an IP address, an email address, and a number: +Parsing a string with multiple data types + +Parse a string containing a timestamp, an IP address, an email address, and a number: ```esql ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}" +| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}""" | KEEP date, ip, email, num ``` -By default, GROK outputs keyword string columns. `int` and `float` types can be converted by appending `:type` to the semantics in the pattern. For example `{NUMBER:num:int}`: +Type conversion for numeric fields + +Convert numeric fields to specific types by appending `:type` to the semantics in the pattern. For example, `{NUMBER:num:int}` converts the `num` field to an integer: ```esql ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" +| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}""" | KEEP date, ip, email, num ``` -For other type conversions, use Type conversion functions: +Using type conversion functions + +For other type conversions, use type conversion functions like `TO_DATETIME`: ```esql ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" -| GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" +| GROK a """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}""" | KEEP date, ip, email, num | EVAL date = TO_DATETIME(date) ``` -If a field name is used more than once, GROK creates a multi-valued column: +Handling multi-valued columns + +When a field name is used more than once, `GROK` creates a multi-valued column: ```esql FROM addresses | KEEP city.name, zip_code -| GROK zip_code "%{WORD:zip_parts} %{WORD:zip_parts}" +| GROK zip_code """%{WORD:zip_parts} %{WORD:zip_parts}""" ``` ### Limitations diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hash.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hash.txt index 5485869a90e45..86aaac976f99d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hash.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hash.txt @@ -1,6 +1,6 @@ # HASH -The HASH function computes the hash of a given input using a specified algorithm. +Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, and SHA-512. ## Syntax @@ -8,23 +8,21 @@ The HASH function computes the hash of a given input using a specified algorithm ### Parameters -#### algorithm +#### `algorithm` -The hash algorithm to be used. +Hash algorithm to use. -The supported algorithms are: -- "MD5" -- "SHA-1" -- "SHA-256" +#### `input` -#### input - -The value to be hashed. +Input to hash. ## Examples ```esql -FROM messages -| EVAL hashed_content = HASH("SHA-1", content) -| KEEP message_id, hashed_content +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) +| KEEP message, md5, sha256 ``` + +This example computes the MD5 and SHA-256 hashes of the `message` field for rows where the `message` is not "Connection error". diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hypot.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hypot.txt index 8033c8e7c33d2..db1f05d31fb6f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hypot.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-hypot.txt @@ -1,6 +1,6 @@ # HYPOT -The HYPOT function is used to calculate the hypotenuse of two numbers. +Calculates the hypotenuse of two numbers. The input can be any numeric values, and the return value is always a double. If either input is `null`, the function returns `null`. Hypotenuses of infinities are also `null`. ## Syntax @@ -10,19 +10,17 @@ The HYPOT function is used to calculate the hypotenuse of two numbers. #### number1 -This is a numeric value. If it's `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. #### number2 -This is also a numeric value. If it's `null`, the function will also return `null`. +Numeric expression. If `null`, the function returns `null`. ## Examples -Check the hypotenuse of two variables through the following example: - ```esql ROW a = 3.0, b = 4.0 | EVAL c = HYPOT(a, b) ``` -Note that the HYPOT function returns the hypotenuse in double data type. Besides, if any of the numbers is infinity, the hypotenuse returns `null`. \ No newline at end of file +Calculates the hypotenuse of a right triangle with sides `a = 3.0` and `b = 4.0`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt index e06773023ebdd..5f49a5e012571 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ip_prefix.txt @@ -1,6 +1,6 @@ # IP_PREFIX -The IP_PREFIX function truncates an IP address to a specified prefix length. +Truncates an IP address to a specified prefix length. ## Syntax @@ -8,27 +8,25 @@ The IP_PREFIX function truncates an IP address to a specified prefix length. ### Parameters -#### ip +#### `ip` -The IP address that you want to truncate. This function supports both IPv4 and IPv6 addresses. +The IP address to truncate. Supports both IPv4 and IPv6 addresses and must be of type `ip`. -#### prefixLengthV4 +#### `prefixLengthV4` -The prefix length for IPv4 addresses. +The prefix length to apply for IPv4 addresses. -#### prefixLengthV6 +#### `prefixLengthV6` -The prefix length for IPv6 addresses. +The prefix length to apply for IPv6 addresses. ## Examples +Truncating IPv4 and IPv6 addresses + ```esql ROW ip4 = TO_IP("1.2.3.4"), ip6 = TO_IP("fe80::cae2:65ff:fece:feb9") | EVAL ip4_prefix = IP_PREFIX(ip4, 24, 0), ip6_prefix = IP_PREFIX(ip6, 0, 112) ``` -```esql -FROM network_logs -| EVAL truncated_ip = IP_PREFIX(ip_address, 16, 0) -| KEEP ip_address, truncated_ip -``` +This example truncates the IPv4 address `1.2.3.4` to a `/24` prefix and the IPv6 address `fe80::cae2:65ff:fece:feb9` to a `/112` prefix. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt index b55077f6b5437..c5cb2f56a8f34 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-keep.txt @@ -1,6 +1,6 @@ -# KEEP +## KEEP -The KEEP command allows you to specify which columns to return and in what order. +The `KEEP` command specifies which columns are returned and the order in which they appear in the output. ## Syntax @@ -8,9 +8,9 @@ The KEEP command allows you to specify which columns to return and in what order ### Parameters -#### columns +#### `columns` -A comma-separated list of columns to retain. Wildcards are supported. If an existing column matches multiple provided wildcards or column names, certain rules apply. +A comma-separated list of columns to retain. Supports wildcards. If a column matches multiple expressions, precedence rules determine the final output. ## Note @@ -28,46 +28,54 @@ Important: only the columns in the KEEP command can be used after a KEEP command ## Examples -Return columns in a specified order: +### Return columns in a specific order + +The following query returns the `emp_no`, `first_name`, `last_name`, and `height` columns in the specified order: ```esql FROM employees | KEEP emp_no, first_name, last_name, height ``` -If you do not want to mention each column by name, you can use wildcards to select all columns that match a certain pattern: +### Use wildcards to match column names + +This query keeps all columns with names starting with `h`: ```esql FROM employees | KEEP h* ``` -The wildcard asterisk (`*`) by itself translates to all columns that are not matched by other arguments. +### Combine specific columns and wildcards -This command will first return all columns with a name that starts with `h`, followed by all other columns: +The asterisk wildcard (`*`) matches all columns not explicitly specified. This query returns all columns starting with `h` first, followed by all other columns: ```esql FROM employees | KEEP h*, * ``` -The following examples demonstrate how precedence rules function when a field name corresponds to multiple expressions. +### Precedence of complete field names over wildcards -Clear field name takes precedence over wildcard expressions: +When a column matches both a complete field name and a wildcard, the complete field name takes precedence: ```esql FROM employees | KEEP first_name, last_name, first_name* ``` -Wildcard expressions have the same priority, with the last one winning (despite it being a less specific match): +### Wildcard precedence and ordering + +If a column matches multiple wildcard expressions, the rightmost expression takes precedence, even if it is less specific: ```esql FROM employees | KEEP first_name*, last_name, first_na* ``` -A simple wildcard expression `*` has the minimum precedence. The sequence of output is determined by other arguments: +### Lowest precedence for the `*` wildcard + +The `*` wildcard has the lowest precedence. The order of other arguments determines the output order: ```esql FROM employees diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-kql.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-kql.txt new file mode 100644 index 0000000000000..c70a20326c116 --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-kql.txt @@ -0,0 +1,25 @@ +## KQL + +Performs a KQL query and returns `true` if the provided KQL query string matches the row. + +## Syntax + +`KQL(query)` + +### Parameters + +#### `query` + +Query string in KQL query string format. + +## Examples + +```esql +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5 +``` + +This example filters rows where the `author` field matches "Faulkner," keeps the `book_no` and `author` columns, sorts the results by `book_no`, and limits the output to 5 rows. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt index 7e0f77bc911eb..0a05a9b57c2cc 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-least.txt @@ -1,6 +1,6 @@ -# LEAST +## LEAST -The LEAST function returns the smallest value from multiple columns. +Returns the minimum value from multiple columns. This function is similar to `MV_MIN` but is designed to operate on multiple columns simultaneously. ## Syntax @@ -8,11 +8,11 @@ The LEAST function returns the smallest value from multiple columns. ### Parameters -#### first +#### `first` The first column to evaluate. -#### rest +#### `rest` The remaining columns to evaluate. @@ -23,7 +23,4 @@ ROW a = 10, b = 20 | EVAL l = LEAST(a, b) ``` -```esql -ROW x = 5, y = 15, z = 10 -| EVAL min_value = LEAST(x, y, z) -``` +This example calculates the minimum value between columns `a` and `b`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt index 74997f638f463..26492f9b15e03 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-left.txt @@ -1,6 +1,6 @@ # LEFT -The LEFT function returns a substring from the beginning of a specified string. +Returns a substring that extracts a specified number of characters from the beginning of a string. ## Syntax @@ -8,18 +8,16 @@ The LEFT function returns a substring from the beginning of a specified string. ### Parameters -#### string +#### `string` -The string from which a substring will be extracted. +The string from which to return a substring. -#### length +#### `length` -The number of characters to extract from the string. +The number of characters to return. ## Examples -The following example extracts the first three characters from the `last_name` field: - ```esql FROM employees | KEEP last_name @@ -28,8 +26,4 @@ FROM employees | LIMIT 5 ``` -```esql -ROW full_name = "John Doe" -| EVAL first_name = LEFT(full_name, 4) -| KEEP first_name -``` +Extracts the first three characters from the `last_name` column, sorts the results alphabetically, and limits the output to the first five rows. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt index 996464cf42ba1..4c614fc42bd57 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-length.txt @@ -1,6 +1,6 @@ -# LENGTH +## LENGTH -The LENGTH function calculates the character length of a given string. +Returns the character length of a string. ## Syntax @@ -8,21 +8,17 @@ The LENGTH function calculates the character length of a given string. ### Parameters -#### string +#### `string` -The string expression for which the length is to be calculated. +String expression. If `null`, the function returns `null`. ## Examples -The following example calculates the character length of the `first_name` field: - ```esql -FROM employees -| KEEP first_name, last_name -| EVAL fn_length = LENGTH(first_name) +FROM airports +| WHERE country == "India" +| KEEP city +| EVAL fn_length = LENGTH(city) ``` -```esql -ROW message = "Hello, World!" -| EVAL message_length = LENGTH(message) -``` +This example calculates the character length of the `city` field for airports located in India. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt index 2778f82cbaff2..f8acaf8f4ec34 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt @@ -1,6 +1,6 @@ -# LIMIT +## LIMIT -The LIMIT command is used to restrict the number of rows returned by a query. +The `LIMIT` command restricts the number of rows returned by a query. ## Syntax @@ -8,81 +8,80 @@ The LIMIT command is used to restrict the number of rows returned by a query. ### Parameters -#### max_number_of_rows +#### `max_number_of_rows` -This parameter specifies the maximum number of rows to be returned. +The maximum number of rows to return. -## Examples +## Description -This example demonstrates how to limit the number of rows returned to 5. +The `LIMIT` command restricts the number of rows returned by a query. For example: ```esql -FROM employees -| SORT emp_no ASC -| LIMIT 5 +FROM index +| WHERE field == "value" ``` -This example shows how to limit the number of rows after applying a filter: +is equivalent to: ```esql -FROM employees -| WHERE department == "Engineering" -| LIMIT 10 +FROM index +| WHERE field == "value" +| LIMIT 1000 ``` -This example demonstrates limiting the number of rows after performing an aggregation: +Queries cannot return more than 10,000 rows, regardless of the value specified in the `LIMIT` command. This is a configurable upper limit. -```esql -FROM employees -| STATS avg_salary = AVG(salary) BY department -| LIMIT 3 -``` +### Overcoming the 10,000 Row Limit -This example shows how to limit the number of rows after sorting the data: +To address this limitation: +- Modify the query to reduce the result set size by using `WHERE` to filter the data. +- Perform post-query processing within the query itself. For example, use the `STATS` command to aggregate data. + +The 10,000-row limit applies only to the number of rows returned by the query, not to the number of documents processed. The query operates on the full dataset. + +Consider these examples: ```esql -FROM employees -| SORT hire_date DESC -| LIMIT 7 +FROM index +| WHERE field0 == "value" +| LIMIT 20000 ``` -This example demonstrates the use of `LIMIT` in conjunction with multiple other commands: +and ```esql -FROM employees -| WHERE hire_date > "2020-01-01" -| SORT salary DESC -| KEEP first_name, last_name, salary -| LIMIT 5 +FROM index +| STATS AVG(field1) BY field2 +| LIMIT 20000 ``` -`LIMIT` can and should be used as soon as possible in the query +In both cases, the filtering by `field0` in the first query or the grouping by `field2` in the second query is applied to all documents in the `index`. However, the output is capped at 10,000 rows, even if more rows are available. -For example this query uses SORT and LIMIT as soon as it can and before further computations: +### Configuring Limits + +The default and maximum limits can be adjusted using the following dynamic cluster settings: +- `esql.query.result_truncation_default_size` +- `esql.query.result_truncation_max_size` + +Increasing these limits may lead to higher memory usage, longer processing times, and increased internode traffic within and across clusters. + +## Examples + +Limit the result to the first 5 rows, sorted by `emp_no` in ascending order: ```esql -FROM sets -| EVAL count = MV_COUNT(values) -| SORT count DESC +FROM employees +| SORT emp_no ASC | LIMIT 5 -| EVAL min = MV_MIN(values), max = MV_MAX(values), avg = MV_AVG(value) -| KEEP set_id, min, max, avg ``` ## Limitations -There is no way to achieve pagination with LIMIT, there is no offset parameter. - -A query will never return more than 10,000 rows. This limitation only applies to the number of rows retrieved by the query. The query and any aggregations will still run on the full dataset. +- Queries cannot return more than 10,000 rows, even if the `LIMIT` value exceeds this threshold. -To work around this limitation: + To work around this limitation: -- Reduce the size of the result set by modifying the query to only return relevant data. This can be achieved by using the WHERE command to select a smaller subset of the data. -- Shift any post-query processing to the query itself. The ES|QL STATS ... BY command can be used to aggregate data within the query. + - Reduce the size of the result set by modifying the query to only return relevant data. This can be achieved by using the WHERE command to select a smaller subset of the data. + - Shift any post-query processing to the query itself. The ES|QL STATS ... BY command can be used to aggregate data within the query. -## Notes - -The default and maximum limits can be adjusted using the following dynamic cluster settings: - -- `esql.query.result_truncation_default_size` -- `esql.query.result_truncation_max_size` +- Adjusting the default or maximum limits can impact performance and resource usage. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt index 1dafd3fa8c998..7f4fb7d5267c4 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-locate.txt @@ -1,6 +1,6 @@ # LOCATE -The LOCATE function returns the position of a specified substring within a string. +Returns an integer indicating the position of a substring within another string. If the substring is not found, it returns `0`. Note that string positions start from `1`. ## Syntax @@ -8,29 +8,28 @@ The LOCATE function returns the position of a specified substring within a strin ### Parameters -#### string +#### `string` -The string in which you want to search for the substring. +An input string. -#### substring +#### `substring` -The substring you want to find in the string. +A substring to locate within the input string. -#### start +#### `start` -The starting index for the search. +The start index. This parameter is optional. ## Examples +Locate a substring within a string + ```esql ROW a = "hello" | EVAL a_ll = LOCATE(a, "ll") ``` -```esql -ROW phrase = "Elasticsearch is powerful" -| EVAL position = LOCATE(phrase, "powerful") -``` +This example finds the position of the substring `"ll"` within the string `"hello"`. The result is `3`. ## Notes diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt index 0c476551c02d4..72ed7be5105c6 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-log.txt @@ -1,6 +1,6 @@ # LOG -The LOG function calculates the logarithm of a given value to a specified base. +Calculates the logarithm of a numeric value to a specified base. If the base is not provided, it defaults to the natural logarithm (base e). ## Syntax @@ -8,22 +8,34 @@ The LOG function calculates the logarithm of a given value to a specified base. ### Parameters -#### base +#### `base` -The base of the logarithm. If the base is `null`, the function will return `null`. If the base is not provided, the function will return the natural logarithm (base e) of the value. +- Base of the logarithm. If `null`, the function returns `null`. If not provided, the function calculates the natural logarithm (base e). -#### number +#### `number` -The numeric value for which the logarithm is to be calculated. If the number is `null`, the function will return `null`. +- Numeric expression. If `null`, the function returns `null`. ## Examples +Logarithm with a specified base + +Calculate the logarithm of 8 to base 2: + ```esql ROW base = 2.0, value = 8.0 | EVAL s = LOG(base, value) ``` +Natural logarithm (base e) + +Calculate the natural logarithm of 100: + ```esql ROW value = 100 | EVAL s = LOG(value) ``` + +## Limitations + +- Logs of zero, negative numbers, and a base of one return `null` and generate a warning. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt index 29e266a197b32..9f129aa039177 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-ltrim.txt @@ -1,6 +1,6 @@ # LTRIM -The LTRIM function is used to remove leading whitespaces from a string. +Removes leading whitespaces from a string. ## Syntax @@ -8,9 +8,9 @@ The LTRIM function is used to remove leading whitespaces from a string. ### Parameters -#### string +#### `string` -This is the string expression from which you want to remove leading whitespaces. If the string is `null`, the function will return `null`. +String expression. If `null`, the function returns `null`. ## Examples @@ -22,8 +22,4 @@ ROW message = " some text ", color = " red " | EVAL color = CONCAT("'", color, "'") ``` -```esql -ROW text = " example text " -| EVAL trimmed_text = LTRIM(text) -| EVAL formatted_text = CONCAT("Trimmed: '", trimmed_text, "'") -``` +This example removes leading whitespaces from the `message` and `color` columns, then wraps the resulting strings in single quotes. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt index e75f91b018069..c49309ecd2b4a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt @@ -1,36 +1,47 @@ # MATCH -`MATCH` is a function used to execute a match query on a specified field. It works on various field types including text fields, boolean, dates, and numeric types. It returns 'true' when the provided query matches the row. +The `MATCH` function performs a match query on the specified field. It is equivalent to the `match` query in the Elasticsearch Query DSL and can be used to search for values in various field types, including text, semantic_text, keyword, boolean, dates, and numeric types. + +`MATCH` supports function named parameters to specify additional options for the match query. For a simplified syntax, the match operator `:` can be used instead of `MATCH`. The function returns `true` if the provided query matches the row. ## Syntax -`MATCH (field, query)` +`MATCH(field, query, options)` ### Parameters #### `field` -This represents the field that the query will target. If the field contains multiple values, -`MATCH` will process each value. +The field that the query will target. #### `query` -This is the value that is being searched in the provided field. +The value to find in the specified field. + +#### `options` + +(Optional) Additional match query options provided as function named parameters. Refer to the match query documentation for more details. ## Examples -In this example, `"Faulkner"` is matched against the `author` field in `books` data. `MATCH` returns true if it finds the provided query, in this case `"Faulkner"` in the author field. The query then keeps the columns `book_no` and `author`, sorts by `book_no` and limits the result to 5. +Match on a specific field ```esql FROM books | WHERE MATCH(author, "Faulkner") | KEEP book_no, author | SORT book_no -| LIMIT 5; +| LIMIT 5 ``` -## Notes +This example retrieves books where the `author` field matches "Faulkner," keeping only the `book_no` and `author` fields, sorting by `book_no`, and limiting the results to 5 rows. + +Match with additional options + +```esql +FROM books +| WHERE MATCH(title, "Hobbit Back Again", {"operator": "AND"}) +| KEEP title +``` -- Do not use `MATCH` in production - it is in technical preview and may be changed or removed in a future release -- `MATCH` relies on Elasticsearch Match query under the hood, and should be used for full-text search only. For more traditional - text matching, `LIKE` or `RLIKE` should be used instead. +This example searches for books where the `title` field matches "Hobbit Back Again" using the `AND` operator, and keeps only the `title` field in the results. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt index 8f30ac8ac94c8..3b6b1f2568e9b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-max.txt @@ -1,6 +1,6 @@ # MAX -The MAX function calculates the maximum value of a specified field. +The `MAX` function returns the maximum value of a field. ## Syntax @@ -8,22 +8,26 @@ The MAX function calculates the maximum value of a specified field. ### Parameters -#### field +#### `field` -The field for which the maximum value is to be calculated. +The field for which the maximum value is calculated. ## Examples -Calculate the maximum number of languages known by employees: +Basic Usage ```esql FROM employees | STATS MAX(languages) ``` -The MAX function can be used with inline functions: +Calculate the maximum value of the `languages` field. + +Using Inline Functions ```esql FROM employees | STATS max_avg_salary_change = MAX(MV_AVG(salary_change)) ``` + +Calculate the maximum value of the average salary change by first averaging the multiple values per row using the `MV_AVG` function and then applying the `MAX` function. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt index 0e7b1900bd003..eb21eb0ed2c6a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median.txt @@ -1,6 +1,6 @@ # MEDIAN -The MEDIAN function calculates the median value of a numeric field. The median is the value that is greater than half of all values and less than half of all values, also known as the 50% percentile. +The `MEDIAN` function calculates the value that is greater than half of all values and less than half of all values, also known as the 50th percentile. The result is usually approximate. ## Syntax @@ -8,20 +8,22 @@ The MEDIAN function calculates the median value of a numeric field. The median i ### Parameters -#### number +#### `number` -The numeric field for which the median is calculated. +The input numeric value for which the median is calculated. ## Examples -Calculate the median salary: +Calculating the median and 50th percentile of salaries ```esql FROM employees -| STATS MEDIAN(salary) +| STATS MEDIAN(salary), PERCENTILE(salary, 50) ``` -Calculate the median of the maximum values of a multivalued column: +Calculating the median of maximum values in a multivalued column + +To calculate the median of the maximum values of a multivalued column, first use `MV_MAX` to get the maximum value per row, and then use the result with the `MEDIAN` function: ```esql FROM employees @@ -30,4 +32,5 @@ FROM employees ## Limitations -- The MEDIAN function is usually approximate and non-deterministic. This means you can get slightly different results using the same data. +- The `MEDIAN` function is non-deterministic, meaning you may get slightly different results when using the same data. +- Like the `PERCENTILE` function, the `MEDIAN` function provides approximate results. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt index 6bd3de7db5cf6..4a704137facb8 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-median_absolute_deviation.txt @@ -1,6 +1,8 @@ # MEDIAN_ABSOLUTE_DEVIATION -The MEDIAN_ABSOLUTE_DEVIATION function calculates the median absolute deviation, a measure of variability. It is particularly useful for describing data that may have outliers or may not follow a normal distribution. In such cases, it can be more descriptive than standard deviation. The function computes the median of each data point’s deviation from the median of the entire sample. +Returns the median absolute deviation, a robust measure of variability. It is particularly useful for describing data with outliers or non-normal distributions, as it can be more descriptive than standard deviation. The median absolute deviation is calculated as the median of the absolute deviations from the median of the entire sample. For a random variable `X`, it is defined as `median(|median(X) - X|)`. + +**Note:** This function is usually approximate, similar to `PERCENTILE`. ## Syntax @@ -8,27 +10,30 @@ The MEDIAN_ABSOLUTE_DEVIATION function calculates the median absolute deviation, ### Parameters -#### number +#### `number` -The numeric expression for which the median absolute deviation is to be calculated. +The input numeric field or expression. ## Examples -Calculate the median salary and the median absolute deviation of salaries: +Basic Usage ```esql FROM employees | STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary) ``` -Calculate the median absolute deviation of the maximum values of a multivalued column: +Calculate the median and the median absolute deviation of employee salaries. + +Using Inline Functions ```esql FROM employees | STATS m_a_d_max_salary_change = MEDIAN_ABSOLUTE_DEVIATION(MV_MAX(salary_change)) ``` +Calculate the median absolute deviation of the maximum values of a multivalued column by first using `MV_MAX` to get the maximum value per row. + ## Limitations -- The `MEDIAN_ABSOLUTE_DEVIATION` function is non-deterministic, which means you can get slightly different results using the same data. -- The `MEDIAN_ABSOLUTE_DEVIATION` function is usually approximate, which means the results may not be exact. +- `MEDIAN_ABSOLUTE_DEVIATION` is non-deterministic, meaning that slightly different results may be returned when using the same data. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt index 6b4848c7fc9a7..87f04fef44042 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-min.txt @@ -1,6 +1,6 @@ # MIN -The MIN function calculates the minimum value of a specified field. +The `MIN` function calculates the minimum value of a field. ## Syntax @@ -8,22 +8,26 @@ The MIN function calculates the minimum value of a specified field. ### Parameters -#### field +#### `field` -The field for which the minimum value is to be calculated. +The field for which the minimum value is calculated. ## Examples -Calculate the minimum number of languages spoken by employees: +Basic Usage ```esql FROM employees | STATS MIN(languages) ``` -The MIN function can be used with inline functions: +Calculate the minimum value of the `languages` field. + +Using Inline Functions ```esql FROM employees | STATS min_avg_salary_change = MIN(MV_AVG(salary_change)) ``` + +Calculate the minimum value of the average salary change by first averaging the multiple values per row using the `MV_AVG` function and then applying the `MIN` function. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt index 46196cf32931b..2eb0b47ba25b1 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_append.txt @@ -1,6 +1,6 @@ # MV_APPEND -MV_APPEND is a function that concatenates the values of two multi-value fields. +Concatenates the values of two multi-value fields into a single field. ## Syntax @@ -8,9 +8,15 @@ MV_APPEND is a function that concatenates the values of two multi-value fields. ### Parameters -#### field1 +#### `field1` -The first multi-value field to be concatenated. +The first multi-value field to concatenate. + +#### `field2` + +The second multi-value field to concatenate. + +## Examples ```esql ROW a = ["foo", "bar"], b = ["baz", "qux"] @@ -23,3 +29,7 @@ ROW x = [1, 2, 3], y = [4, 5, 6] | EVAL z = MV_APPEND(x, y) | KEEP x, y, z ``` + +## Limitations + +No specific limitations are mentioned in the source documentation. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt index 8b071e8629cf3..0cd9a66568de7 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_avg.txt @@ -1,6 +1,6 @@ # MV_AVG -The MV_AVG function calculates the average of all values in a multivalued field and returns a single value. +Converts a multivalued field into a single-valued field containing the average of all its values. ## Syntax @@ -8,7 +8,7 @@ The MV_AVG function calculates the average of all values in a multivalued field ### Parameters -#### number +#### `number` A multivalued expression. @@ -19,8 +19,11 @@ ROW a=[3, 5, 1, 6] | EVAL avg_a = MV_AVG(a) ``` -**Retrieving the average value from a multivalued field** +Calculate the average of the values in the multivalued column `a`. + ```esql FROM bag_of_numbers | EVAL min = MV_AVG(numbers) ``` + +Retrieve the average value from a multivalued field diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt index 7a4d9fff9466a..91f08db58273d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_concat.txt @@ -1,6 +1,6 @@ # MV_CONCAT -MV_CONCAT is a function that transforms a multivalued string expression into a single valued column. It concatenates all values and separates them with a specified delimiter. +Converts a multivalued string expression into a single-valued column by concatenating all values, separated by a specified delimiter. ## Syntax @@ -8,26 +8,30 @@ MV_CONCAT is a function that transforms a multivalued string expression into a s ### Parameters -#### string +#### `string` -A multivalue expression. +A multivalued expression. -#### delim +#### `delim` -This is the delimiter that separates the concatenated values. +The delimiter used to separate the concatenated values. ## Examples -The following example concatenates the values in the array ["foo", "zoo", "bar"] with a comma and a space as the delimiter: +Concatenating string values ```esql ROW a=["foo", "zoo", "bar"] | EVAL j = MV_CONCAT(a, ", ") ``` -If you want to concatenate non-string columns, you need to convert them to strings first using the `TO_STRING` function: +Concatenates the values in the array ["foo", "zoo", "bar"] with a comma and a space as the delimiter: + +Concatenating non-string values ```esql ROW a=[10, 9, 8] | EVAL j = MV_CONCAT(TO_STRING(a), ", ") ``` + +Converts the numeric values in the multivalued column `a` to strings using `TO_STRING`, then concatenates them into a single string, separated by `", "`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt index b468be2ba6af1..785bc819a4084 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_count.txt @@ -1,6 +1,6 @@ # MV_COUNT -The MV_COUNT function calculates the total number of values in a multivalued expression. +Converts a multivalued expression into a single-valued column containing the count of the number of values. ## Syntax @@ -8,7 +8,7 @@ The MV_COUNT function calculates the total number of values in a multivalued exp ### Parameters -#### field +#### `field` A multivalued expression. @@ -19,8 +19,10 @@ ROW a=["foo", "zoo", "bar"] | EVAL count_a = MV_COUNT(a) ``` -**Counting the number of element in a multivalued field** +Count the number of values in the multivalued column `a`. + ```esql FROM bag_of_numbers | EVAL count = MV_COUNT(numbers) ``` +Count the number of element in a multivalued field `numbers` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt index 644ddd6d5f405..ae0bd738c85ef 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_dedupe.txt @@ -1,6 +1,8 @@ # MV_DEDUPE -The MV_DEDUPE function is used to eliminate duplicate values from a multivalued field. +Removes duplicate values from a multivalued field. + +**Note:** `MV_DEDUPE` may, but won’t always, sort the values in the column. ## Syntax @@ -8,9 +10,9 @@ The MV_DEDUPE function is used to eliminate duplicate values from a multivalued ### Parameters -#### field +#### `field` -This is a multivalue expression. +A multivalue expression. ## Examples @@ -19,11 +21,4 @@ ROW a=["foo", "foo", "bar", "foo"] | EVAL dedupe_a = MV_DEDUPE(a) ``` -```esql -ROW b=["apple", "apple", "banana", "apple", "banana"] -| EVAL dedupe_b = MV_DEDUPE(b) -``` - -## Notes - -While MV_DEDUPE may sort the values in the column, it's not guaranteed to always do so. +This example removes duplicate values from the multivalued column `a`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt index 3248391d3d658..db48c902ee699 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_expand.txt @@ -1,6 +1,8 @@ # MV_EXPAND -The MV_EXPAND command is used to expand multivalued columns into individual rows, replicating the other columns for each new row. +The `MV_EXPAND` command expands multivalued columns into one row per value, duplicating other columns. + +> **Note:** This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. ## Syntax @@ -10,33 +12,31 @@ The MV_EXPAND command is used to expand multivalued columns into individual rows #### column -This is the multivalued column that you want to expand. +The multivalued column to expand. -## Examples +## Notes -Expanding a multivalued column `a` into individual rows: +The output rows produced by `MV_EXPAND` can be in any order and may not respect preceding `SORT` commands. To ensure a specific ordering, place a `SORT` command after any `MV_EXPAND` commands. + + +## Examples ```esql ROW a=[1,2,3], b="b", j=["a","b"] | MV_EXPAND a ``` - -Expanding two multivalued columns `a` and `j` into individual rows: +Expand a multivalued column `a` into individual rows: ```esql ROW a=[1,2,3], b="b", j=["a","b"] | MV_EXPAND a | MV_EXPAND j ``` - -Expanding a multivalued column and then filtering the results: +Expand two multivalued columns `a` and `j` into individual rows: ```esql ROW a=[1,2,3,4,5], b="b" | MV_EXPAND a | WHERE a > 2 ``` - -## Notes - -This feature is currently in technical preview and may be subject to changes or removal in future releases. +Expand a multivalued column and then filtering the results: diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt index 3cbba2efc7425..df7a1a5b0769c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_first.txt @@ -1,6 +1,6 @@ # MV_FIRST -The MV_FIRST function converts a multivalued expression into a single valued column containing the first value. +Converts a multivalued expression into a single-valued column containing the first value. This is particularly useful when working with functions like `SPLIT` that produce multivalued columns in a known order. ## Syntax @@ -8,23 +8,22 @@ The MV_FIRST function converts a multivalued expression into a single valued col ### Parameters -#### field +#### `field` -A multivalue expression. +A multivalued expression. ## Examples +Extracting the first value from a multivalued column + ```esql ROW a="foo;bar;baz" | EVAL first_a = MV_FIRST(SPLIT(a, ";")) ``` -**Retrieving the first element from a multivalued field** -```esql -FROM bag_of_numbers -| EVAL first = MV_FIRST(numbers) -``` +This example splits the string `a` into multiple values using the `SPLIT` function and extracts the first value, resulting in `first_a = "foo"`. ## Notes -The MV_FIRST function is particularly useful when reading from a function that emits multivalued columns in a known order, such as SPLIT. However, it's important to note that the order in which multivalued fields are read from underlying storage is not guaranteed. While it's often ascending, this should not be relied upon. If you need the minimum value, use the MV_MIN function instead of MV_FIRST. MV_MIN has optimizations for sorted values, so there isn't a performance benefit to MV_FIRST. +- The order in which multivalued fields are read from underlying storage is not guaranteed. While it is often ascending, this behavior should not be relied upon. +- If you need the minimum value, use `MV_MIN` instead of `MV_FIRST`. The `MV_MIN` function is optimized for sorted values and offers no performance disadvantage compared to `MV_FIRST`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt index c85995575c486..f01eb8ff2a254 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_last.txt @@ -1,6 +1,8 @@ # MV_LAST -The MV_LAST function converts a multivalued expression into a single valued column containing the last value. +Converts a multivalue expression into a single-valued column containing the last value. This is particularly useful when working with functions that produce multivalued columns in a known order, such as `SPLIT`. + +The order in which multivalued fields are read from underlying storage is not guaranteed. While it is often ascending, this behavior should not be relied upon. If you need the maximum value, use `MV_MAX` instead of `MV_LAST`. `MV_MAX` is optimized for sorted values and does not offer a performance advantage over `MV_LAST`. ## Syntax @@ -8,25 +10,17 @@ The MV_LAST function converts a multivalued expression into a single valued colu ### Parameters -#### field +#### `field` A multivalue expression. - - ## Examples +Extracting the last value from a multivalued column + ```esql ROW a="foo;bar;baz" | EVAL last_a = MV_LAST(SPLIT(a, ";")) ``` -**Retrieving the last element from a multivalued field** -```esql -FROM bag_of_numbers -| EVAL last = MV_LAST(numbers) -``` - -## Notes - -The MV_LAST function is particularly useful when reading from a function that emits multivalued columns in a known order, such as SPLIT. However, the order in which multivalued fields are read from underlying storage is not guaranteed. It is often ascending, but this should not be relied upon. If you need the maximum value, use the MV_MAX function instead of MV_LAST. MV_MAX has optimizations for sorted values, so there is no performance benefit to using MV_LAST. +This example splits the string `a` into multiple values using the `SPLIT` function and then extracts the last value, resulting in `last_a = "baz"`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt index b38c1bd0252ed..2e6389f779721 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_max.txt @@ -1,6 +1,6 @@ # MV_MAX -MV_MAX function converts a multivalued expression into a single valued column containing the maximum value. +Converts a multivalued expression into a single-valued column containing the maximum value. ## Syntax @@ -8,21 +8,26 @@ MV_MAX function converts a multivalued expression into a single valued column co ### Parameters -#### field +#### `field` -A multivalue expression. +Multivalue expression. ## Examples -The following example demonstrates the use of MV_MAX function: - ```esql ROW a=[3, 5, 1] | EVAL max_a = MV_MAX(a) ``` -**Retrieving the max value from a multivalued field** +Finds the maximum value in the multivalued column `a`, resulting in `max_a = 5`. + ```esql FROM bag_of_numbers | EVAL max = MV_MAX(numbers) ``` + +Finds the maximum value in the column `a` by comparing the strings' UTF-8 representations, resulting in `max_a = "zoo"`. + +## Supported Types + +This function can be used with any column type, including `keyword` columns. For `keyword` columns, it picks the last string by comparing their UTF-8 representation byte by byte. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt index 013cce53deded..cc445b2afd492 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median.txt @@ -1,6 +1,6 @@ # MV_MEDIAN -The MV_MEDIAN function converts a multivalued field into a single valued field containing the median value. +Converts a multivalued field into a single-valued field containing the median value. ## Syntax @@ -8,9 +8,9 @@ The MV_MEDIAN function converts a multivalued field into a single valued field c ### Parameters -#### number +#### `number` -A multivalue expression. +Multivalue expression. ## Examples @@ -19,9 +19,11 @@ ROW a=[3, 5, 1] | EVAL median_a = MV_MEDIAN(a) ``` -If the row has an even number of values for a column, the result will be the average of the middle two entries. If the column is not floating point, the average rounds **down**: +Calculate the median value of the multivalued column `a`. ```esql ROW a=[3, 7, 1, 6] | EVAL median_a = MV_MEDIAN(a) ``` + +For rows with an even number of values, the result is the average of the middle two entries. If the column is not of a floating-point type, the average rounds **down**. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median_absolute_deviation.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median_absolute_deviation.txt index 6f47135f5b097..27f3da7e8afde 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median_absolute_deviation.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_median_absolute_deviation.txt @@ -1,6 +1,6 @@ # MV_MEDIAN_ABSOLUTE_DEVIATION -The MV_MEDIAN_ABSOLUTE_DEVIATION function transforms a multi-valued field into a single-valued field that retains the median absolute deviation. It computes this as a median of the deviation of each datum from the entire sample's median. In other words, for a random variable `X`, the median absolute deviation can be represented as `median(|median(X) - X|)`. +Converts a multivalued field into a single-valued field containing the median absolute deviation. The median absolute deviation is calculated as the median of each data point’s deviation from the median of the entire sample. For a random variable `X`, it is defined as `median(|median(X) - X|)`. ## Syntax @@ -8,17 +8,22 @@ The MV_MEDIAN_ABSOLUTE_DEVIATION function transforms a multi-valued field into a ### Parameters -#### number +#### `number` -A multi-valued expression. - -*Notice*: If the field comprises an even amount of values, the median is deduced as an average of the two central values. If the value isn't a floating-point number, the average values are rounded towards 0. +A multivalue expression. ## Examples +Calculating the median absolute deviation and median + ```esql ROW values = [0, 2, 5, 6] | EVAL median_absolute_deviation = MV_MEDIAN_ABSOLUTE_DEVIATION(values), median = MV_MEDIAN(values) ``` -This example illustrates the computation of the median absolute deviation and the median from a list of numerical values. +This example calculates the median absolute deviation and the median for the multivalued field `values`. + +## Notes + +- If the field contains an even number of values, the medians are calculated as the average of the middle two values. +- If the values are not floating-point numbers, the averages are rounded towards 0. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt index 514f25420331b..0b682e55add0f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_min.txt @@ -1,6 +1,6 @@ # MV_MIN -The MV_MIN function converts a multivalued expression into a single valued column containing the minimum value. +Converts a multivalued expression into a single-valued column containing the minimum value. ## Syntax @@ -8,19 +8,26 @@ The MV_MIN function converts a multivalued expression into a single valued colum ### Parameters -#### field +#### `field` -This is a multivalue expression. +A multivalued expression. + +## Supported Types + +This function can be used with any column type, including `keyword` columns. For `keyword` columns, it selects the first string by comparing their UTF-8 representation byte by byte. ## Examples -```esql +#```esql ROW a=[2, 1] | EVAL min_a = MV_MIN(a) ``` -**Retrieving the min value from a multivalued field** -```esql +Extracts the minimum value from the multivalued column `a`, resulting in `min_a = 1`. + +#```esql FROM bag_of_numbers | EVAL min = MV_MIN(numbers) ``` + +Extracts the minimum value from the multivalued column `numbers` by comparing the values lexicographically. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_percentile.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_percentile.txt index 1d6d0d802f6ec..ab9e422f24cf7 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_percentile.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_percentile.txt @@ -1,6 +1,6 @@ # MV_PERCENTILE -This function converts a multivalued field into a single-valued field. The single-valued field it produces contains the value at which a specified percentage of observed values occur. +Converts a multivalued field into a single-valued field containing the value at which a certain percentage of observed values occur. ## Syntax @@ -8,19 +8,19 @@ This function converts a multivalued field into a single-valued field. The singl ### Parameters -#### number +#### `number` -This refers to a multivalue expression. +A multivalue expression. -#### percentile +#### `percentile` -Value for the percentile to calculate. The value should range from 0 and 100. Values outside this range return null. +The percentile to calculate. Must be a number between 0 and 100. Numbers outside this range will return `null`. ## Examples -Consider an instance where you want to calculate the 50th percentile (or median) of a set of numbers - `[5, 5, 10, 12, 5000]`. This can be done using the following statement. - ```esql ROW values = [5, 5, 10, 12, 5000] | EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values) -``` \ No newline at end of file +``` + +This example calculates the 50th percentile (median) of the multivalued field `values` and compares it to the result of the `MV_MEDIAN` function. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt index 85845e5fa8de1..e5d07132de9c8 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_pseries_weighted_sum.txt @@ -1,6 +1,6 @@ # MV_PSERIES_WEIGHTED_SUM -The MV_PSERIES_WEIGHTED_SUM function transforms a multivalued expression into a single-valued column. It does this by multiplying each element in the input list by its corresponding term in a P-Series and then calculating the sum. +Converts a multivalued expression into a single-valued column by multiplying each element in the input list by its corresponding term in the P-Series and computing the sum. ## Syntax @@ -8,24 +8,22 @@ The MV_PSERIES_WEIGHTED_SUM function transforms a multivalued expression into a ### Parameters -#### number +#### `number` -This is a multivalue expression. +A multivalue expression. -#### p +#### `p` -A number that represents the *p* parameter in the P-Series. It influences the contribution of each element to the weighted sum. +A constant number representing the *p* parameter in the P-Series. It determines the impact of each element’s contribution to the weighted sum. ## Examples +Calculating the weighted sum of a multivalued column + ```esql ROW a = [70.0, 45.0, 21.0, 21.0, 21.0] | EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5) | KEEP sum ``` -```esql -ROW b = [10.0, 20.0, 30.0, 40.0, 50.0] -| EVAL weighted_sum = MV_PSERIES_WEIGHTED_SUM(b, 2.0) -| KEEP weighted_sum -``` +This example calculates the weighted sum of the multivalued column `a` using a P-Series parameter of `1.5`. The result is stored in the `sum` column. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt index fccf790cfb79e..ec7a7c3cd8de0 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_slice.txt @@ -1,6 +1,6 @@ # MV_SLICE -The MV_SLICE function is used to extract a subset of a multivalued field using specified start and end index values. +The `MV_SLICE` function extracts a subset of a multivalued field based on specified start and end index values. It is particularly useful when working with functions that produce multivalued columns in a known order, such as `SPLIT` or `MV_SORT`. ## Syntax @@ -8,26 +8,44 @@ The MV_SLICE function is used to extract a subset of a multivalued field using s ### Parameters -#### field +#### `field` -This is a multivalue expression. If `null`, the function will return `null`. +- A multivalue expression. If `null`, the function returns `null`. -#### start +#### `start` -This is the start position. If `null`, the function will return `null`. The start argument can be negative, where an index of -1 is used to specify the last value in the list. +- The starting position of the slice. If `null`, the function returns `null`. +- Can be negative, where `-1` refers to the last value in the list. -#### end +#### `end` (Optional) -This is the end position (included). This parameter is optional; if omitted, the position at `start` is returned. The end argument can be negative, where an index of -1 is used to specify the last value in the list. +- The ending position of the slice (inclusive). If omitted, only the value at the `start` position is returned. +- Can be negative, where `-1` refers to the last value in the list. ## Examples +Extracting specific slices from a multivalued field + ```esql ROW a = [1, 2, 2, 3] | EVAL a1 = MV_SLICE(a, 1), a2 = MV_SLICE(a, 2, 3) ``` +This example extracts: +- `a1` as the value starting at index `1` (second value in the list). +- `a2` as the values from index `2` to `3` (third and fourth values in the list). + +Using negative indices to slice from the end of the list + ```esql ROW a = [1, 2, 2, 3] | EVAL a1 = MV_SLICE(a, -2), a2 = MV_SLICE(a, -3, -1) ``` + +This example extracts: +- `a1` as the value starting at the second-to-last index (`-2`). +- `a2` as the values from the third-to-last index (`-3`) to the last index (`-1`). + +## Notes + +- The order in which multivalued fields are read from underlying storage is not guaranteed. While it is often ascending, this behavior should not be relied upon. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt index a7ddc9452c1ba..c3e6b7146268c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sort.txt @@ -1,6 +1,6 @@ # MV_SORT -The MV_SORT function sorts a multivalued field in lexicographical order. +Sorts a multivalued field in lexicographical order. ## Syntax @@ -8,34 +8,25 @@ The MV_SORT function sorts a multivalued field in lexicographical order. ### Parameters -#### field +#### `field` -This is a multivalue expression. If the value is `null`, the function will return `null`. +- Multivalue expression. If `null`, the function returns `null`. -#### order +#### `order` -This parameter determines the sort order. The valid options are `ASC` and `DESC`. If not specified, the default is `ASC`. +- Sort order. The valid options are `ASC` and `DESC`. The default is `ASC`. ## Examples -Without order parameter - ```esql -ROW names = ["Alice", "Bob", "Charlie"] -| EVAL sorted_names = MV_SORT(names) +ROW a = [4, 2, -3, 2] +| EVAL sa = mv_sort(a), sd = mv_sort(a, "DESC") ``` -With order parameter +This example sorts the multivalued field `a` in ascending order (`sa`) and descending order (`sd`). -```esql -ROW a = [4, 2, -3, 2] -| EVAL sd = MV_SORT(a, "DESC") -``` -**Sorting a multivalued field** ```esql FROM bag_of_numbers | EVAL sorted = MV_SORT(numbers) ``` - - diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt index 7e09a7ceaff06..de237cc2e918b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_sum.txt @@ -1,6 +1,6 @@ # MV_SUM -The MV_SUM function converts a multivalued field into a single valued field containing the sum of all the values. +Converts a multivalued field into a single-valued field containing the sum of all its values. ## Syntax @@ -8,9 +8,9 @@ The MV_SUM function converts a multivalued field into a single valued field cont ### Parameters -#### number +#### `number` -This is a multivalue expression. +A multivalued expression. ## Examples @@ -19,7 +19,4 @@ ROW a=[3, 5, 6] | EVAL sum_a = MV_SUM(a) ``` -```esql -ROW numbers=[1, 2, 3, 4, 5] -| EVAL total_sum = MV_SUM(numbers) -``` +This example calculates the sum of the values in the multivalued column `a`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt index c6349624e05af..c1a982b089c81 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-mv_zip.txt @@ -1,6 +1,6 @@ # MV_ZIP -The MV_ZIP function combines the values from two multivalued fields with a specified delimiter. +Combines the values from two multivalued fields with a delimiter that joins them together. ## Syntax @@ -8,30 +8,32 @@ The MV_ZIP function combines the values from two multivalued fields with a speci ### Parameters -#### string1 +#### `string1` -A multivalue expression. +Multivalue expression. -#### string2 +#### `string2` -A multivalue expression. +Multivalue expression. -#### delim +#### `delim` -An optional parameter that specifies the delimiter used to join the values. If omitted, a comma (`,`) is used as the default delimiter. +Optional. The delimiter used to join the values. If omitted, `,` is used as the default delimiter. ## Examples -The following example demonstrates how to use the MV_ZIP function: +Combining two multivalued fields with a custom delimiter ```esql ROW a = ["x", "y", "z"], b = ["1", "2"] -| EVAL c = MV_ZIP(a, b, "-") +| EVAL c = mv_zip(a, b, "-") | KEEP a, b, c ``` -```esql -ROW names = ["Alice", "Bob", "Charlie"], ids = ["001", "002", "003"] -| EVAL combined = MV_ZIP(names, ids, ":") -| KEEP names, ids, combined -``` +This example combines the values from two multivalued fields `a` and `b` using the `-` delimiter. + +#### Result + +| a | b | c | +|------------------|-------------|----------------| +| ["x", "y", "z"] | ["1", "2"] | ["x-1", "y-2", "z"] | diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt index 15fd1e506a1d4..cba069d1fd162 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-now.txt @@ -1,6 +1,6 @@ -# NOW +## NOW -The NOW function returns the current date and time. +Returns the current date and time. ## Syntax @@ -8,14 +8,18 @@ The NOW function returns the current date and time. ### Parameters -This function does not require any parameters. +This function does not take any parameters. ## Examples +#Retrieve the current date and time + ```esql ROW current_date = NOW() ``` +#Retrieve logs from the last hour + ```esql FROM sample_data | WHERE @timestamp > NOW() - 1 hour diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt index a6ebcfdbd6bdb..e8311dc56310f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-operators.txt @@ -1,240 +1,204 @@ +```markdown # ES|QL Operators -This document provides an overview of the operators supported by ES|QL. +This document provides an overview of the operators available in ES|QL, categorized into binary, unary, logical, and other operators. Each operator is accompanied by an example query to demonstrate its usage. -## Binary Operators +--- -### Equality `==` +## Binary Operators -The equality operator checks if the values of two operands are equal or not. +Binary operators are used to compare or perform arithmetic operations between two values. -Example: +### Equality (`==`) +Checks if two values are equal. ```esql FROM employees -| WHERE emp_no == 10001 +| WHERE first_name == "John" ``` -### Inequality `!=` - -The inequality operator checks if the values of two operands are equal or not. - -Example: +### Inequality (`!=`) +Checks if two values are not equal. ```esql FROM employees -| WHERE emp_no != 10001 +| WHERE department != "HR" ``` -### Less Than `<` - -The less than operator checks if the value of the left operand is less than the value of the right operand. - -Example: +### Less Than (`<`) +Checks if a value is less than another. ```esql FROM employees | WHERE salary < 50000 ``` -### Less Than or Equal To `<=` - -This operator checks if the value of the left operand is less than or equal to the value of the right operand. - -Example: +### Less Than or Equal To (`<=`) +Checks if a value is less than or equal to another. ```esql FROM employees -| WHERE salary <= 50000 +| WHERE hire_date <= "2020-01-01" ``` -### Greater Than `>` - -The greater than operator checks if the value of the left operand is greater than the value of the right operand. - -Example: +### Greater Than (`>`) +Checks if a value is greater than another. ```esql FROM employees -| WHERE salary > 50000 +| WHERE age > 30 ``` -### Greater Than or Equal To `>=` - -This operator checks if the value of the left operand is greater than or equal to the value of the right operand. - -Example: +### Greater Than or Equal To (`>=`) +Checks if a value is greater than or equal to another. ```esql FROM employees -| WHERE salary >= 50000 +| WHERE experience_years >= 5 ``` -### Add `+` - -The add operator adds the values of the operands. - -Example: +### Add (`+`) +Adds two values. ```esql FROM employees | EVAL total_compensation = salary + bonus ``` -### Subtract `-` - -The subtract operator subtracts the right-hand operand from the left-hand operand. - -Example: +### Subtract (`-`) +Subtracts one value from another. ```esql FROM employees -| EVAL remaining_salary = salary - tax +| EVAL remaining_vacation_days = total_vacation_days - used_vacation_days ``` -### Multiply `*` - -The multiply operator multiplies the values of the operands. - -Example: +### Multiply (`*`) +Multiplies two values. ```esql FROM employees -| EVAL yearly_salary = salary * 12 +| EVAL annual_salary = monthly_salary * 12 ``` -### Divide `/` - -The divide operator divides the left-hand operand by the right-hand operand. - -Example: +### Divide (`/`) +Divides one value by another. ```esql FROM employees -| EVAL monthly_salary = salary / 12 +| EVAL average_salary = total_salary / employee_count ``` -### Modulus `%` - -The modulus operator returns the remainder of the division of the left operand by the right operand. - -Example: +### Modulus (`%`) +Returns the remainder of a division. ```esql FROM employees -| EVAL remainder = salary % 12 +| EVAL remainder = employee_id % 2 ``` +--- + ## Unary Operators -### Negation (`-`) +Unary operators operate on a single operand. -Example: +### Negation (`-`) +Negates a numeric value. ```esql -FROM employees -| EVAL negative_salary = -salary +ROW value = 10 +| EVAL negative_value = -value ``` -## Logical Operators +--- -### AND +## Logical Operators -Logical AND operator. +Logical operators are used to combine or negate conditions. -Example: +### AND +Returns `true` if both conditions are true. ```esql FROM employees -| WHERE salary > 50000 AND bonus > 10000 +| WHERE age > 30 AND department == "Engineering" ``` ### OR - -Logical OR operator. - -Example: +Returns `true` if at least one condition is true. ```esql FROM employees -| WHERE salary > 50000 OR bonus > 10000 +| WHERE department == "HR" OR department == "Finance" ``` ### NOT - -Logical NOT operator. - -Example: +Negates a condition. ```esql FROM employees -| WHERE NOT (salary > 50000) +| WHERE NOT still_hired ``` +--- + ## Other Operators ### IS NULL and IS NOT NULL +Checks if a value is `NULL` or not. -The `IS NULL` operator returns true if the value is null. - -Example: - +#### IS NULL ```esql FROM employees -| WHERE manager IS NULL +| WHERE birth_date IS NULL +| KEEP first_name, last_name ``` -The `IS NOT NULL` operator returns true if the value is not null. - -Example: - +#### IS NOT NULL ```esql FROM employees -| WHERE manager IS NOT NULL +| WHERE is_rehired IS NOT NULL +| STATS COUNT(emp_no) ``` -### IN - -The `IN` operator checks if a value is within a set of values (literals, fields or expressions). - -Example: +### Cast (`::`) +Casts a value to a specific type. ```esql FROM employees -| WHERE department IN ("Sales", "Marketing", "HR") +| EVAL salary_as_string = salary::KEYWORD ``` +### IN +Checks if a value is in a list of values. + ```esql ROW a = 1, b = 4, c = 3 -| WHERE c-a IN (3, b / 2, a) +| WHERE c - a IN (3, b / 2, a) ``` ### LIKE +Filters data based on string patterns using wildcards. -Use `LIKE` to filter data based on string patterns using wildcards. - -The following wildcard characters are supported: -- `*` matches zero or more characters. -- `?` matches one character. - -Example: - +#Basic usage ```esql FROM employees -| WHERE first_name LIKE "?b*" -| KEEP first_name, last_name +| WHERE first_name LIKE "J*" ``` -### RLIKE - -Use `RLIKE` to filter data based on string patterns using regular expressions. +#Escaping special characters +```esql +ROW message = "foo * bar" +| WHERE message LIKE "foo \\* bar" +``` -Example: +### RLIKE +Filters data based on string patterns using regular expressions. ```esql FROM employees -| WHERE first_name RLIKE ".leja.*" -| KEEP first_name, last_name +| WHERE first_name RLIKE "J.*" ``` ### Cast `::` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt index 5a2a6252728ca..c06d1efcd3094 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-overview.txt @@ -1,103 +1,105 @@ -## ES|QL Overview +```markdown +# Elasticsearch Query Language (ES|QL) -### ES|QL +The Elasticsearch Query Language (ES|QL) is a powerful and intuitive language designed to filter, transform, and analyze data stored in Elasticsearch. It is built to be user-friendly and accessible to a wide range of users, including end users, SRE teams, application developers, and administrators. ES|QL enables users to perform complex data operations such as filtering, aggregation, and time-series analysis, as well as generate visualizations and statistical insights. -The Elasticsearch Query Language (ES|QL) provides a powerful way to filter, transform, and analyze data stored in Elasticsearch. It is designed to be easy to learn and use by all types of end users. +## Key Features of ES|QL -Users can author ES|QL queries to find specific events, perform statistical analysis, and generate visualizations. It supports a wide range of commands and functions that enable users to perform various data operations, such as filtering, aggregation, time-series analysis, and more. +- **Pipe-based Syntax**: ES|QL uses a pipe (`|`) syntax to chain operations, where the output of one operation becomes the input for the next. This step-by-step approach simplifies complex data transformations and analysis. +- **Rich Command Set**: ES|QL supports a wide range of commands and functions for data manipulation, including filtering, aggregation, enrichment, and statistical analysis. +- **Ease of Use**: Designed to be easy to learn and use, ES|QL is suitable for both technical and non-technical users. +- **Integration with Elasticsearch**: ES|QL queries are executed directly within Elasticsearch, leveraging its compute engine for high performance and scalability. -ES|QL makes use of "pipes" (`|`) to manipulate and transform data in a step-by-step fashion. This approach allows users to compose a series of operations, where the output of one operation becomes the input for the next, enabling complex data transformations and analysis. +--- -### Known Limitations +## Known Limitations of ES|QL -#### Result Set Size Limit +While ES|QL is a powerful tool, it has some limitations to be aware of: -By default, an ES|QL query returns up to 1000 rows. You can increase the number of rows up to 10,000 using the `LIMIT` command. Queries do not return more than 10,000 rows, regardless of the `LIMIT` command’s value. This limit only applies to the number of rows that are retrieved by the query. Queries and aggregations run on the full data set. +### Result Set Size +- By default, ES|QL queries return up to 1,000 rows. This can be increased to a maximum of 10,000 rows using the `LIMIT` command. This upper limit is configurable but comes with trade-offs such as increased memory usage and processing time. -To overcome this limitation: -- Reduce the result set size by modifying the query to only return relevant data. Use `WHERE` to select a smaller subset of the data. -- Shift any post-query processing to the query itself. You can use the ES|QL `STATS ... BY` command to aggregate data in the query. +### Field Types +- ES|QL supports a wide range of field types, including `boolean`, `date`, `keyword`, `text`, `long`, and `double`. However, some field types, such as `binary`, `nested`, and `histogram`, are not yet supported. +- When querying multiple indices, fields with conflicting types must be explicitly converted to a single type using type conversion functions. -The default and maximum limits can be changed using these dynamic cluster settings: -- `esql.query.result_truncation_default_size` -- `esql.query.result_truncation_max_size` +### Full-Text Search +- Full-text search is in technical preview and has limitations. For example, full-text search functions like `MATCH` must be used directly after the `FROM` command or close to it. Additionally, disjunctions (`OR`) in `WHERE` clauses are restricted unless all clauses use full-text functions. -#### Field Types +### Time Series Data Streams +- ES|QL does not currently support querying time series data streams (TSDS). -ES|QL currently supports the following field types: -- `alias` -- `boolean` -- `date` -- `double` (`float`, `half_float`, `scaled_float` are represented as `double`) -- `ip` -- `keyword` family including `keyword`, `constant_keyword`, and `wildcard` -- `int` (`short` and `byte` are represented as `int`) -- `long` -- `null` -- `text` -- `unsigned_long` (preview) -- `version` +### Date Math +- Date math expressions are limited. For example, subtracting two datetime values or using parentheses in date math expressions is not supported. -Spatial types: -- `geo_point` -- `geo_shape` -- `point` -- `shape` +### Multivalued Fields +- Functions generally return `null` when applied to multivalued fields unless explicitly documented otherwise. Use multivalue functions to handle such fields. -Unsupported types: -- TSDB metrics: `counter`, `position`, `aggregate_metric_double` -- Date/time: `date_nanos`, `date_range` -- Other types: `binary`, `completion`, `dense_vector`, `double_range`, `flattened`, `float_range`, `histogram`, `integer_range`, `ip_range`, `long_range`, `nested`, `rank_feature`, `rank_features`, `search_as_you_type` +### Timezone Support +- ES|QL only supports the UTC timezone. -Querying a column with an unsupported type returns an error. If a column with an unsupported type is not explicitly used in a query, it is returned with `null` values, with the exception of nested fields. Nested fields are not returned at all. +### Kibana Integration +- The Discover interface in Kibana has a 10,000-row limit for displayed results and a 50-column limit for displayed fields. These limits apply only to the UI and not to the underlying query execution. -#### _source Availability +--- -ES|QL does not support configurations where the `_source` field is disabled. ES|QL’s support for synthetic `_source` is currently experimental. +## Using ES|QL in Kibana -#### Full-Text Search +ES|QL is integrated into Kibana, allowing users to query and visualize data directly from the Discover interface. Key points for using ES|QL in Kibana include: -Because of the way ES|QL treats `text` values, queries on `text` fields are like queries on `keyword` fields: they are case-sensitive and need to match the full string. -To perform full-text search on `text` fields, search functions such as `MATCH` should be used. +- **Enablement**: ES|QL is enabled by default in Kibana but can be disabled via the `enableESQL` setting in Advanced Settings. +- **Query Bar**: The query bar in Discover supports ES|QL syntax, with features like auto-complete and query history for ease of use. +- **Visualization**: ES|QL queries can be used to create visualizations, which can be saved to dashboards or used for alerting. +- **Time Filtering**: Use the standard time filter or custom time parameters (`?_tstart` and `?_tend`) to filter data by time range. -#### Time Series Data Streams - -ES|QL does not support querying time series data streams (TSDS). - -#### Date Math Limitations +### Example Query in Kibana +```esql +FROM kibana_sample_data_logs +| WHERE @timestamp > NOW() - 1 day +| STATS total_bytes = SUM(bytes) BY geo.dest +| SORT total_bytes DESC +| LIMIT 5 +``` -Date math expressions work well when the leftmost expression is a datetime. However, using parentheses or putting the datetime to the right is not always supported yet. Date math does not allow subtracting two datetimes. +This query retrieves the top 5 destinations by total bytes in the last 24 hours. -#### Timezone Support +--- -ES|QL only supports the UTC timezone. +## Cross-Cluster Querying with ES|QL -### Cross-Cluster Querying +ES|QL supports querying across multiple clusters, enabling users to analyze data stored in different Elasticsearch clusters. To query remote clusters, use the format `:` in the `FROM` command. -Using ES|QL across clusters allows you to execute a single query across multiple clusters. This feature is in technical preview and may be changed or removed in a future release. +### Example Cross-Cluster Query +```esql +FROM cluster_one:employees,cluster_two:other-employees-* +| STATS avg_salary = AVG(salary) BY department +| SORT avg_salary DESC +``` -#### Prerequisites +This query retrieves the average salary by department across two clusters and sorts the results in descending order. -- Remote clusters must be configured. -- The local coordinating node must have the `remote_cluster_client` node role. -- Security privileges must be configured appropriately. +--- -#### Querying Across Clusters +## Using the ES|QL REST API -In the `FROM` command, specify data streams and indices on remote clusters using the format `:`. For example: +The ES|QL REST API allows users to execute ES|QL queries programmatically. Queries are sent as HTTP POST requests to the `_query` endpoint. -```esql -FROM cluster_one:my-index-000001 -| LIMIT 10 +### Example REST API Request +```json +POST /_query +{ + "query": "FROM employees | WHERE salary > 50000 | SORT salary DESC | LIMIT 10" +} ``` -### Using ES|QL in Kibana +### Key Points +- The `query` field contains the ES|QL query as a string. +- Use the `params` field to pass query parameters dynamically. +- The API returns results in JSON format, making it easy to integrate with other applications. -ES|QL can be used in Kibana to query and aggregate data, create visualizations, and set up alerts. +--- -#### Important Information +## Summary -- ES|QL is enabled by default in Kibana. -- The query bar in Discover allows you to write and execute ES|QL queries. -- The results table shows up to 10,000 rows, and Discover shows no more than 50 columns. -- You can create visualizations and alerts based on ES|QL queries. +ES|QL is a versatile and user-friendly query language for Elasticsearch, offering powerful capabilities for data analysis and transformation. While it has some limitations, its integration with Kibana and support for cross-cluster querying make it a valuable tool for a wide range of use cases. Whether you're analyzing logs, building dashboards, or creating alerts, ES|QL provides the flexibility and performance needed to work with Elasticsearch data effectively. +``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt index 499e666de2f03..aea8b7c6dac28 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-percentile.txt @@ -1,6 +1,6 @@ # PERCENTILE -The PERCENTILE function calculates the value at a specified percentile of observed values. +The `PERCENTILE` function calculates the value at which a specified percentage of observed values occur. For example, the 95th percentile is the value greater than 95% of the observed values, while the 50th percentile corresponds to the `MEDIAN`. ## Syntax @@ -8,28 +8,35 @@ The PERCENTILE function calculates the value at a specified percentile of observ ### Parameters -#### number +#### `number` -The numeric expression that represents the set of values to be analyzed. +The numeric field or expression for which the percentile is calculated. -#### percentile +#### `percentile` -The percentile to compute. The value should be between 0 and 100. +The percentile value to calculate (e.g., 0 for the minimum, 50 for the median, 100 for the maximum). ## Examples +Basic Percentile Calculation + ```esql FROM employees | STATS p0 = PERCENTILE(salary, 0), p50 = PERCENTILE(salary, 50), p99 = PERCENTILE(salary, 99) ``` +This example calculates the 0th percentile (minimum), 50th percentile (median), and 99th percentile of the `salary` field. + +Using Inline Functions + ```esql FROM employees | STATS p80_max_salary_change = PERCENTILE(MV_MAX(salary_change), 80) ``` +This example calculates the 80th percentile of the maximum values in a multivalued column `salary_change`. The `MV_MAX` function is used to determine the maximum value per row before applying the `PERCENTILE` function. + ## Notes - PERCENTILE is usually approximate. - - PERCENTILE is also non-deterministic. This means you can get slightly different results using the same data. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt index e97942a90a15f..d3b15417f5208 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pi.txt @@ -1,6 +1,6 @@ # PI -The PI function returns the mathematical constant Pi, which is the ratio of a circle's circumference to its diameter. +Returns Pi, the mathematical constant representing the ratio of a circle’s circumference to its diameter. ## Syntax @@ -8,16 +8,12 @@ The PI function returns the mathematical constant Pi, which is the ratio of a ci ### Parameters -This function does not require any parameters. +This function does not take any parameters. ## Examples -```esql -ROW PI() -``` +Returning the value of Pi ```esql -FROM employees -| EVAL pi_value = PI() -| KEEP pi_value +ROW PI() ``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt index 22a0ca966e0d0..c5b5f4ef8f86a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-pow.txt @@ -1,6 +1,6 @@ # POW -The POW function calculates the value of a base number raised to the power of an exponent number. +The `POW` function calculates the value of a base raised to the power of an exponent. ## Syntax @@ -8,22 +8,36 @@ The POW function calculates the value of a base number raised to the power of an ### Parameters -#### base +#### `base` -This is a numeric expression for the base. +Numeric expression for the base. If `null`, the function returns `null`. -#### exponent +#### `exponent` -This is a numeric expression for the exponent. +Numeric expression for the exponent. If `null`, the function returns `null`. ## Examples +Basic usage + ```esql ROW base = 2.0, exponent = 2 | EVAL result = POW(base, exponent) ``` +Calculate `2.0` raised to the power of `2`. + +Fractional exponent (root calculation) + +The exponent can be a fraction, which is similar to performing a root. For example, an exponent of `0.5` calculates the square root of the base: + ```esql ROW base = 4, exponent = 0.5 | EVAL s = POW(base, exponent) ``` + +Calculate the square root of `4` using an exponent of `0.5`. + +## Limitations + +- It is possible to overflow a double result when using this function. In such cases, the function will return `null`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-qstr.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-qstr.txt index 89f5d8a81c1be..401e8162dedc0 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-qstr.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-qstr.txt @@ -1,8 +1,6 @@ -# QSTR +## QSTR -The QSTR function performs a query string query, returning true if the provided query string matches a row. - -Please note this functionality is currently in its technical preview stage, which means it might undergo changes or removal in future releases. Elastic commits to address any issues during this period. However, since it's a technical preview, it doesn't come under the support SLA of official GA features. +Performs a query string query and returns `true` if the provided query string matches the row. ## Syntax @@ -10,22 +8,18 @@ Please note this functionality is currently in its technical preview stage, whic ### Parameters -#### query +#### `query` -The query parameter must be a string written in the Lucene query format. +Query string in Lucene query string format. ## Examples -Conduct a query string query on a book's author: - ```esql FROM books | WHERE QSTR("author: Faulkner") | KEEP book_no, author | SORT book_no -| LIMIT 5; +| LIMIT 5 ``` -## Notes - -- Do not use `QSTR` in production - it is in technical preview and may be changed or removed in a future release +This example filters rows where the `author` field matches "Faulkner," keeps the `book_no` and `author` columns, sorts by `book_no`, and limits the output to 5 rows. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt index bd482258c21f6..667708b17907b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rename.txt @@ -1,6 +1,6 @@ -# RENAME +## RENAME -The RENAME command is used to change the names of one or more columns in a table. +The `RENAME` command is used to rename one or more columns in a table. If a column with the new name already exists, it will be replaced by the renamed column. ## Syntax @@ -8,28 +8,32 @@ The RENAME command is used to change the names of one or more columns in a table ### Parameters -#### old_nameX +#### `old_nameX` -This is the current name of the column that you want to rename. +The name of the column you want to rename. -#### new_nameX +#### `new_nameX` -This is the new name that you want to assign to the column. If a column with the new name already exists, the existing column will be replaced. If multiple columns are renamed to the same name, all but the rightmost column with the same new name will be dropped. +The new name for the column. If it conflicts with an existing column name, the existing column is dropped. If multiple columns are renamed to the same name, all but the rightmost column with the same new name are dropped. ## Examples -The following example renames the column "still_hired" to "employed": +### Rename a single column + +Rename the `still_hired` column to `employed`: ```esql FROM employees | KEEP first_name, last_name, still_hired -| RENAME still_hired AS employed +| RENAME still_hired AS employed ``` -You can rename multiple columns with a single RENAME command: +### Rename multiple columns + +Rename `first_name` to `fn` and `last_name` to `ln` in a single command: ```esql FROM employees | KEEP first_name, last_name | RENAME first_name AS fn, last_name AS ln -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt index f6a830f6ee3f5..9a3e312b4d31a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-repeat.txt @@ -1,6 +1,6 @@ # REPEAT -The REPEAT function generates a string by repeating a specified string a certain number of times. +The `REPEAT` function constructs a string by concatenating a given string with itself a specified number of times. ## Syntax @@ -8,13 +8,13 @@ The REPEAT function generates a string by repeating a specified string a certain ### Parameters -#### string +#### `string` -The string that you want to repeat. +The string to be repeated. -#### number +#### `number` -The number of times you want to repeat the string. +The number of times the string should be repeated. ## Examples @@ -23,7 +23,4 @@ ROW a = "Hello!" | EVAL triple_a = REPEAT(a, 3) ``` -```esql -ROW greeting = "Hi" -| EVAL repeated_greeting = REPEAT(greeting, 5) -``` +This example creates a new column `triple_a` by repeating the string `"Hello!"` three times. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt index 930efd579f610..6c1470f3fad41 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-replace.txt @@ -1,6 +1,6 @@ # REPLACE -The REPLACE function substitutes any match of a regular expression within a string with a replacement string. +The `REPLACE` function substitutes any match of a regular expression in a string with a specified replacement string. ## Syntax @@ -8,17 +8,17 @@ The REPLACE function substitutes any match of a regular expression within a stri ### Parameters -#### string +#### `string` -The string expression where the replacement will occur. +String expression. -#### regex +#### `regex` -The regular expression that will be matched in the string. +Regular expression. -#### newString +#### `newString` -The string that will replace the matched regular expression in the string. +Replacement string. ## Examples diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-reverse.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-reverse.txt index f1372b06abf61..680fefb3a6504 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-reverse.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-reverse.txt @@ -1,6 +1,6 @@ # REVERSE -The REVERSE function returns a reversed form of the input string. +The `REVERSE` function returns a new string with the characters of the input string in reverse order. ## Syntax @@ -8,22 +8,36 @@ The REVERSE function returns a reversed form of the input string. ### Parameters -#### str +#### `str` -The string you want to reverse. If the string is `null`, the function will also return `null`. +String expression. If `null`, the function returns `null`. ## Examples -Here's an example of how to reverse a string: +Reversing a simple string ```esql ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); ``` -REVERSE also works with unicode characters, keeping unicode grapheme clusters intact during reversal: +| message | message_reversed | +|-----------|------------------| +| Some Text | txeT emoS | + +Reversing a string with emojis ```esql ROW bending_arts = "💧🪨🔥💨" | EVAL bending_arts_reversed = REVERSE(bending_arts); -``` \ No newline at end of file +``` + +| bending_arts | bending_arts_reversed | +|--------------|-----------------------| +| 💧🪨🔥💨 | 💨🔥🪨💧 | + +`REVERSE` works with Unicode and preserves grapheme clusters during reversal. + +## Limitations + +If Elasticsearch is running with a JDK version less than 20, the function may not properly reverse grapheme clusters. For example, "👍🏽😊" might be reversed to "🏽👍😊" instead of the correct "😊👍🏽". Elastic Cloud and the JDK bundled with Elasticsearch use newer JDKs, so this issue typically arises only if an older JDK is explicitly used. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt index 081fe025522c5..5904cc2520b2f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-right.txt @@ -1,6 +1,6 @@ # RIGHT -The RIGHT function extracts a specified number of characters from the end of a string. +Returns a substring by extracting a specified number of characters from the right side of a string. ## Syntax @@ -8,18 +8,16 @@ The RIGHT function extracts a specified number of characters from the end of a s ### Parameters -#### string +#### `string` -The string from which a substring is to be returned. +The string from which to return a substring. -#### length +#### `length` -The number of characters to return from the end of the string. +The number of characters to return. ## Examples -The following example extracts the last three characters from the `last_name` field: - ```esql FROM employees | KEEP last_name @@ -28,8 +26,4 @@ FROM employees | LIMIT 5 ``` -```esql -ROW full_name = "John Doe" -| EVAL last_part = RIGHT(full_name, 4) -| KEEP last_part -``` +Extracts the last three characters from the `last_name` column, sorts the results alphabetically, and limits the output to the first five rows. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt index 1e62d7a4c4915..a8f5ed668a899 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-round.txt @@ -1,6 +1,6 @@ # ROUND -The ROUND function rounds a numeric value to a specified number of decimal places. +Rounds a number to the specified number of decimal places. By default, it rounds to 0 decimal places, returning the nearest integer. If the precision is a negative number, it rounds to the specified number of digits left of the decimal point. ## Syntax @@ -8,24 +8,25 @@ The ROUND function rounds a numeric value to a specified number of decimal place ### Parameters -#### number +#### `number` -The numeric value to be rounded. +The numeric value to round. If `null`, the function returns `null`. -#### decimals +#### `decimals` -The number of decimal places to which the number should be rounded. The default value is 0. +The number of decimal places to round to. Defaults to 0. If `null`, the function returns `null`. ## Examples -The following example rounds the height of employees to one decimal place after converting it from meters to feet: - +Rounding a height value to one decimal place ```esql FROM employees | KEEP first_name, last_name, height | EVAL height_ft = ROUND(height * 3.281, 1) ``` +This example converts the `height` column from meters to feet and rounds the result to one decimal place. + ```esql FROM sales | KEEP product_name, revenue diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt index 26b994ecbee6c..3c7bd22843bb5 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-row.txt @@ -1,6 +1,6 @@ # ROW -The ROW command is used to generate a row with one or more columns with specified values. This can be particularly useful for testing purposes. +The `ROW` command generates a single row with one or more columns, each assigned a specified value. This is particularly useful for testing purposes. ## Syntax @@ -8,37 +8,37 @@ The ROW command is used to generate a row with one or more columns with specifie ### Parameters -#### {column name} +#### `columnX` -This is the name of the column. If there are duplicate column names, only the rightmost duplicate will create a column. +The name of the column. +If duplicate column names are provided, only the rightmost duplicate creates a column. -#### {value} +#### `valueX` -This is the value for the column. It can be a literal, an expression, or a function. +The value assigned to the column. This can be a literal, an expression, or a function. ## Examples -1. Creating a row with simple literal values: - ```esql +Basic usage + +Create a row with three columns, each assigned a specific value: + +```esql ROW a = 1, b = "two", c = null ``` -2. Creating a row with multi-value columns using square brackets: - ```esql +Multi-value columns + +Use square brackets to assign multiple values to a single column: + +```esql ROW a = [2, 1] ``` -3. Creating a row with a function: - ```esql -ROW a = ROUND(1.23, 0) -``` +Using functions -4. Combining literals, multi-value columns, and functions: - ```esql -ROW x = 5, y = [3, 4], z = TO_STRING(123) -``` +Generate a row where a column's value is calculated using a function: -5. Using nested functions within a row: - ```esql -ROW a = ABS(-10), b = CONCAT("Hello", " ", "World"), c = TO_BOOLEAN("true") +```esql +ROW a = ROUND(1.23, 0) ``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt index 1a57382fe8c3e..ad01facba5244 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-rtrim.txt @@ -1,6 +1,6 @@ # RTRIM -The RTRIM function is used to remove trailing whitespaces from a string. +Removes trailing whitespaces from a string. ## Syntax @@ -8,14 +8,12 @@ The RTRIM function is used to remove trailing whitespaces from a string. ### Parameters -#### string +#### `string` -This is the string expression from which trailing whitespaces will be removed. +String expression. If `null`, the function returns `null`. ## Examples -The following example demonstrates how to use the RTRIM function: - ```esql ROW message = " some text ", color = " red " | EVAL message = RTRIM(message) @@ -23,3 +21,5 @@ ROW message = " some text ", color = " red " | EVAL message = CONCAT("'", message, "'") | EVAL color = CONCAT("'", color, "'") ``` + +This example removes trailing whitespaces from the `message` and `color` columns, then wraps the resulting strings in single quotes. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt index 13e046076e30b..6368411d3c9a8 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-show.txt @@ -1,6 +1,6 @@ -# SHOW +## SHOW -The SHOW command retrieves details about the deployment and its capabilities. +The `SHOW` command provides information about the deployment and its capabilities. ## Syntax @@ -8,9 +8,9 @@ The SHOW command retrieves details about the deployment and its capabilities. ### Parameters -#### item +#### `item` -The only acceptable value is `INFO`. +This parameter can only be `INFO`. ## Examples @@ -18,4 +18,4 @@ Retrieve the deployment’s version, build date, and hash: ```esql SHOW INFO -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt index 083d913dd99b4..aabe47a38a152 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-signum.txt @@ -1,6 +1,6 @@ # SIGNUM -The SIGNUM function returns the sign of a given number. It outputs `-1` for negative numbers, `0` for `0`, and `1` for positive numbers. +Returns the sign of a given number. It outputs `-1` for negative numbers, `0` for `0`, and `1` for positive numbers. ## Syntax @@ -8,9 +8,9 @@ The SIGNUM function returns the sign of a given number. It outputs `-1` for nega ### Parameters -#### number +#### `number` -A numeric expression. +Numeric expression. If `null`, the function returns `null`. ## Examples @@ -19,7 +19,24 @@ ROW d = 100.0 | EVAL s = SIGNUM(d) ``` -```esql -ROW d = -50.0 -| EVAL s = SIGNUM(d) -``` +This example calculates the sign of the number `100.0`. + + +## Notes + +If SORT is used right after a KEEP command, make sure it only uses column names in KEEP, +or move the SORT before the KEEP, e.g. +- not correct: KEEP date | SORT @timestamp, +- correct: SORT @timestamp | KEEP date + +By default, the sorting order is ascending. You can specify an explicit sort order by using `ASC` for ascending or `DESC` for descending. + +If two rows have the same sort key, they are considered equal. You can provide additional sort expressions to act as tie breakers. + +When sorting on multivalued columns, the lowest value is used when sorting in ascending order and the highest value is used when sorting in descending order. + +By default, `null` values are treated as being larger than any other value. This means that with an ascending sort order, `null` values are sorted last, and with a descending sort order, `null` values are sorted first. You can change this by providing `NULLS FIRST` or `NULLS LAST`. + +## Limitations +- **Multivalued Columns**: When sorting on multivalued columns, the lowest value is used for ascending order and the highest value for descending order. +- **Null Values**: By default, null values are treated as larger than any other value. This can be changed using `NULLS FIRST` or `NULLS LAST`. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt index 3d28baafd53d5..3b4ef78d1bd5b 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sin.txt @@ -1,6 +1,6 @@ # SIN -The SIN function calculates the sine of a given angle. +Returns the sine of an angle. ## Syntax @@ -8,9 +8,9 @@ The SIN function calculates the sine of a given angle. ### Parameters -#### angle +#### `angle` -The angle for which the sine value is to be calculated. The angle should be in radians. +An angle, in radians. If `null`, the function returns `null`. ## Examples @@ -19,7 +19,4 @@ ROW a=1.8 | EVAL sin = SIN(a) ``` -```esql -ROW angle=0.5 -| EVAL sine_value = SIN(angle) -``` +Calculate the sine of the angle `1.8` radians and store the result in a new column named `sin`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt index eaec5ceb54862..e1eaf459566b8 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sinh.txt @@ -1,25 +1,22 @@ # SINH -The SINH function calculates the hyperbolic sine of a given angle. +Returns the hyperbolic sine of a number. ## Syntax -`SINH(angle)` +`SINH(number)` ### Parameters -#### angle +#### number -The angle in radians for which the hyperbolic sine is to be calculated. If the parameter is null, the function will return null. +A numeric expression. If `null`, the function returns `null`. ## Examples ```esql ROW a=1.8 -| EVAL sinh=SINH(a) +| EVAL sinh = SINH(a) ``` -```esql -ROW angle=0.5 -| EVAL hyperbolic_sine = SINH(angle) -``` +Calculate the hyperbolic sine of the value `1.8`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt index 593d94021b71b..ad400c7c6c26a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sort.txt @@ -1,6 +1,6 @@ -# SORT +## SORT -The SORT command is used to arrange a table based on one or more columns. +The `SORT` command organizes a table by one or more columns. ## Syntax @@ -8,13 +8,15 @@ The SORT command is used to arrange a table based on one or more columns. ### Parameters -#### columnX +#### `columnX` -The column on which the sorting is to be performed. +The column to sort on. ## Examples -Sort a table based on the 'height' column: +### Basic sorting + +Sort the table by the `height` column in ascending order (default behavior): ```esql FROM employees @@ -22,7 +24,9 @@ FROM employees | SORT height ``` -Explicitly sort in ascending order with `ASC`: +### Explicitly sorting in descending order with `DESC` + +Sort the table by the `height` column in descending order: ```esql FROM employees @@ -30,7 +34,9 @@ FROM employees | SORT height DESC ``` -Provide additional sort expressions to act as tie breakers: +### Providing additional sort expressions to act as tie breakers + +Sort the table by `height` in descending order, and use `first_name` in ascending order as a tie breaker: ```esql FROM employees @@ -38,7 +44,9 @@ FROM employees | SORT height DESC, first_name ASC ``` -Sort `null` values first using `NULLS FIRST`: +### Sorting `null` values first using `NULLS FIRST` + +Sort the table by `first_name` in ascending order, placing `null` values first: ```esql FROM employees diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-space.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-space.txt index fb00d39e6b426..d34e58ddf1e18 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-space.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-space.txt @@ -1,6 +1,6 @@ # SPACE -The SPACE function creates a string composed of a specific number of spaces. +Returns a string composed of a specified number of spaces. ## Syntax @@ -8,15 +8,14 @@ The SPACE function creates a string composed of a specific number of spaces. ### Parameters -#### number +#### `number` -The number of spaces the function should generate. +The number of spaces to include in the resulting string. ## Examples -This example demonstrates how to use the SPACE function to insert a space into a string: - ```esql ROW message = CONCAT("Hello", SPACE(1), "World!"); ``` -In this example, the SPACE function creates a single space, which is then used to separate the words "Hello" and "World!" in the resulting string. If desired, the `number` parameter could be adjusted in order to generate more spaces. \ No newline at end of file + +This example creates a string with the word "Hello," followed by a single space, and then the word "World!". diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt index 14ff9ecb94bcd..27dc7d36d2bad 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-split.txt @@ -1,6 +1,6 @@ # SPLIT -The SPLIT function is used to divide a single string into multiple strings. +The `SPLIT` function splits a single-valued string into multiple strings based on a specified delimiter. ## Syntax @@ -8,13 +8,13 @@ The SPLIT function is used to divide a single string into multiple strings. ### Parameters -#### string +#### `string` -This is the string expression that you want to split. +String expression. If `null`, the function returns `null`. -#### delim +#### `delim` -This is the delimiter used to split the string. Currently, only single byte delimiters are supported. +Delimiter used to split the string. Only single-byte delimiters are currently supported. ## Examples @@ -23,7 +23,4 @@ ROW words="foo;bar;baz;qux;quux;corge" | EVAL word = SPLIT(words, ";") ``` -```esql -ROW sentence="hello world;this is ES|QL" -| EVAL words = SPLIT(sentence, " ") -``` +This example splits the string `words` into multiple strings using the semicolon (`;`) as the delimiter. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt index d1839b7d6d06a..aa3694cea87ee 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sqrt.txt @@ -1,6 +1,6 @@ # SQRT -The SQRT function calculates the square root of a given number. +Returns the square root of a number. The input can be any numeric value, and the return value is always a double. Square roots of negative numbers and infinities are `null`. ## Syntax @@ -8,20 +8,22 @@ The SQRT function calculates the square root of a given number. ### Parameters -#### number +#### `number` -This is a numeric expression. +Numeric expression. If `null`, the function returns `null`. ## Examples - ```esql ROW d = 100.0 | EVAL s = SQRT(d) ``` +Calculate the square root of the value `100.0`. + ```esql FROM employees | KEEP first_name, last_name, height | EVAL sqrt_height = SQRT(height) ``` +Keep only the first_name, last_name, height columns, and then create a new `sqrt_height` which equals to the square root of all the values in the height column. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt index b9baa82bfe5ae..8b7a421e99e9a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_centroid_agg.txt @@ -1,6 +1,6 @@ # ST_CENTROID_AGG -The ST_CENTROID_AGG function calculates the spatial centroid over a field with spatial point geometry type. +Calculates the spatial centroid over a field with a spatial point geometry type. ## Syntax @@ -8,20 +8,15 @@ The ST_CENTROID_AGG function calculates the spatial centroid over a field with s ### Parameters -#### field +#### `field` -The field parameter represents the column that contains the spatial point geometry data. +The field containing spatial point geometry data. ## Examples -Here is an example of how to use the ST_CENTROID_AGG function: - ```esql FROM airports | STATS centroid = ST_CENTROID_AGG(location) ``` -```esql -FROM city_boundaries -| STATS city_centroid = ST_CENTROID_AGG(boundary) -``` +Calculate the spatial centroid of the `location` field in the `airports` dataset. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt index 50f5608b22046..87bd34d7aeb55 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_contains.txt @@ -1,6 +1,6 @@ # ST_CONTAINS -The ST_CONTAINS function determines if the first specified geometry encompasses the second one. This function is the inverse of the ST_WITHIN function. +Determines whether the first geometry contains the second geometry. This is the inverse of the `ST_WITHIN` function. ## Syntax @@ -10,20 +10,26 @@ The ST_CONTAINS function determines if the first specified geometry encompasses #### geomA -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. #### geomB -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. The second parameter must have the same coordinate system as the first. Combining `geo_*` and `cartesian_*` parameters is not allowed. ## Examples +Filtering city boundaries containing a specific polygon + ```esql FROM airport_city_boundaries | WHERE ST_CONTAINS(city_boundary, TO_GEOSHAPE("POLYGON((109.35 18.3, 109.45 18.3, 109.45 18.4, 109.35 18.4, 109.35 18.3))")) | KEEP abbrev, airport, region, city, city_location ``` +This query filters city boundaries that contain the specified polygon and keeps selected fields in the output. + +Filtering regional boundaries containing a specific polygon + ```esql FROM regions | WHERE ST_CONTAINS(region_boundary, TO_GEOSHAPE("POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))")) diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt index 41433a4069f0c..f501fc449dd38 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_disjoint.txt @@ -1,6 +1,6 @@ # ST_DISJOINT -The ST_DISJOINT function checks if two geometries or geometry columns are disjoint, meaning they do not intersect. This function is the inverse of the ST_INTERSECTS function. In mathematical terms, if A and B are two geometries, ST_Disjoint(A, B) is true if and only if the intersection of A and B is empty. +Determines whether two geometries or geometry columns are disjoint. This is the inverse of the `ST_INTERSECTS` function. Mathematically, two geometries are disjoint if their intersection is empty: `ST_Disjoint(A, B) ⇔ A ⋂ B = ∅`. ## Syntax @@ -10,14 +10,18 @@ The ST_DISJOINT function checks if two geometries or geometry columns are disjoi #### geomA -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. #### geomB -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. The second parameter must have the same coordinate system as the first. Combining `geo_*` and `cartesian_*` parameters is not supported. ## Examples +Filtering disjoint geometries + +The following query filters rows where the `city_boundary` geometry is disjoint from a specified polygon. It keeps only the specified columns in the result. + ```esql FROM airport_city_boundaries | WHERE ST_DISJOINT(city_boundary, TO_GEOSHAPE("POLYGON((-10 -60, 120 -60, 120 60, -10 60, -10 -60))")) @@ -32,4 +36,4 @@ FROM airport_city_boundaries ## Limitations -It's important to note that the second parameter must have the same coordinate system as the first. This means you cannot combine `geo_*` and `cartesian_*` parameters. +- The two geometries must share the same coordinate system. For example, you cannot mix `geo_*` and `cartesian_*` types in the same function call. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt index 5a007367dc0b8..059bd93aa4829 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_distance.txt @@ -1,6 +1,6 @@ # ST_DISTANCE -The ST_DISTANCE function calculates the distance between two points. +Computes the distance between two points. For cartesian geometries, it calculates the pythagorean distance in the same units as the original coordinates. For geographic geometries, it computes the circular distance along the great circle in meters. ## Syntax @@ -10,28 +10,24 @@ The ST_DISTANCE function calculates the distance between two points. #### geomA -This is an expression of type `geo_point` or `cartesian_point`. +- Expression of type `geo_point` or `cartesian_point`. +- If `null`, the function returns `null`. #### geomB -This is an expression of type `geo_point` or `cartesian_point`. +- Expression of type `geo_point` or `cartesian_point`. +- If `null`, the function returns `null`. +- Must have the same coordinate system as `geomA`. Combining `geo_point` and `cartesian_point` parameters is not supported. ## Examples -```esql -FROM airports -| WHERE abbrev == "CPH" -| EVAL distance = ST_DISTANCE(location, city_location) -| KEEP abbrev, name, location, city_location, distance -``` +Calculating the distance between two points ```esql FROM airports -| WHERE abbrev == "JFK" +| WHERE abbrev == "CPH" | EVAL distance = ST_DISTANCE(location, city_location) | KEEP abbrev, name, location, city_location, distance ``` -## Limitations - -- It's important to note that the second parameter must have the same coordinate system as the first. Therefore, it's not possible to combine `geo_point` and `cartesian_point` parameters. +This example calculates the distance between the airport's location and the city's location for the airport with the abbreviation "CPH". diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_envelope.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_envelope.txt index 6ecdf385da18b..a400101648a3f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_envelope.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_envelope.txt @@ -1,6 +1,6 @@ # ST_ENVELOPE -The ST_ENVELOPE function determines the minimum bounding box for the provided geometry. +Determines the minimum bounding box of the supplied geometry. ## Syntax @@ -8,17 +8,17 @@ The ST_ENVELOPE function determines the minimum bounding box for the provided ge ### Parameters -#### geometry +#### `geometry` -The `geometry` parameter refers to the input geometry. This should be an expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If the parameter is `null`, the function will also return `null`. +Expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If `null`, the function returns `null`. ## Examples -Here is an example where ST_ENVELOPE is used to determine the minimum bounding box of a city's boundary: - ```esql FROM airport_city_boundaries | WHERE abbrev == "CPH" | EVAL envelope = ST_ENVELOPE(city_boundary) | KEEP abbrev, airport, envelope ``` + +This example calculates the minimum bounding box for the `city_boundary` geometry of the airport with the abbreviation "CPH". \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_extent_agg.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_extent_agg.txt index 87a6299daa741..2f584794d0674 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_extent_agg.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_extent_agg.txt @@ -1,6 +1,6 @@ # ST_EXTENT_AGG -This function calculates the spatial extent over a field that has a geometry type, returning a bounding box that contains all values of the specified field. +Calculates the spatial extent over a field with a geometry type, returning a bounding box for all values of the field. ## Syntax @@ -8,13 +8,13 @@ This function calculates the spatial extent over a field that has a geometry typ ### Parameters -#### field +#### `field` -The field of geometry type over which the spatial extent will be calculated. +The field containing geometry data over which the spatial extent is calculated. ## Examples -The following example calculates the spatial extent over the 'location' field for all airports in India: +Calculate the spatial extent of airport locations in India ```esql FROM airports @@ -22,4 +22,4 @@ FROM airports | STATS extent = ST_EXTENT_AGG(location) ``` -This query returns a bounding box that encompasses all airport locations in India. \ No newline at end of file +This example calculates the bounding box for the `location` field of airports in India. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt index 63e2ff127d1a9..48eaeb7ec716d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_intersects.txt @@ -1,6 +1,6 @@ # ST_INTERSECTS -The ST_INTERSECTS function checks if two geometries intersect. They intersect if they share any point, including points within their interiors (points along lines or within polygons). This function is the inverse of the ST_DISJOINT function. In mathematical terms, ST_Intersects(A, B) is true if the intersection of A and B is not empty. +Determines whether two geometries intersect. Two geometries intersect if they share any point in common, including points along lines or within polygons. This function is the inverse of `ST_DISJOINT`. ## Syntax @@ -8,23 +8,21 @@ The ST_INTERSECTS function checks if two geometries intersect. They intersect if ### Parameters -#### geomA +#### `geomA` -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. +An expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. -#### geomB +#### `geomB` -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters. +An expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. The coordinate system of `geomB` must match that of `geomA`. Combining `geo_*` and `cartesian_*` parameters is not allowed. ## Examples +Checking if a location intersects with a polygon + ```esql FROM airports | WHERE ST_INTERSECTS(location, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))")) ``` -```esql -FROM city_boundaries -| WHERE ST_INTERSECTS(boundary, TO_GEOSHAPE("POLYGON((10 10, 20 10, 20 20, 10 20, 10 10))")) -| KEEP city_name, boundary -``` +This example filters airports to find those whose `location` intersects with the specified polygon. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt index 20e61d6a234df..a150fee1b1b4d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_within.txt @@ -1,6 +1,6 @@ # ST_WITHIN -The ST_WITHIN function checks if the first geometry is located within the second geometry. +Determines whether the first geometry is within the second geometry. This is the inverse of the `ST_CONTAINS` function. ## Syntax @@ -8,24 +8,22 @@ The ST_WITHIN function checks if the first geometry is located within the second ### Parameters -#### geomA +#### `geomA` -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If the value is `null`, the function will return `null`. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. -#### geomB +#### `geomB` -This is an expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If the value is `null`, the function will return `null`. It's important to note that the second parameter must have the same coordinate system as the first. This means you cannot combine `geo_*` and `cartesian_*` parameters. +Expression of type `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`. If `null`, the function returns `null`. The second parameter must have the same coordinate system as the first. Combining `geo_*` and `cartesian_*` parameters is not allowed. ## Examples +Filtering rows where a city boundary is within a specified polygon + ```esql FROM airport_city_boundaries | WHERE ST_WITHIN(city_boundary, TO_GEOSHAPE("POLYGON((109.1 18.15, 109.6 18.15, 109.6 18.65, 109.1 18.65, 109.1 18.15))")) | KEEP abbrev, airport, region, city, city_location ``` -```esql -FROM parks -| WHERE ST_WITHIN(park_boundary, TO_GEOSHAPE("POLYGON((40.7128 -74.0060, 40.7128 -73.9352, 40.7306 -73.9352, 40.7306 -74.0060, 40.7128 -74.0060))")) -| KEEP park_name, park_boundary -``` +This example filters rows where the `city_boundary` geometry is entirely within the specified polygon. It then keeps only the `abbrev`, `airport`, `region`, `city`, and `city_location` columns. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt index 18e35874333fb..c1b19d393e5cc 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_x.txt @@ -1,6 +1,6 @@ # ST_X -The ST_X function extracts the `x` coordinate from a given point. +Extracts the `x` coordinate from the supplied point. For points of type `geo_point`, this corresponds to the `longitude` value. ## Syntax @@ -8,20 +8,15 @@ The ST_X function extracts the `x` coordinate from a given point. ### Parameters -#### point +#### `point` -This is an expression of type `geo_point` or `cartesian_point`. +Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. ## Examples -Here is an example of how to use the ST_X function: +Extract the `x` (longitude) and `y` (latitude) coordinates from a `geo_point`: ```esql ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") | EVAL x = ST_X(point), y = ST_Y(point) -``` - -```esql -ROW point = TO_CARTESIANPOINT("POINT(100.0 200.0)") -| EVAL x = ST_X(point), y = ST_Y(point) -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmax.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmax.txt index bf6c95eee2b8a..12775a0f427e0 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmax.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmax.txt @@ -1,6 +1,6 @@ # ST_XMAX -The ST_XMAX function extracts the maximum value of the x coordinates from the supplied geometry. +Extracts the maximum value of the `x` coordinates from the supplied geometry. For geometries of type `geo_point` or `geo_shape`, this corresponds to the maximum `longitude` value. ## Syntax @@ -8,14 +8,12 @@ The ST_XMAX function extracts the maximum value of the x coordinates from the su ### Parameters -#### point +#### `point` -This is an expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. The function returns `null` if the point is `null`. +Expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If `null`, the function returns `null`. ## Examples -Here's an example of how to use the ST_XMAX function: - ```esql FROM airport_city_boundaries | WHERE abbrev == "CPH" @@ -24,4 +22,4 @@ FROM airport_city_boundaries | KEEP abbrev, airport, xmin, xmax, ymin, ymax ``` -In this example, the ST_XMAX function is used to extract the maximum x coordinate from the envelope of the 'city_boundary' field. +This example calculates the bounding box of the city boundary for the airport with the abbreviation "CPH" and extracts the minimum and maximum `x` and `y` coordinates. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmin.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmin.txt index f96d3705f5897..0f39dbb736d59 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmin.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_xmin.txt @@ -1,6 +1,6 @@ # ST_XMIN -ST_XMIN retrieves the minimum 'x' coordinate from the provided geometry. +Extracts the minimum value of the `x` coordinates from the supplied geometry. For geometries of type `geo_point` or `geo_shape`, this corresponds to the minimum `longitude` value. ## Syntax @@ -8,17 +8,15 @@ ST_XMIN retrieves the minimum 'x' coordinate from the provided geometry. ### Parameters -#### point +#### `point` -This is an expression of either `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape` type. If this parameter is null, the function will return null. - -## Explanation - -ST_XMIN function extracts the minimum value of the 'x' coordinates from the provided geometry data. In cases where the geometry is either of type `geo_point` or `geo_shape`, this is equivalent to extracting the minimum longitude value. +Expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If `null`, the function returns `null`. ## Examples -This example query returns the bounding envelope coordinates of Copenhagen Airport: +Extracting the bounding box coordinates of a city boundary + +The following query calculates the minimum and maximum `x` and `y` coordinates of the bounding box for the city boundary of the airport with the abbreviation "CPH": ```esql FROM airport_city_boundaries @@ -27,5 +25,3 @@ FROM airport_city_boundaries | EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) | KEEP abbrev, airport, xmin, xmax, ymin, ymax ``` - -In this query, the `ST_XMIN` function is used to extract the smallest 'x' value from the geometric 'envelope' surrounding the airport. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt index 1e918c4a1913a..08eac86b2bd14 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_y.txt @@ -1,6 +1,6 @@ # ST_Y -The ST_Y function extracts the `y` coordinate from a given point. +Extracts the `y` coordinate from the supplied point. For points of type `geo_point`, this corresponds to the `latitude` value. ## Syntax @@ -8,19 +8,15 @@ The ST_Y function extracts the `y` coordinate from a given point. ### Parameters -#### point +#### `point` -This is an expression of type `geo_point` or `cartesian_point`. +Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. ## Examples - ```esql ROW point = TO_GEOPOINT("POINT(42.97109629958868 14.7552534006536)") | EVAL x = ST_X(point), y = ST_Y(point) ``` -```esql -ROW point = TO_GEOPOINT("POINT(34.052235 -118.243683)") -| EVAL latitude = ST_Y(point) -``` +This example extracts the `x` (longitude) and `y` (latitude) coordinates from a `geo_point`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymax.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymax.txt index 560c76082eb1d..09a4dbab6c04d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymax.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymax.txt @@ -1,6 +1,6 @@ # ST_YMAX -Extracts the maximum value of the `y` coordinates from the given geometry input. +Extracts the maximum value of the `y` coordinates from the supplied geometry. For geometries of type `geo_point` or `geo_shape`, this corresponds to the maximum `latitude` value. ## Syntax @@ -8,13 +8,15 @@ Extracts the maximum value of the `y` coordinates from the given geometry input. ### Parameters -#### point +#### `point` -An expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If the value is `null`, the function also returns `null`. +Expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If `null`, the function returns `null`. ## Examples -Here is an example of using the `ST_YMAX` function: +Extracting the bounding box coordinates of a city boundary + +The following query calculates the bounding box of the city boundary for the airport with the abbreviation "CPH" and extracts the minimum and maximum `x` and `y` coordinates: ```esql FROM airport_city_boundaries @@ -23,9 +25,3 @@ FROM airport_city_boundaries | EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) | KEEP abbrev, airport, xmin, xmax, ymin, ymax ``` - -The example above first uses the `ST_ENVELOPE` function to find the smaller rectangular polygon that contains `city_boundary`. Then it uses the `ST_XMIN`, `ST_XMAX`, `ST_YMIN`, and `ST_YMAX` functions to calculate the minimum and maximum `x` and `y` coordinates of the rectangle, respectively. Lastly, it keeps only the columns of interest: `abbrev`, `airport`, `xmin`, `xmax`, `ymin`, and `ymax`. - -When the `point` parameter is of type `geo_point` or `geo_shape`, using the `ST_YMAX` function is equivalent to finding the maximum `latitude` value. - -Where applicable, if there are limitations impacting this function, they will be mentioned in a "Limitations" section at the end of this document. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymin.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymin.txt index 574c0c3855dfa..37107e16564cb 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymin.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-st_ymin.txt @@ -1,6 +1,6 @@ # ST_YMIN -The ST_YMIN function extracts the smallest value of the `y` coordinates from the provided geometry. +Extracts the minimum value of the `y` coordinates from the supplied geometry. For geometries of type `geo_point` or `geo_shape`, this corresponds to the minimum `latitude` value. ## Syntax @@ -8,13 +8,13 @@ The ST_YMIN function extracts the smallest value of the `y` coordinates from the ### Parameters -#### point +#### `point` -A given expression of types `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If the value is `null`, the function will also return `null`. +Expression of type `geo_point`, `geo_shape`, `cartesian_point`, or `cartesian_shape`. If `null`, the function returns `null`. ## Examples -This example demonstrates how to extract the minimum `y` coordinate from a geographical boundary outline: +Extracting the minimum `y` coordinate from a geometry ```esql FROM airport_city_boundaries @@ -24,4 +24,4 @@ FROM airport_city_boundaries | KEEP abbrev, airport, xmin, xmax, ymin, ymax ``` -In the case of `geo_point` or `geo_shape`, using the ST_YMIN function is equivalent to retrieving the minimum `latitude` value. +This example calculates the bounding box of a city boundary for the airport with the abbreviation "CPH" and extracts the minimum and maximum `x` and `y` coordinates, keeping only the relevant fields in the output. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt index bc19d7bf8d2f2..31b096518f349 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-starts_with.txt @@ -1,6 +1,6 @@ # STARTS_WITH -The STARTS_WITH function returns a boolean value indicating whether a keyword string begins with a specified string. +Determines whether a keyword string starts with a specified prefix and returns a boolean result. ## Syntax @@ -8,27 +8,20 @@ The STARTS_WITH function returns a boolean value indicating whether a keyword st ### Parameters -#### str +#### `str` -This is a string expression. +String expression. If `null`, the function returns `null`. -#### prefix +#### `prefix` -This is a string expression that will be checked if it is the starting sequence of the `str` parameter. +String expression. If `null`, the function returns `null`. ## Examples -This example checks if the last name of employee records starts with "B": - ```esql FROM employees | KEEP last_name | EVAL ln_S = STARTS_WITH(last_name, "B") ``` -```esql -FROM employees -| KEEP first_name, last_name -| EVAL fn_S = STARTS_WITH(first_name, "A") -| WHERE fn_S -``` +This example checks if the `last_name` column values start with the letter "B". \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt index 2ac2f8cc8f9d7..29ff6e3876841 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-stats.txt @@ -1,57 +1,32 @@ -# STATS ... BY +# STATS -The `STATS ... BY` command groups rows based on a common value and calculates one or more aggregated values over these grouped rows. +The `STATS` command groups rows based on a common value and calculates one or more aggregated values over the grouped rows. ## Syntax -```esql -STATS [column1 =] expression1[, ..., [columnN =] expressionN] [BY grouping_expression1[, ..., grouping_expressionN]] -``` +`STATS [column1 =] expression1 [WHERE boolean_expression1][, ..., [columnN =] expressionN [WHERE boolean_expressionN]] [BY grouping_expression1[, ..., grouping_expressionN]]` ### Parameters -#### columnX +#### `columnX` -The name for the aggregated value in the output. If not provided, the name defaults to the corresponding expression (`expressionX`). +The name by which the aggregated value is returned. If omitted, the name defaults to the corresponding expression (`expressionX`). If multiple columns have the same name, all but the rightmost column with this name are ignored. -#### expressionX +#### `expressionX` An expression that computes an aggregated value. -#### grouping_expressionX +#### `grouping_expressionX` An expression that outputs the values to group by. If its name coincides with one of the computed columns, that column will be ignored. -## Description - -The `STATS ... BY` command groups rows based on a common value and calculates one or more aggregated values over these grouped rows. -If `BY` is omitted, the output table contains exactly one row with the aggregations applied over the entire dataset. - -The following aggregation functions are supported: +#### `boolean_expressionX` -- `AVG` -- `COUNT` -- `COUNT_DISTINCT` -- `MAX` -- `MEDIAN` -- `MEDIAN_ABSOLUTE_DEVIATION` -- `MIN` -- `PERCENTILE` -- `ST_CENTROID_AGG` -- `STD_DEV` -- `SUM` -- `TOP` -- `VALUES` -- `WEIGHTED_AVG` - -The following grouping functions are supported: - -- `BUCKET` -- `CATEGORIZE` +The condition that must be met for a row to be included in the evaluation of `expressionX`. ## Examples -Calculate a statistic and group by the values of another column: +### Calculating a statistic and grouping by the values of another column ```esql FROM employees @@ -59,50 +34,87 @@ FROM employees | SORT languages ``` -Omitting `BY` returns one row with the aggregations applied over the entire dataset: +Group rows by the `languages` column and calculate the count of `emp_no` for each group. + +### Omitting `BY` to return one row with the aggregations applied over the entire dataset ```esql FROM employees | STATS avg_lang = AVG(languages) ``` -It’s possible to calculate multiple values: +Calculate the average number of languages across all rows. + +### Calculating multiple values ```esql FROM employees | STATS avg_lang = AVG(languages), max_lang = MAX(languages) ``` -If the grouping key is multivalued then the input row is in all groups: +Calculate both the average and maximum number of languages. + +### Filtering rows that go into an aggregation using `WHERE` + +```esql +FROM employees +| STATS avg50s = AVG(salary)::LONG WHERE birth_date < "1960-01-01" +``` + +Group rows by `gender` and calculate the average salary for employees born before and after 1960. + +### Mixing aggregations with and without filters, and optional grouping + +```esql +FROM employees +| EVAL Ks = salary / 1000 // thousands +| STATS under_40K = COUNT(*) WHERE Ks < 40, inbetween = COUNT(*) WHERE Ks >= 40 and Ks <= 60, over_60K = COUNT(*) WHERE Ks > 60, total = COUNT(*) +``` + +Calculate counts for salary ranges (under 40K, between 40K and 60K, over 60K) and the total count. + +### Grouping by a multivalued key ```esql -ROW i=1, a=["a", "b"] | STATS MIN(i) BY a | SORT a ASC +ROW i=1, a=["a", "b"] +| STATS MIN(i) BY a +| SORT a ASC ``` -It’s also possible to group by multiple values: +Group rows by the multivalued column `a` and calculate the minimum value of `i` for each group. + +### Grouping by multiple values ```esql FROM employees -| EVAL hired = DATE_FORMAT("YYYY", hire_date) +| EVAL hired = DATE_FORMAT("yyyy", hire_date) | STATS avg_salary = AVG(salary) BY hired, languages.long | EVAL avg_salary = ROUND(avg_salary) | SORT hired, languages.long ``` -If all grouping keys are multivalued then the input row is in all groups: +Group rows by the year of hire and the `languages.long` column, then calculate and round the average salary for each group. + +### Grouping by multiple multivalued keys ```esql -ROW i=1, a=["a", "b"], b=[2, 3] | STATS MIN(i) BY a, b | SORT a ASC, b ASC +ROW i=1, a=["a", "b"], b=[2, 3] +| STATS MIN(i) BY a, b +| SORT a ASC, b ASC ``` -Both the aggregating functions and the grouping expressions accept other functions. This is useful for using `STATS...BY` on multivalue columns. +Group rows by the multivalued columns `a` and `b` and calculate the minimum value of `i` for each group. + +### Using functions in aggregating and grouping expressions ```esql FROM employees | STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10) ``` -An example of grouping by an expression is grouping employees on the first letter of their last name: +Calculate the average salary change using the `MV_AVG` function and round the result to 10 decimal places. + +### Grouping by an expression ```esql FROM employees @@ -110,14 +122,18 @@ FROM employees | SORT `LEFT(last_name, 1)` ``` -Specifying the output column name is optional. If not specified, the new column name is equal to the expression. The following query returns a column named `AVG(salary)`: +Group rows by the first letter of the `last_name` column and calculate the count for each group. + +### Specifying the output column name (optional) ```esql FROM employees | STATS AVG(salary) ``` -Because this name contains special characters, it needs to be quoted with backticks (`) when using it in subsequent commands: +Calculate the average salary. The output column name defaults to `AVG(salary)`. + +### Using quoted column names in subsequent commands ```esql FROM employees @@ -125,16 +141,11 @@ FROM employees | EVAL avg_salary_rounded = ROUND(`AVG(salary)`) ``` -STATS works with grouping functions such as BUCKET, e.g. grouping data based on their timestamp: - -```esql -FROM sample_data -| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW() -| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 1 hour) -``` +Use the calculated column `AVG(salary)` in subsequent commands by quoting its name. -## Notes +## Limitations -- If multiple columns share the same name, all but the rightmost column with this name are ignored. -- `STATS` without any groups is much faster than adding a group. Grouping on a single expression is more optimized than grouping on multiple expressions. -- If the grouping key is multivalued, the input row is included in all groups. +- Individual `null` values are skipped when computing aggregations. +- `STATS` without any groups is faster than adding a group. +- Grouping on a single expression is more optimized than grouping on multiple expressions. For example, grouping on a single `keyword` column is significantly faster than grouping on two `keyword` columns. +- Avoid combining columns with functions like `CONCAT` for grouping, as it does not improve performance. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-std_dev.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-std_dev.txt index f3063438a4997..6950518d29a4e 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-std_dev.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-std_dev.txt @@ -1,6 +1,6 @@ # STD_DEV -The STD_DEV function calculates the standard deviation of a numeric field. +Calculates the standard deviation of a numeric field. ## Syntax @@ -14,16 +14,20 @@ A numeric field for which the standard deviation is calculated. ## Examples -This example calculates the standard deviation of the 'height' column: +Calculate the standard deviation of a field ```esql FROM employees | STATS STD_DEV(height) ``` -In this example, we first calculate the maximum salary change for each employee using the `MV_MAX` function. The `STD_DEV` function is then used to calculate the standard deviation of these maximum salary changes: +Calculate the standard deviation of the `height` field. + +Use inline functions with STD_DEV ```esql FROM employees -| STATS std_dev_salary_change = STD_DEV(MV_MAX(salary_change)) +| STATS stddev_salary_change = STD_DEV(MV_MAX(salary_change)) ``` + +Calculate the standard deviation of each employee’s maximum salary changes by first applying the `MV_MAX` function to determine the maximum salary change per row, and then using `STD_DEV` on the result. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt index a47ec9546047a..e76ce4936c122 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-substring.txt @@ -1,29 +1,28 @@ # SUBSTRING -The SUBSTRING function extracts a portion of a string, as specified by a starting position and an optional length. +The `SUBSTRING` function extracts a portion of a string based on a specified start position and an optional length. ## Syntax -`SUBSTRING(string, start, [length])` +`SUBSTRING(string, start, length)` ### Parameters -#### string +#### `string` -The string expression from which to extract the substring. +The string expression to extract the substring from. If `null`, the function returns `null`. -#### start +#### `start` -The starting position for the substring extraction. +The starting position for the substring. A negative value is interpreted as being relative to the end of the string. -#### length +#### `length` (Optional) -The length of the substring to be extracted from the starting position. -This parameter is optional. If it's omitted, the function will return all positions following the start position. +The length of the substring to extract, starting from the `start` position. If omitted, the function returns all characters from the `start` position to the end of the string. ## Examples -The following example returns the first three characters of every last name: +Extract the first three characters of every last name ```esql FROM employees @@ -31,7 +30,9 @@ FROM employees | EVAL ln_sub = SUBSTRING(last_name, 1, 3) ``` -A negative start position is interpreted as being relative to the end of the string. This example returns the last three characters of every last name: +This example extracts the first three characters from the `last_name` column. + +Extract the last three characters of every last name ```esql FROM employees @@ -39,10 +40,18 @@ FROM employees | EVAL ln_sub = SUBSTRING(last_name, -3, 3) ``` -If the length parameter is omitted, the SUBSTRING function returns the remainder of the string. This example returns all characters except for the first: +This example extracts the last three characters from the `last_name` column by using a negative start position. + +Extract all characters except for the first ```esql FROM employees | KEEP last_name | EVAL ln_sub = SUBSTRING(last_name, 2) ``` + +This example extracts all characters from the `last_name` column starting from the second character, as the `length` parameter is omitted. + +## Limitations + +No specific limitations are mentioned for the `SUBSTRING` function. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt index 9e782699db9ba..8955e9a099eea 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-sum.txt @@ -1,6 +1,6 @@ # SUM -The SUM function calculates the total sum of a numeric expression. +The `SUM` function calculates the total of a numeric expression. ## Syntax @@ -8,22 +8,26 @@ The SUM function calculates the total sum of a numeric expression. ### Parameters -#### number +#### `number` -The numeric expression that you want to calculate the sum of. +A numeric expression to be summed. ## Examples -Calculate the sum of a numeric field: +#Summing a field ```esql FROM employees | STATS SUM(languages) ``` -The SUM function can be used with inline functions: +Calculate the total number of languages across all employees. + +#Using inline functions ```esql FROM employees | STATS total_salary_changes = SUM(MV_MAX(salary_change)) ``` + +Calculate the total of each employee’s maximum salary changes by applying the `MV_MAX` function to each row and summing the results. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt index 704a89bb44665..bbf8063498c39 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt @@ -1,172 +1,156 @@ -# ES|QL Syntax Guide +```markdown +# ES|QL Syntax Reference -This guide provides an overview of the ES|QL syntax and examples of its usage. +## Overview -## Basic Syntax +The Elasticsearch Query Language (ES|QL) provides a powerful and flexible way to query, filter, transform, and analyze data stored in Elasticsearch. ES|QL uses a piped syntax (`|`) to chain commands and functions, enabling users to compose complex queries in a step-by-step manner. Each query starts with a source command (e.g., `FROM`) and can be followed by one or more processing commands. -An ES|QL query is composed of a source command followed by an optional series of processing commands, separated by a pipe character (`|`). For example: +### Basic Syntax -```esql -source-command -| processing-command1 -| processing-command2 -``` - -The result of a query is the table produced by the final processing command. +An ES|QL query is composed of: +1. A **source command**: Retrieves data from indices, data streams, or aliases. +2. A series of **processing commands**: Transform or filter the data. -For readability, each processing command is typically written on a new line. However, an ES|QL query can also be written as a single line: +Commands are separated by the pipe character (`|`), and the result of one command is passed as input to the next. For example: ```esql -source-command | processing-command1 | processing-command2 +FROM employees +| WHERE height > 2 +| SORT height DESC ``` -## Identifiers +The result of the query is the table produced by the final processing command. -Identifiers in ES|QL need to be quoted with backticks (```) if they don’t start with a letter, `_` or `@`, or if any of the other characters is not a letter, number, or `_`. +### Identifiers -For example: +Identifiers must be quoted with backticks (`` ` ``) if: +- They don’t start with a letter, `_`, or `@`. +- They contain characters other than letters, numbers, or `_`. +For example: ```esql FROM index | KEEP `1.field` ``` -When referencing a function alias that itself uses a quoted identifier, the backticks of the quoted identifier need to be escaped with another backtick. - -For example: +### Literals +#### String Literals +String literals are enclosed in double quotes (`"`). If the string contains quotes, escape them with `\\` or use triple quotes (`"""`): ```esql -FROM index -| STATS COUNT(`1.field`) -| EVAL my_count = `COUNT(``1.field``)` +ROW name = """Indiana "Indy" Jones""" ``` -## Literals - -ES|QL currently supports numeric, string and timespan literals. +#### Numeric Literals +Numeric literals can be expressed in decimal or scientific notation: +```esql +ROW value1 = 1969, value2 = 3.14, value3 = 4E5 +``` -### String Literals +### Comments -A string literal is a sequence of unicode characters delimited by double quotes (`"`). For example: +ES|QL supports C++-style comments: +- Single-line comments: `//` +- Multi-line comments: `/* */` ```esql -FROM index -| WHERE first_name == "Georgi" +// Query the employees index +FROM employees +| WHERE height > 2 ``` -If the literal string itself contains quotes, these need to be escaped (`\\"`). ES|QL also supports the triple-quotes (`"""`) delimiter, for convenience: +### Timespan Literals +Timespan literals represent datetime intervals and are expressed as a combination of a number and a temporal unit (e.g., `1 day`, `24h`, `7 weeks`). They are not whitespace-sensitive: ```esql -ROW name = """Indiana "Indy" Jones""" +1day +1 day +1 day ``` -The special characters CR, LF and TAB can be provided with the usual escaping: `\r`, `\n`, `\t`, respectively. - -### Numerical Literals - -The numeric literals are accepted in decimal and in the scientific notation with the exponent marker (`e` or `E`), starting either with a digit, decimal point `.` or the negative sign `-`. - -For example: - -- `1969` -- integer notation -- `3.14` -- decimal notation -- `.1234` -- decimal notation starting with decimal point -- `4E5` -- scientific notation (with exponent marker) -- `1.2e-3` -- scientific notation with decimal point -- `-.1e2` -- scientific notation starting with the negative sign +Timespan literals can be used in various commands and functions, such as `WHERE`, `DATE_TRUNC`, and `BUCKET`. -The integer numeric literals are implicitly converted to the `integer`, `long` or the `double` type, whichever can first accommodate the literal’s value. +--- -## Timespan Literals +## Example Queries Using Timespan Literals -Datetime intervals and timespans can be expressed using timespan literals. Timespan literals are a combination of a number and a qualifier. +Below are five example queries showcasing the use of timespan literals in combination with different commands and functions. -These qualifiers are supported: - -- `millisecond`/`milliseconds`/`ms` -- `second`/`seconds`/`sec`/`s` -- `minute`/`minutes`/`min` -- `hour`/`hours`/`h` -- `day`/`days`/`d` -- `week`/`weeks`/`w` -- `month`/`months`/`mo` -- `quarter`/`quarters`/`q` -- `year`/`years`/`yr`/`y` - -Timespan literals are not whitespace sensitive, and should not be wrapped with quotes: -- GOOD: 1day -- GOOD: 1 day -- BAD: "day" -- BAD: "2 days" - -## Example Queries with Timespan Literals - -Here are some example queries that use timespan literals: - -1. Retrieve logs from the last 24 hours and calculate the average response time: +Filtering Logs from the Last 24 Hours +This query retrieves logs from the last 24 hours and calculates the total number of logs per hour. ```esql FROM logs-* -| WHERE @timestamp > NOW() - 24h -| STATS avg_response_time = AVG(response_time) +| WHERE @timestamp >= NOW() - 24h +| EVAL hour = DATE_TRUNC(1 hour, @timestamp) +| STATS log_count = COUNT(*) BY hour +| SORT hour ``` -2. Get the count of events per day for the last 7 days: +Grouping by Weekly Buckets +This query groups employee hire dates into weekly buckets for the last 7 weeks and calculates the number of hires per week. ```esql -FROM events -| WHERE @timestamp > NOW() - 7 days -| STATS daily_count = COUNT(*) BY day = DATE_TRUNC(1 day, @timestamp) -| SORT day +FROM employees +| WHERE hire_date >= NOW() - 7 weeks +| STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, 1 week) +| SORT week ``` -3. Find the maximum temperature recorded in the last month: +Calculating Monthly Averages +This query calculates the average salary of employees grouped by monthly buckets for the year 2023. ```esql -FROM weather_data -| WHERE @timestamp > NOW() - 1 month -| STATS max_temp = MAX(temperature) +FROM employees +| WHERE hire_date >= "2023-01-01T00:00:00Z" AND hire_date < "2024-01-01T00:00:00Z" +| EVAL month = DATE_TRUNC(1 month, hire_date) +| STATS avg_salary = AVG(salary) BY month +| SORT month ``` -4. Calculate the total sales for each week in the last quarter: +Creating Hourly Buckets for the Last Day +This query creates hourly buckets for the last 1 day and calculates the total number of events in each bucket. ```esql -FROM sales -| WHERE @timestamp > NOW() - 1 quarter -| STATS weekly_sales = SUM(sales_amount) BY week = BUCKET(@timestamp, 1 week) -| SORT week +FROM events +| WHERE @timestamp >= NOW() - 1 day +| STATS event_count = COUNT(*) BY hour = BUCKET(@timestamp, 1 hour) +| SORT hour ``` -5. Retrieve error logs from the last 15 minutes and group by error type: +Filtering and Aggregating by Custom Time Range +This query filters logs within a custom time range and calculates the maximum response time for each 6-hour interval. ```esql -FROM error_logs -| WHERE @timestamp > NOW() - 15 minutes -| STATS error_count = COUNT(*) BY error_type -| SORT error_count DESC +FROM logs-* +| WHERE @timestamp >= "2023-10-01T00:00:00Z" AND @timestamp < "2023-10-02T00:00:00Z" +| EVAL interval = DATE_TRUNC(6 hours, @timestamp) +| STATS max_response_time = MAX(response_time) BY interval +| SORT interval ``` -#### Comments +--- -ES|QL uses C++ style comments: -- Double slash `//` for single line comments -- `/*` and `*/` for block comments +## Key Features of ES|QL Syntax +### Named Parameters in Functions +Some functions, like `MATCH`, support named parameters for additional options: ```esql -// Query the employees index -FROM employees -| WHERE height > 2 +FROM library +| WHERE MATCH(author, "Frank Herbert", {"minimum_should_match": 2, "operator": "AND"}) +| LIMIT 5 ``` -```esql -FROM /* Query the employees index */ employees -| WHERE height > 2 -``` +### Supported Commands and Functions +ES|QL supports a wide range of commands and functions for filtering, transforming, and analyzing data. For example: +- **Commands**: `FROM`, `WHERE`, `SORT`, `STATS`, `EVAL`, `KEEP`, `DROP`, `LIMIT`, `BUCKET`, `DATE_TRUNC` +- **Functions**: `COUNT`, `AVG`, `MAX`, `MIN`, `DATE_EXTRACT`, `DATE_DIFF`, `CASE` -```esql -FROM employees -/* Query the - * employees - * index */ -| WHERE height > 2 +Refer to the [Commands](#commands) and [Functions](#functions) sections for detailed descriptions and examples. + +--- + +## Conclusion + +ES|QL provides a robust and intuitive syntax for querying and analyzing data in Elasticsearch. By leveraging its piped syntax, timespan literals, and rich set of commands and functions, users can perform complex data transformations and aggregations with ease. The examples above demonstrate the flexibility and power of ES|QL, making it a valuable tool for data exploration and analysis. ``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt index 56940c07ba0a4..059ea12dcbf12 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tan.txt @@ -1,6 +1,6 @@ -# TAN +## TAN -The TAN function calculates the tangent of a given angle. +The `TAN` function calculates the tangent of a given angle. ## Syntax @@ -8,18 +8,15 @@ The TAN function calculates the tangent of a given angle. ### Parameters -#### angle +#### `angle` -The angle for which the tangent is to be calculated. The angle should be in radians. If the angle is `null`, the function will return `null`. +An angle, in radians. If `null`, the function returns `null`. ## Examples -```esql -ROW a=1.8 -| EVAL tan=TAN(a) -``` +Calculate the tangent of the angle `1.8` radians: ```esql -ROW angle=0.5 -| EVAL tangent = TAN(angle) -``` +ROW a=1.8 +| EVAL tan = TAN(a) +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt index 311608f9afbac..0f623b83e8cfb 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tanh.txt @@ -1,16 +1,16 @@ # TANH -The TANH function calculates the hyperbolic tangent of a given angle. +Returns the hyperbolic tangent of a number. ## Syntax -`TANH(angle)` +`TANH(number)` ### Parameters -#### angle +#### `number` -This is the angle in radians for which you want to calculate the hyperbolic tangent. +Numeric expression. If `null`, the function returns `null`. ## Examples @@ -19,7 +19,4 @@ ROW a=1.8 | EVAL tanh = TANH(a) ``` -```esql -ROW angle=0.5 -| EVAL hyperbolic_tangent = TANH(angle) -``` +Calculate the hyperbolic tangent of the value `1.8`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt index c18de72c77c69..45d8c5cdebdaa 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-tau.txt @@ -1,6 +1,6 @@ # TAU -TAU function returns the mathematical constant τ (tau), which is the ratio of a circle's circumference to its radius. +Returns the ratio of a circle’s circumference to its radius. ## Syntax @@ -8,7 +8,7 @@ TAU function returns the mathematical constant τ (tau), which is the ratio of a ### Parameters -This function does not require any parameters. +This function does not take any parameters. ## Examples @@ -16,8 +16,4 @@ This function does not require any parameters. ROW TAU() ``` -```esql -FROM sample_data -| EVAL tau_value = TAU() -| KEEP tau_value -``` +This example returns the mathematical constant τ (tau). diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt index 2dee65a690118..1f02e8fb48210 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_boolean.txt @@ -1,6 +1,6 @@ # TO_BOOLEAN -The TO_BOOLEAN function converts an input value into a boolean value. +Converts an input value to a boolean value. A string value of `true` will be case-insensitively converted to the Boolean `true`. For anything else, including the empty string, the function will return `false`. The numerical value of `0` will be converted to `false`, and anything else will be converted to `true`. ## Syntax @@ -8,19 +8,22 @@ The TO_BOOLEAN function converts an input value into a boolean value. ### Parameters -#### field +#### `field` -The input value. This can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples -The following example demonstrates the use of the TO_BOOLEAN function: - ```esql ROW str = ["true", "TRuE", "false", "", "yes", "1"] | EVAL bool = TO_BOOLEAN(str) ``` +This example converts a multi-valued string column into boolean values. For instance: +- `"true"` and `"TRuE"` are converted to `true`. +- `"false"`, `""` (empty string), and other non-`true` strings are converted to `false`. +- `"1"` is converted to `true`. + ```esql ROW num = [0, 1, 2, -1] | EVAL bool = TO_BOOLEAN(num) diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt index edf1d2020882f..4e5f682fe4ac9 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianpoint.txt @@ -1,6 +1,6 @@ # TO_CARTESIANPOINT -The TO_CARTESIANPOINT function converts an input value into a `cartesian_point` value. +Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it adheres to the WKT Point format. ## Syntax @@ -8,20 +8,20 @@ The TO_CARTESIANPOINT function converts an input value into a `cartesian_point` ### Parameters -#### field +#### `field` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Convert WKT-formatted strings to `cartesian_point` values: + ```esql ROW wkt = ["POINT(4297.11 -1475.53)", "POINT(7580.93 2272.77)"] | MV_EXPAND wkt | EVAL pt = TO_CARTESIANPOINT(wkt) ``` -```esql -ROW wkt = ["POINT(1000.0 2000.0)", "POINT(3000.0 4000.0)"] -| MV_EXPAND wkt -| EVAL pt = TO_CARTESIANPOINT(wkt) -``` +## Limitations + +- The input string must strictly follow the WKT Point format for successful conversion. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt index 8fb5ad5be2579..fc19a6fb45a7d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_cartesianshape.txt @@ -1,6 +1,6 @@ # TO_CARTESIANSHAPE -The TO_CARTESIANSHAPE function converts an input value into a `cartesian_shape` value. +Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it adheres to the WKT (Well-Known Text) format. ## Syntax @@ -10,21 +10,19 @@ The TO_CARTESIANSHAPE function converts an input value into a `cartesian_shape` #### field -The input value. This can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Converting WKT strings to `cartesian_shape` + ```esql ROW wkt = ["POINT(4297.11 -1475.53)", "POLYGON ((3339584.72 1118889.97, 4452779.63 4865942.27, 2226389.81 4865942.27, 1113194.90 2273030.92, 3339584.72 1118889.97))"] | MV_EXPAND wkt | EVAL geom = TO_CARTESIANSHAPE(wkt) ``` -```esql -ROW wkt = ["POINT(1000.0 2000.0)", "POLYGON ((1000.0 2000.0, 2000.0 3000.0, 3000.0 4000.0, 1000.0 2000.0))"] -| MV_EXPAND wkt -| EVAL geom = TO_CARTESIANSHAPE(wkt) -``` +This example converts a multi-valued column containing WKT strings into `cartesian_shape` values. ## Notes diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_date_nanos.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_date_nanos.txt new file mode 100644 index 0000000000000..b4c2d8cdf21ee --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_date_nanos.txt @@ -0,0 +1,38 @@ +# TO_DATE_NANOS + +Converts an input to a nanosecond-resolution date value (`date_nanos`). + +## Syntax + +`TO_DATE_NANOS(field)` + +### Parameters + +#### `field` + +The input value to be converted. This can be a single- or multi-valued column or an expression. + +## Examples + +Converting a timestamp string to `date_nanos` + +```esql +ROW timestamp = "2023-10-26T12:34:56.123456789Z" +| EVAL nanos_date = TO_DATE_NANOS(timestamp) +``` + +This example converts a timestamp string into a `date_nanos` value. + +Handling values outside the `date_nanos` range + +```esql +ROW timestamp = "2500-01-01T00:00:00.000000000Z" +| EVAL nanos_date = TO_DATE_NANOS(timestamp) +``` + +This example attempts to convert a timestamp outside the valid `date_nanos` range. The result will be `null` with a warning. + +## Limitations + +- The valid range for `date_nanos` is from `1970-01-01T00:00:00.000000000Z` to `2262-04-11T23:47:16.854775807Z`. Values outside this range will result in `null` and trigger a warning. +- Integer values cannot be converted into `date_nanos` because the range of integer nanoseconds only spans approximately 2 seconds after the epoch. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_dateperiod.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_dateperiod.txt index 8375d683dd745..8bcb12953dd7d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_dateperiod.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_dateperiod.txt @@ -1,6 +1,6 @@ # TO_DATEPERIOD -The `TO_DATEPERIOD` function converts an input value into a `date_period` value. +Converts an input value into a `date_period` value. ## Syntax @@ -8,15 +8,15 @@ The `TO_DATEPERIOD` function converts an input value into a `date_period` value. ### Parameters -#### field +#### `field` The input value. This must be a valid constant date period expression. ## Examples -This example demonstrates the usage of the `TO_DATEPERIOD` function: - ```esql ROW x = "2024-01-01"::datetime -| EVAL y = x + "3 DAYS"::date_period, z = x - TO_DATEPERIOD("3 days"); -``` \ No newline at end of file +| EVAL y = x + "3 DAYS"::date_period, z = x - TO_DATEPERIOD(`3 days`) +``` + +This example demonstrates how to add and subtract a `date_period` value to/from a datetime field. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt index 5510b7d3d4692..5f2d4b6e3b360 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_datetime.txt @@ -1,6 +1,6 @@ # TO_DATETIME -The TO_DATETIME function converts an input value into a date value. +Converts an input value to a date value. A string will only be successfully converted if it follows the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. For other date formats, use the `DATE_PARSE` function. When converting from nanosecond resolution to millisecond resolution, the nanosecond date is truncated, not rounded. ## Syntax @@ -8,29 +8,43 @@ The TO_DATETIME function converts an input value into a date value. ### Parameters -#### field +#### `field` -The input value to be converted, either single or multi-valued column or an expression. -If of type string, the input must follow the `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'` format. To convert strings in other formats, use DATE_PARSE. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples -The following example converts a string into a date value: +Converting strings to datetime ```esql ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"] | EVAL datetime = TO_DATETIME(string) ``` -If the input parameter is of a numeric type, its value will be interpreted as milliseconds since the Unix epoch. For example: +In this example, the first two values in the `string` column are successfully converted to datetime values because they follow the required format. However, the last value does not match the format and is converted to `null`. When this happens, a **Warning** header is added to the response, providing details about the failure: + +``` +"Line 1:112: evaluation of [TO_DATETIME(string)] failed, treating result as null. "Only first 20 failures recorded."" +``` + +A subsequent header will include the failure reason and the problematic value: + +``` +"java.lang.IllegalArgumentException: failed to parse date field [1964-06-02 00:00:00] +with format [yyyy-MM-dd'T'HH:mm:ss.SSS'Z']" +``` + +Converting numeric values to datetime + +If the input is numeric, the value is interpreted as milliseconds since the [Unix epoch](https://en.wikipedia.org/wiki/Unix_time). For example: ```esql ROW int = [0, 1] | EVAL dt = TO_DATETIME(int) ``` -## Notes +In this example, the numeric values `0` and `1` are converted to datetime values representing the Unix epoch and one millisecond after the epoch, respectively. -- Can only convert string with the exact format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use the `DATE_PARSE` function. +## Notes -- When converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded. +- A string will only be successfully converted if it’s respecting the format yyyy-MM-dd'T'HH:mm:ss.SSS'Z'. To convert dates in other formats, use DATE_PARSE. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt index a6c75fb627f73..64b92f493c4c4 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_degrees.txt @@ -1,6 +1,6 @@ # TO_DEGREES -The TO_DEGREES function converts a numerical value from radians to degrees. +Converts a number in radians to degrees. ## Syntax @@ -8,18 +8,23 @@ The TO_DEGREES function converts a numerical value from radians to degrees. ### Parameters -#### number +#### `number` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. It can be a single- or multi-valued column or an expression. ## Examples +Convert a list of radian values to degrees: + ```esql ROW rad = [1.57, 3.14, 4.71] | EVAL deg = TO_DEGREES(rad) ``` -```esql -ROW angle_in_radians = 1.0 -| EVAL angle_in_degrees = TO_DEGREES(angle_in_radians) -``` +### Result + +| rad | deg | +|-------|-----------| +| 1.57 | 89.954373 | +| 3.14 | 179.908747| +| 4.71 | 269.86312 | \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt index f5f57af2a1ca0..081d062e5dd0a 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geopoint.txt @@ -1,6 +1,6 @@ # TO_GEOPOINT -The TO_GEOPOINT function converts an input value into a `geo_point` value. +Converts an input value to a `geo_point` value. A string will only be successfully converted if it adheres to the WKT Point format. ## Syntax @@ -8,18 +8,19 @@ The TO_GEOPOINT function converts an input value into a `geo_point` value. ### Parameters -#### field +#### `field` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Convert a WKT Point string to a `geo_point` value: + ```esql ROW wkt = "POINT(42.97109630194 14.7552534413725)" | EVAL pt = TO_GEOPOINT(wkt) ``` -```esql -ROW wkt = "POINT(34.052235 -118.243683)" -| EVAL pt = TO_GEOPOINT(wkt) -``` +## Limitations + +- The input string must strictly follow the WKT Point format for successful conversion. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt index 47f4baa3a8df1..c7ae44553a42e 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_geoshape.txt @@ -1,6 +1,6 @@ # TO_GEOSHAPE -The TO_GEOSHAPE function converts an input value into a `geo_shape` value. +Converts an input value to a `geo_shape` value. A string will only be successfully converted if it adheres to the WKT (Well-Known Text) format. ## Syntax @@ -8,9 +8,9 @@ The TO_GEOSHAPE function converts an input value into a `geo_shape` value. ### Parameters -#### field +#### `field` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples @@ -19,7 +19,4 @@ ROW wkt = "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" | EVAL geom = TO_GEOSHAPE(wkt) ``` -```esql -ROW wkt = "LINESTRING (30 10, 10 30, 40 40)" -| EVAL geom = TO_GEOSHAPE(wkt) -``` +This example converts a WKT string representing a polygon into a `geo_shape` value. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt index 9a4098ff61fa3..754599630cb6c 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_integer.txt @@ -1,6 +1,6 @@ # TO_INTEGER -The TO_INTEGER function converts an input value into an integer. +Converts an input value to an integer. If the input is a date type, it is interpreted as milliseconds since the Unix epoch and converted to an integer. Boolean values are converted to integers: `true` becomes `1` and `false` becomes `0`. ## Syntax @@ -8,21 +8,42 @@ The TO_INTEGER function converts an input value into an integer. ### Parameters -#### field +#### `field` -The input value. This can be a single or multi-valued column or an expression. - -## Description - -The TO_INTEGER function converts an input value into an integer. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Converting long values to integers + ```esql ROW long = [5013792, 2147483647, 501379200000] | EVAL int = TO_INTEGER(long) ``` -## Notes +| long | int | +|----------------|--------------| +| 5013792 | 5013792 | +| 2147483647 | 2147483647 | +| 501379200000 | null | + +In this example, the first two values are successfully converted to integers. However, the last value exceeds the range of an integer, resulting in a `null` value. When such a failure occurs, a warning is added to the response. + +### Warning Example + +If a value cannot be converted, the response includes a warning header with details about the failure: + +``` +"Line 1:61: evaluation of [TO_INTEGER(long)] failed, treating result as null. Only first 20 failures recorded." +``` + +Additionally, another header provides the failure reason and the problematic value: + +``` +"org.elasticsearch.xpack.esql.core.InvalidArgumentException: [501379200000] out of [integer] range" +``` + +## Limitations -- If the input parameter is of a date type, its value is interpreted as milliseconds since the Unix epoch and converted to an integer. A boolean value of true is converted to integer 1, and false is converted to 0. +- Values that exceed the range of an integer will result in a `null` value. +- A warning is generated when conversion fails, and only the first 20 failures are recorded in the response headers. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt index 3bffd3a4e7d0f..92fc3fff3a8f6 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_ip.txt @@ -1,6 +1,6 @@ # TO_IP -The TO_IP function converts an input string into an IP value. +Converts an input string to an IP value. ## Syntax @@ -8,21 +8,32 @@ The TO_IP function converts an input string into an IP value. ### Parameters -#### field +#### `field` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Converting strings to IP values + ```esql ROW str1 = "1.1.1.1", str2 = "foo" | EVAL ip1 = TO_IP(str1), ip2 = TO_IP(str2) | WHERE CIDR_MATCH(ip1, "1.0.0.0/8") ``` +In this example: +- The string `"1.1.1.1"` is successfully converted to an IP value and stored in `ip1`. +- The string `"foo"` cannot be converted to an IP value, resulting in a `null` value for `ip2`. + +When a conversion fails, a *Warning* header is added to the response. The header provides details about the failure, including the source of the issue and the offending value. For instance: + ```esql -ROW ip_str = "192.168.1.1" -| EVAL ip = TO_IP(ip_str) -| KEEP ip +"Line 1:68: evaluation of [TO_IP(str2)] failed, treating result as null. Only first 20 failures recorded." ``` +The failure reason and the problematic value are also included in a subsequent header: + +```esql +"java.lang.IllegalArgumentException: 'foo' is not an IP string literal." +``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt index ec844c50461f5..ed0340a2e1d96 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_long.txt @@ -1,6 +1,6 @@ # TO_LONG -The TO_LONG function converts an input value into a long value. +Converts an input value to a long value. If the input is of a date type, it is interpreted as milliseconds since the Unix epoch and converted to a long. Boolean values are converted as follows: `true` to `1` and `false` to `0`. ## Syntax @@ -8,22 +8,19 @@ The TO_LONG function converts an input value into a long value. ### Parameters -#### field +#### `field` -The input value. This can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples +Converting strings to long values + ```esql ROW str1 = "2147483648", str2 = "2147483648.2", str3 = "foo" | EVAL long1 = TO_LONG(str1), long2 = TO_LONG(str2), long3 = TO_LONG(str3) ``` -```esql -ROW str1 = "1234567890", str2 = "9876543210" -| EVAL long1 = TO_LONG(str1), long2 = TO_LONG(str2) -``` - -## Notes - -- If the input parameter is of a date type, its value is interpreted as milliseconds since the Unix epoch and converted to a long value. A boolean value of true is converted to a long value of 1, and false is converted to 0. +- `str1` is successfully converted to a long value. +- `str2` is also converted to a long value, truncating the decimal part. +- `str3` cannot be converted, resulting in a `null` value. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt index 9bc7d3caeab4a..0f3a545d9c396 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_lower.txt @@ -1,6 +1,6 @@ # TO_LOWER -The TO_LOWER function converts the input string to lowercase. +Returns a new string with all characters in the input string converted to lowercase. ## Syntax @@ -8,9 +8,9 @@ The TO_LOWER function converts the input string to lowercase. ### Parameters -#### str +#### `str` -The string expression that you want to convert to lowercase. +String expression. If `null`, the function returns `null`. ## Examples @@ -19,9 +19,4 @@ ROW message = "Some Text" | EVAL message_lower = TO_LOWER(message) ``` -```esql -FROM employees -| KEEP first_name, last_name -| EVAL first_name_lower = TO_LOWER(first_name) -| EVAL last_name_lower = TO_LOWER(last_name) -``` +This example converts the string in the `message` column to lowercase and stores the result in a new column named `message_lower`. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt index 6af4d409fe70d..830ba0cef6d86 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_radians.txt @@ -1,6 +1,6 @@ # TO_RADIANS -The TO_RADIANS function converts a numerical value from degrees to radians. +Converts a number in degrees to radians. ## Syntax @@ -8,18 +8,15 @@ The TO_RADIANS function converts a numerical value from degrees to radians. ### Parameters -#### number +#### `number` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. It can be a single- or multi-valued column or an expression. ## Examples +Convert a list of degree values to radians: + ```esql ROW deg = [90.0, 180.0, 270.0] | EVAL rad = TO_RADIANS(deg) -``` - -```esql -ROW angle_deg = 45.0 -| EVAL angle_rad = TO_RADIANS(angle_deg) -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt index 9fa0a489741fe..35734e3c99c2f 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_string.txt @@ -1,6 +1,6 @@ # TO_STRING -The TO_STRING function converts an input value into a string. +Converts an input value into a string. ## Syntax @@ -8,22 +8,22 @@ The TO_STRING function converts an input value into a string. ### Parameters -#### field +#### `field` -This is the input value. It can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples -Here is an example of how to use the TO_STRING function: +Converting a single value to a string ```esql ROW a=10 | EVAL j = TO_STRING(a) ``` -The TO_STRING function also works well on multi-valued fields: +Converting a multivalued field to a string ```esql ROW a=[10, 9, 8] | EVAL j = TO_STRING(a) -``` \ No newline at end of file +``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_timeduration.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_timeduration.txt index 3c95cf4f75cc6..50e4e1c71ad17 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_timeduration.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_timeduration.txt @@ -1,6 +1,6 @@ # TO_TIMEDURATION -The `TO_TIMEDURATION` function converts an input value into a `time_duration` value. +Converts an input value into a `time_duration` value. ## Syntax @@ -8,16 +8,17 @@ The `TO_TIMEDURATION` function converts an input value into a `time_duration` va ### Parameters -#### field +#### `field` -This is the input value. It must be a valid constant time duration expression. +The input value. Must be a valid constant time duration expression. ## Examples -Here's an example of how to use the `TO_TIMEDURATION` function: +Adding and subtracting time durations ```esql ROW x = "2024-01-01"::datetime -| EVAL y = x + "3 hours"::time_duration, z = x - TO_TIMEDURATION("3 hours"); +| EVAL y = x + "3 hours"::time_duration, z = x - TO_TIMEDURATION(`3 hours`) ``` -In this example, `TO_TIMEDURATION` function is used to convert the string "3 hours" into a `time_duration` value, which is then subtracted from the datetime value stored in the variable `x`. + +This example demonstrates how to add and subtract a time duration (`3 hours`) to and from a datetime value (`2024-01-01`). diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt index b4783943c4d98..6a58bf44d3dca 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_upper.txt @@ -1,6 +1,6 @@ # TO_UPPER -The TO_UPPER function converts the input string to uppercase. +Converts the input string to upper case and returns the result. ## Syntax @@ -8,20 +8,15 @@ The TO_UPPER function converts the input string to uppercase. ### Parameters -#### str +#### `str` -The string expression that you want to convert to uppercase. +String expression. If `null`, the function returns `null`. ## Examples +Convert a string to upper case: + ```esql ROW message = "Some Text" | EVAL message_upper = TO_UPPER(message) -``` - -```esql -FROM employees -| KEEP first_name, last_name -| EVAL first_name_upper = TO_UPPER(first_name) -| EVAL last_name_upper = TO_UPPER(last_name) -``` +``` \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt index 447a1752622a1..965b9bd07d26e 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-to_version.txt @@ -1,6 +1,6 @@ # TO_VERSION -The TO_VERSION function converts an input string into a version value. +Converts an input string to a version value. ## Syntax @@ -8,9 +8,9 @@ The TO_VERSION function converts an input string into a version value. ### Parameters -#### field +#### `field` -The input value to be converted. This can be a single or multi-valued column or an expression. +The input value to be converted. This can be a single- or multi-valued column or an expression. ## Examples @@ -18,7 +18,4 @@ The input value to be converted. This can be a single or multi-valued column or ROW v = TO_VERSION("1.2.3") ``` -```esql -ROW version_string = "2.3.4" -| EVAL version = TO_VERSION(version_string) -``` +Convert the string `"1.2.3"` into a version value. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt index b22fb7c9b54d1..39cb12ca5baf5 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-top.txt @@ -1,6 +1,6 @@ -# TOP +## TOP -The TOP function collects the top values for a specified field. +The `TOP` function collects the top values for a specified field, including repeated values. ## Syntax @@ -8,30 +8,23 @@ The TOP function collects the top values for a specified field. ### Parameters -#### field +#### `field` -The field for which the top values are to be collected. +The field to collect the top values for. -#### limit +#### `limit` -The maximum number of values to be collected. +The maximum number of values to collect. -#### order +#### `order` -The order in which the top values are calculated. It can be either `asc` (ascending) or `desc` (descending). +The order in which to calculate the top values. Can be either `asc` (ascending) or `desc` (descending). ## Examples -Collect the top 3 salaries from the employees data: - +Collecting top salaries ```esql FROM employees | STATS top_salaries = TOP(salary, 3, "desc"), top_salary = MAX(salary) ``` - -Collect the top 5 products in ascending order: - -```esql -FROM sales -| STATS top_products = TOP(product_id, 5, "asc"), max_sales = MAX(sales_amount) -``` +This example collects the top three salaries in descending order and calculates the maximum salary. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt index ad4aa621a009a..8211a67a99764 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-trim.txt @@ -1,6 +1,6 @@ # TRIM -The TRIM function removes leading and trailing whitespaces from a string. +Removes leading and trailing whitespaces from a string. ## Syntax @@ -8,20 +8,16 @@ The TRIM function removes leading and trailing whitespaces from a string. ### Parameters -#### string +#### `string` -This is the string expression that you want to trim. +String expression. If `null`, the function returns `null`. ## Examples ```esql -ROW message = " some text ", color = " red " +ROW message = " some text ", color = " red " | EVAL message = TRIM(message) | EVAL color = TRIM(color) ``` -```esql -ROW text = " example text ", label = " label " -| EVAL text = TRIM(text) -| EVAL label = TRIM(label) -``` +This example removes leading and trailing whitespaces from the `message` and `color` columns. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt index 10c8021ce96d6..24a0116733aec 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-values.txt @@ -1,24 +1,21 @@ # VALUES -The VALUES function returns all values in a group as a multivalued field. +The `VALUES` function retrieves all values in a group as a multivalued field. The order of the returned values is not guaranteed. To ensure the values are returned in order, use the `MV_SORT` function. ## Syntax -`VALUES (field)` +`VALUES(field)` ### Parameters -#### field +#### `field` -The field for which all values are to be returned. - -## Description - -The VALUES function is used to return all values in a group as a multivalued field. It's important to note that the order of the returned values is not guaranteed. If you need the values returned in a specific order, you should use the `MV_SORT` function. +The field from which to retrieve all values. ## Examples -The following example demonstrates how to use the VALUES function: +Retrieve and sort first names by their first letter +The following query extracts the first letter of each employee's first name, groups the data by this letter, and retrieves all first names in each group as a multivalued field. The `MV_SORT` function is used to sort the names within each group. ```esql FROM employees @@ -27,6 +24,7 @@ FROM employees | SORT first_letter ``` -## Limitations +## Notes -- The VALUES function can consume a significant amount of memory. ES|QL does not currently support growing aggregations beyond memory. Therefore, if the function collects more values than can fit into memory, it will fail the query with a Circuit Breaker Error. +- This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. +- The `VALUES` function can consume a significant amount of memory. ES|QL does not currently support growing aggregations beyond available memory. If the aggregation collects more values than can fit into memory, the query will fail with a Circuit Breaker Error. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt index 611edc73cb983..1ed6aa3dc65b1 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-weighted_avg.txt @@ -1,6 +1,6 @@ -# WEIGHTED_AVG +## WEIGHTED_AVG -The WEIGHTED_AVG function calculates the weighted average of a numeric expression. +Calculates the weighted average of a numeric expression. ## Syntax @@ -8,13 +8,13 @@ The WEIGHTED_AVG function calculates the weighted average of a numeric expressio ### Parameters -#### number +#### `number` -A numeric value that you want to calculate the weighted average for. +A numeric value. -#### weight +#### `weight` -A numeric value that represents the weight of the corresponding number. +A numeric weight. ## Examples @@ -26,10 +26,4 @@ FROM employees | SORT languages ``` -```esql -FROM sales -| STATS weighted_sales = WEIGHTED_AVG(revenue, units_sold) BY region -| EVAL weighted_sales = ROUND(weighted_sales, 2) -| KEEP weighted_sales, region -| SORT region -``` +This example calculates the weighted average of employee salaries using their heights as weights, grouped by languages. The result is rounded and sorted by language. \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt index ccd7e12517ffb..cbe2d7befb1cc 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt @@ -1,6 +1,6 @@ -# WHERE +## WHERE -The WHERE command filters the rows in a table based on a specified condition, returning only those rows where the condition evaluates to `true`. +The `WHERE` command filters rows from the input table, returning only those for which the specified condition evaluates to `true`. ## Syntax @@ -8,33 +8,23 @@ The WHERE command filters the rows in a table based on a specified condition, re ### Parameters -#### expression +#### `expression` -A boolean expression that defines the condition for filtering the rows. +A boolean expression that determines which rows are included in the output. -## Notes - -WHERE supports the following types of functions: -- Mathematical functions -- String functions -- Date-time functions -- Type conversation functions -- Conditional functions and expressions -- Multi-value functions -- Operators +## Examples -Aggregation functions are WHERE supported for EVAL. +### Basic Usage -## Examples +Filter rows where the `still_hired` field is `true`: -#### Example 1: Filtering Based on Boolean Condition ```esql FROM employees | KEEP first_name, last_name, still_hired | WHERE still_hired == true ``` -If `still_hired` is a boolean field, this can be simplified to: +If `still_hired` is a boolean field, the query can be simplified to: ```esql FROM employees @@ -42,20 +32,29 @@ FROM employees | WHERE still_hired ``` -#### Example 2: Retrieving Data from a Specific Time Range +### Using Date Math + +Retrieve rows from the last hour of logs: + ```esql FROM sample_data | WHERE @timestamp > NOW() - 1 hour ``` -#### Example 3: Using Functions in WHERE Clause +### Using Functions + +Filter rows where the length of the `first_name` field is less than 4: + ```esql FROM employees | KEEP first_name, last_name, height | WHERE LENGTH(first_name) < 4 ``` -#### Example 4: NULL Comparison +### NULL Comparison + +Filter rows where the `birth_date` field is `NULL`: + ```esql FROM employees | WHERE birth_date IS NULL @@ -64,31 +63,71 @@ FROM employees | LIMIT 3 ``` +Filter rows where the `is_rehired` field is not `NULL`: + ```esql FROM employees | WHERE is_rehired IS NOT NULL | STATS COUNT(emp_no) ``` -#### Example 5: Filtering Based on String Patterns Using Wildcards +### Using LIKE for String Patterns + +Filter rows based on string patterns using wildcards. The following wildcard characters are supported: +- `*` matches zero or more characters. +- `?` matches one character. + +Filter rows where `first_name` matches the pattern `?b*`: + ```esql FROM employees -| WHERE first_name LIKE "?b*" +| WHERE first_name LIKE """?b*""" | KEEP first_name, last_name ``` -#### Example 6: Filtering Based on String Patterns Using Regular Expressions +To match the exact characters `*` or `.`, escape them using a backslash (`\\`). For example: + +```esql +ROW message = "foo * bar" +| WHERE message LIKE "foo \\* bar" +``` + +To simplify escaping, use triple-quoted strings: + +```esql +ROW message = "foo * bar" +| WHERE message LIKE """foo \* bar""" +``` + +### Using RLIKE for Regular Expressions + +Filter rows based on regular expressions. For example, filter rows where `first_name` matches the pattern `.leja.*`: + ```esql FROM employees -| WHERE first_name RLIKE ".leja.*" +| WHERE first_name RLIKE """.leja.*""" | KEEP first_name, last_name ``` -#### Example 7: Using IN Operator +Escape special characters in regular expressions using a backslash (`\\`). For example: + ```esql -ROW a = 1, b = 4, c = 3 -| WHERE c-a IN (3, b / 2, a) +ROW message = "foo ( bar" +| WHERE message RLIKE "foo \\( bar" ``` -### Limitations -- The `WHERE` command is subject to the maximum number of rows limitation (10,000). +To simplify escaping, use triple-quoted strings: + +```esql +ROW message = "foo ( bar" +| WHERE message RLIKE """foo \( bar""" +``` + +### Using IN Operator + +Filter rows where an expression matches any value in a list of literals, fields, or expressions: + +```esql +ROW a = 1, b = 4, c = 3 +| WHERE c-a IN (3, b / 2, a) +``` \ No newline at end of file