diff --git a/x-pack/platform/plugins/shared/inference/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/platform/plugins/shared/inference/scripts/evaluation/scenarios/esql/index.spec.ts index cd116d62150b2..4be37659074dc 100644 --- a/x-pack/platform/plugins/shared/inference/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/platform/plugins/shared/inference/scripts/evaluation/scenarios/esql/index.spec.ts @@ -134,16 +134,22 @@ async function evaluateEsqlQuery({ const docBase = await EsqlDocumentBase.load(); + const prompts = docBase.getPrompts(); + const languageDescription = `${prompts.syntax} + + ${prompts.examples} + `; + const usedCommands = await retrieveUsedCommands({ question, answer, - esqlDescription: docBase.getSystemMessage(), + esqlDescription: languageDescription, }); const requestedDocumentation = docBase.getDocumentation(usedCommands, { generateMissingKeywordDoc: false, }); - requestedDocumentation.commands_and_functions = docBase.getSystemMessage(); + requestedDocumentation.commands_and_functions = languageDescription; const evaluation = await evaluationClient.evaluate({ input: ` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/generate_esql.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/generate_esql.ts index 12d0c41510600..e12627c046151 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/generate_esql.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/generate_esql.ts @@ -29,6 +29,7 @@ import { INLINE_ESQL_QUERY_REGEX } from '../../../../common/tasks/nl_to_esql/con import type { EsqlDocumentBase } from '../doc_base'; import { requestDocumentationSchema } from './shared'; import type { NlToEsqlTaskEvent } from '../types'; +import { generateEsqlPrompt } from './prompts'; const MAX_CALLS = 5; @@ -44,14 +45,13 @@ type LlmEsqlTask = ( interface GenerateEsqlTaskOptions extends Pick { connectorId: string; - systemMessage: string; messages: Message[]; chatCompleteApi: ChatCompleteAPI; docBase: EsqlDocumentBase; logger: Pick; metadata?: ChatCompleteMetadata; - system?: string; maxCallsAllowed?: number; + additionalSystemInstructions?: string; } export function generateEsqlTask( @@ -63,7 +63,7 @@ export function generateEsqlTask( export function generateEsqlTask({ chatCompleteApi, connectorId, - systemMessage, + additionalSystemInstructions, messages, toolOptions: { tools, toolChoice }, docBase, @@ -71,7 +71,6 @@ export function generateEsqlTask({ maxRetries, retryConfiguration, logger, - system, metadata, maxCallsAllowed = MAX_CALLS, }: GenerateEsqlTaskOptions & { @@ -108,32 +107,11 @@ export function generateEsqlTask({ retryConfiguration, metadata, stream: true, - system: `${systemMessage} - - # Current task - - Your current task is to respond to the user's question. If there is a tool - suitable for answering the user's question, use that tool, preferably - with a natural language reply included. - - Format any ES|QL query as follows: - \`\`\`esql - - \`\`\` - - When generating ES|QL, it is VERY important that you only use commands and functions present in the - requested documentation, and follow the syntax as described in the documentation and its examples. - Assume that ONLY the set of capabilities described in the provided ES|QL documentation is valid, and - do not try to guess parameters or syntax based on other query languages. - - If what the user is asking for is not technically achievable with ES|QL's capabilities, just inform - the user. DO NOT invent capabilities not described in the documentation just to provide - a positive answer to the user. E.g. Pagination is not supported by the language, do not try to invent - workarounds based on other languages. - - When converting queries from one language to ES|QL, make sure that the functions are available - and documented in ES|QL. E.g., for SPL's LEN, use LENGTH. For IF, use CASE. - ${system ? `## Additional instructions\n\n${system}` : ''}`, + system: generateEsqlPrompt({ + esqlPrompts: docBase.getPrompts(), + additionalSystemInstructions, + hasTools: !functionLimitReached && Object.keys(tools ?? {}).length > 0, + }), messages: [ ...messages, { diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/prompts.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/prompts.ts new file mode 100644 index 0000000000000..b98e89d8c6a67 --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/prompts.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EsqlPrompts } from '../doc_base/load_data'; + +export const requestDocumentationSystemPrompt = ({ esqlPrompts }: { esqlPrompts: EsqlPrompts }) => { + return `You are an assistant that helps with writing ESQL query for Elasticsearch. + +Your current task is to examine the previous conversation, and to request documentation +from the ES|QL handbook to help you get the right information needed to generate a query. + +Below are the ES|QL syntax and some examples from the official ES|QL documentation. + +${esqlPrompts.syntax} + +${esqlPrompts.examples}`; +}; + +export const generateEsqlPrompt = ({ + esqlPrompts, + additionalSystemInstructions, + hasTools = false, +}: { + esqlPrompts: EsqlPrompts; + additionalSystemInstructions?: string; + hasTools?: boolean; +}) => { + const hasToolBlock = hasTools + ? `**IMPORTANT**: If there is a tool suitable for answering the user's question, use that tool, +preferably with a natural language reply included.` + : undefined; + + return `You are an assistant that helps with writing ESQL query for Elasticsearch. +Given a natural language query, you will generate an ESQL query that can be executed against the data source. + +# Current task + +Your current task is to respond to the user's question. + +${hasToolBlock} + +## Documentation + +${esqlPrompts.syntax} + +${esqlPrompts.examples} + +${esqlPrompts.instructions} + +${ + additionalSystemInstructions + ? `\n${additionalSystemInstructions}\n` + : '' +} + +Take your time and think step by step about the natural language query and how to convert it into ESQL. + +Format any ES|QL query as follows: + \`\`\`esql + + \`\`\` + +${hasToolBlock}`; +}; diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/request_documentation.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/request_documentation.ts index 8ea2c3c453ae2..e17068adde446 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/request_documentation.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/actions/request_documentation.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { isEmpty } from 'lodash'; import type { ToolOptions, Message, @@ -13,30 +12,30 @@ import type { ChatCompleteOptions, OutputAPI, } from '@kbn/inference-common'; -import { ToolChoiceType, withoutOutputUpdateEvents } from '@kbn/inference-common'; +import { withoutOutputUpdateEvents } from '@kbn/inference-common'; +import type { EsqlPrompts } from '../doc_base/load_data'; +import { requestDocumentationSystemPrompt } from './prompts'; import { requestDocumentationSchema } from './shared'; export const requestDocumentation = ({ outputApi, - system, + esqlPrompts, messages, connectorId, functionCalling, maxRetries, retryConfiguration, metadata, - toolOptions: { tools, toolChoice }, + toolOptions, }: { outputApi: OutputAPI; - system: string; messages: Message[]; + esqlPrompts: EsqlPrompts; connectorId: string; metadata?: ChatCompleteMetadata; toolOptions: ToolOptions; } & Pick) => { - const hasTools = !isEmpty(tools) && toolChoice !== ToolChoiceType.none; - return outputApi({ id: 'request_documentation', connectorId, @@ -45,9 +44,9 @@ export const requestDocumentation = ({ maxRetries, retryConfiguration, metadata, - system, + system: requestDocumentationSystemPrompt({ esqlPrompts }), previousMessages: messages, - input: `Based on the previous conversation, request documentation + input: `Now, based on the previous conversation, request documentation from the ES|QL handbook to help you get the right information needed to generate a query. @@ -55,22 +54,7 @@ export const requestDocumentation = ({ - Do you need to group data? Request \`STATS\`. - Extract data? Request \`DISSECT\` AND \`GROK\`. - Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. - - ${ - hasTools - ? `### Tools - - The following tools will be available to be called in the step after this. - - \`\`\`json - ${JSON.stringify({ - tools, - toolChoice, - })} - \`\`\`` - : '' - } - `, +`, schema: requestDocumentationSchema, }).pipe(withoutOutputUpdateEvents()); }; diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/aliases.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/aliases.ts index d797815012640..9c9a6c26c7a7d 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/aliases.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/aliases.ts @@ -12,7 +12,8 @@ const aliases: Record = { STATS: ['STATS_BY', 'BY', 'STATS...BY', 'STATS ... BY'], OPERATORS: ['LIKE', 'RLIKE', 'IN'], - LOOKUP_JOIN: ['LOOKUPJOIN'], + LOOKUP_JOIN: ['JOIN', 'LOOKUPJOIN', 'LOOKUP JOIN'], + FROM: ['METADATA'], }; const getAliasMap = () => { diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/esql_doc_base.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/esql_doc_base.ts index 4ce8517878da9..b95018b3d6afa 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/esql_doc_base.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/esql_doc_base.ts @@ -6,17 +6,15 @@ */ import { once } from 'lodash'; -import { loadData, type EsqlDocData, type EsqlDocEntry } from './load_data'; +import { loadData, type EsqlDocData, type EsqlDocEntry, type EsqlPrompts } from './load_data'; import { tryResolveAlias } from './aliases'; import { getSuggestions } from './suggestions'; import type { GetDocsOptions } from './types'; const loadDataOnce = once(loadData); -const overviewEntries = ['SYNTAX', 'OVERVIEW', 'OPERATORS']; - export class EsqlDocumentBase { - private systemMessage: string; + private prompts: EsqlPrompts; private docRecords: Record; static async load(): Promise { @@ -25,12 +23,20 @@ export class EsqlDocumentBase { } constructor(rawData: EsqlDocData) { - this.systemMessage = rawData.systemMessage; + this.prompts = rawData.prompts; this.docRecords = rawData.docs; } - getSystemMessage() { - return this.systemMessage; + getPrompts(): EsqlPrompts { + return this.prompts; + } + + /** @deprecated use individual prompts instead */ + getSystemMessage(): string { + return `${this.prompts.syntax} + + ${this.prompts.examples} + `; } getDocumentation( @@ -38,7 +44,6 @@ export class EsqlDocumentBase { { generateMissingKeywordDoc = true, addSuggestions = true, - addOverview = true, resolveAliases = true, }: GetDocsOptions = {} ) { @@ -54,10 +59,6 @@ export class EsqlDocumentBase { keywords.push(...getSuggestions(keywords)); } - if (addOverview) { - keywords.push(...overviewEntries); - } - return [...new Set(keywords)].reduce>((results, keyword) => { if (Object.hasOwn(this.docRecords, keyword)) { results[keyword] = this.docRecords[keyword].data; diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/load_data.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/load_data.ts index 340f06fd0fced..a8487181c6cb9 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/load_data.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/doc_base/load_data.ts @@ -15,21 +15,37 @@ export interface EsqlDocEntry { data: string; } +export interface EsqlPrompts { + syntax: string; + instructions: string; + examples: string; +} + export interface EsqlDocData { - systemMessage: string; + prompts: EsqlPrompts; docs: Record; } export const loadData = async (): Promise => { - const [systemMessage, docs] = await Promise.all([loadSystemMessage(), loadEsqlDocs()]); + const [prompts, docs] = await Promise.all([loadPrompts(), loadEsqlDocs()]); return { - systemMessage, + prompts, docs, }; }; -const loadSystemMessage = async () => { - return (await readFile(Path.join(__dirname, '../system_message.txt'))).toString('utf-8'); +const loadPrompt = async (fileName: string) => { + return (await readFile(Path.join(__dirname, `../prompts/${fileName}`))).toString('utf-8'); +}; + +const loadPrompts = async () => { + return Promise.all([ + loadPrompt('examples.txt'), + loadPrompt('instructions.txt'), + loadPrompt('syntax.txt'), + ]).then(([examples, instructions, syntax]) => { + return { examples, instructions, syntax }; + }); }; const loadEsqlDocs = async (): Promise> => { diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt index f2351aec5d203..e0d71fcf2bb39 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-from.txt @@ -26,14 +26,6 @@ Retrieve all documents from the `employees` index: FROM employees ``` -### Using Date Math - -Access indices, aliases, or data streams using date math. For example, retrieve today’s index for time series data: - -```esql -FROM -``` - ### Querying Multiple Data Streams, Indices, or Aliases Query multiple data streams, indices, or aliases using a comma-separated list or wildcards: @@ -55,26 +47,5 @@ FROM cluster_one:employees-00001,cluster_two:other-employees-* Enable metadata fields by using the optional `METADATA` directive: ```esql -FROM employees METADATA _id -``` - -### Escaping Index Names - -Escape index names containing special characters using double quotes (`"`) or triple double quotes (`"""`): - -```esql -FROM "this=that","""this[that""" -``` - -## Limitations - -- By default, the `FROM` command applies an implicit limit of 1,000 rows if no explicit `LIMIT` is specified. For example: - ```esql -FROM employees -``` - is equivalent to: - ```esql -FROM employees -| LIMIT 1000 +FROM employees METADATA _id, _score ``` -- Queries cannot return more than 10,000 rows, even if a higher `LIMIT` is specified. This is a configurable upper limit. For more details, refer to the [LIMIT command](#LIMIT). \ No newline at end of file diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt index f8acaf8f4ec34..758fa7df786a5 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-limit.txt @@ -16,55 +16,12 @@ The maximum number of rows to return. The `LIMIT` command restricts the number of rows returned by a query. For example: -```esql -FROM index -| WHERE field == "value" -``` - -is equivalent to: - ```esql FROM index | WHERE field == "value" | LIMIT 1000 ``` -Queries cannot return more than 10,000 rows, regardless of the value specified in the `LIMIT` command. This is a configurable upper limit. - -### Overcoming the 10,000 Row Limit - -To address this limitation: -- Modify the query to reduce the result set size by using `WHERE` to filter the data. -- Perform post-query processing within the query itself. For example, use the `STATS` command to aggregate data. - -The 10,000-row limit applies only to the number of rows returned by the query, not to the number of documents processed. The query operates on the full dataset. - -Consider these examples: - -```esql -FROM index -| WHERE field0 == "value" -| LIMIT 20000 -``` - -and - -```esql -FROM index -| STATS AVG(field1) BY field2 -| LIMIT 20000 -``` - -In both cases, the filtering by `field0` in the first query or the grouping by `field2` in the second query is applied to all documents in the `index`. However, the output is capped at 10,000 rows, even if more rows are available. - -### Configuring Limits - -The default and maximum limits can be adjusted using the following dynamic cluster settings: -- `esql.query.result_truncation_default_size` -- `esql.query.result_truncation_max_size` - -Increasing these limits may lead to higher memory usage, longer processing times, and increased internode traffic within and across clusters. - ## Examples Limit the result to the first 5 rows, sorted by `emp_no` in ascending order: @@ -79,9 +36,7 @@ FROM employees - Queries cannot return more than 10,000 rows, even if the `LIMIT` value exceeds this threshold. - To work around this limitation: - +To work around this limitation: - Reduce the size of the result set by modifying the query to only return relevant data. This can be achieved by using the WHERE command to select a smaller subset of the data. - Shift any post-query processing to the query itself. The ES|QL STATS ... BY command can be used to aggregate data within the query. -- Adjusting the default or maximum limits can impact performance and resource usage. diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt index c49309ecd2b4a..53a5a98f9f8fd 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/esql_docs/esql-match.txt @@ -45,3 +45,15 @@ FROM books ``` This example searches for books where the `title` field matches "Hobbit Back Again" using the `AND` operator, and keeps only the `title` field in the results. + +Match with sorting on score + +Match on a specific field + +```esql +FROM books METADATA _score +| WHERE MATCH(author, "Faulkner") +| SORT _score DESC +| KEEP book_no, author +| LIMIT 5 +``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/examples.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/examples.txt new file mode 100644 index 0000000000000..6ca0c410b947a --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/examples.txt @@ -0,0 +1,246 @@ + + + + What are the 10 latest errors from the logs? + + + FROM logs + | WHERE level == "ERROR" + | SORT @timestamp DESC + | LIMIT 10 + + + + + + What are the titles and descriptions of the blog articles published last month? + + + FROM blogposts + | WHERE published > NOW() - 1 month + | KEEP title, description + | SORT title + | LIMIT 100 + + + + + + How many employees are from the Netherlands? + + + FROM employees + | WHERE country == "NL" + | STATS COUNT(*) + + + + + + How many orders were placed each month over the last year? + + + FROM orders + | WHERE order_date > NOW() - 1 year + | STATS count = COUNT(*) BY date_bucket = BUCKET(order_date, 1 month) + + + + + + Extract the date, message and IP address from the logs using DISSECT + + + FROM postgres-logs* + | DISSECT message "%{date} - %{msg} - %{ip}" + | KEEP date, msg, ip + | EVAL date = DATE_PARSE("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", date) + | LIMIT 100 + + + + + + Find contributors which first name starts with "b", sort them by number of commits and then returns their first and last names for the top 5 + + + FROM commits + | WHERE TO_LOWER(first_name) LIKE "b*" + | STATS doc_count = COUNT(*) by first_name, last_name + | SORT doc_count DESC + | KEEP first_name, last_name + | LIMIT 5 + + + + + + What is the average salary of employees hired in 1985 split in 20 buckets? + + + FROM employees + | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" + | STATS avg_salary = AVG(salary) BY date_bucket = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") + | SORT bucket + + + + + + How many employees were hired in 1985 and what is their salary distribution? + + + FROM employees + | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" + | STATS c = COUNT(1) BY b = BUCKET(salary, 5000.) + | SORT b + + + + + + How many employees were hired after 2023-01-01 and what is the recent hiring rate by country? + + + FROM employees + | EVAL is_recent_hire = CASE(hire_date <= "2023-01-01T00:00:00Z", 1, 0) + | STATS total_recent_hires = SUM(is_recent_hire), total_hires = COUNT(*) BY country + | EVAL recent_hiring_rate = total_recent_hires / total_hires + + + + + + Calculate the failure rate per host over the past 24 hours + + + FROM logs-* + | WHERE @timestamp <= NOW() - 24 hours + | EVAL is_5xx = CASE(http.response.status_code >= 500, 1, 0) + | STATS total_events = COUNT(*), total_failures = SUM(is_5xx) BY host.hostname, bucket = BUCKET(@timestamp, 1 hour) + | EVAL failure_rate_per_host = total_failures / total_events + | DROP total_events, total_failures + + + + + + How many logs are there for each log level in the last 24 hours? + + + FROM logs-* + | WHERE @timestamp <= NOW() - 24 hours + | STATS count = COUNT(*) BY log.level + | SORT count DESC + + + + + + What is the min, max and average value of the numbers in the bag? + + + FROM bag_of_numbers + | EVAL min = MV_MIN(numbers), max = MV_MAX(numbers), avg = MV_AVG(numbers) + | KEEP bad_id, min, max, avg + + + + + + What messages, that do not contain the word "error", are in the logs? + + + FROM logs + | WHERE message NOT LIKE "*error*" + | KEEP message + | LIMIT 100 + + + + + + Find first 10 users with names starting with "A" or "B", sorted by name + + + FROM users + | KEEP name + | WHERE name LIKE "A*" OR name LIKE "B*" + | SORT name + | LIMIT 10 + + + + + + Get user names and birth dates, sorted by birth date + + + FROM personal_info + | EVAL birth=DATE_PARSE("yyyy-MM-dd", birth_date) + | KEEP user_name, birth + | SORT birth + | LIMIT 100 + + + + + + How many unique IP addresses are there from France, Germany and Spain? + + + FROM users + | WHERE country IN (France, Germany, Spain) + | STATS COUNT_DISTINCT(user_ip) + + + + + + How many warning logs are there from the US? + + + FROM logs + | WHERE message LIKE "*warning*" AND location RLIKE ".*us.*" + | STATS COUNT(*) + + + + + + Find books by authors whose names sound like "Frank Herbert" + + + FROM library + | WHERE MATCH(author, "Frank Herbert") + | KEEP title, author + | LIMIT 100 + + + + + + Find the latest articles from the "hr" category, and return their titles and document ids + + + FROM articles METADATA _id + | WHERE category == "hr" + | KEEP _id, title + | SORT publish_date DESC + | LIMIT 100 + + + + + + Find products with a description matching "wireless headphones" and sort them by relevance + + + FROM products METADATA _score + | WHERE MATCH(description, "wireless headphones") + | SORT _score DESC + | KEEP name, description, _score + | LIMIT 100 + + + diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/instructions.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/instructions.txt new file mode 100644 index 0000000000000..076b7fb1677c1 --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/instructions.txt @@ -0,0 +1,57 @@ + + + ## Follow the syntax + + It is CRUCIAL and MANDATORY to only use commands and functions which are present in the syntax definition, + and to follow the syntax as described in the documentation and its examples. Do not try to guess + new functions or commands based on other query languages. Assume that ONLY the set of capabilities described + in the provided ES|QL documentation is valid, and do not try to guess parameters or syntax based + on other query languages. + + ## Respect the mappings or field definitions + + If the user, or a tool, provides in the discussion the mappings or a list of fields present in the index, you should **ONLY** use + the provided fields to create your query. Do not assume other fields may exist. Only use the set of fields + which were provided by the user. + + ## Use a safety LIMIT + + 1. **LIMIT is Mandatory:** All multi-row queries **must** end with a `LIMIT`. The only exception is for single-row aggregations (e.g., `STATS` without a `GROUP BY`). + + 2. **Applying Limits:** + * **User-Specified:** If the user provides a number ("top 10", "get 50"), use it for the `LIMIT`. + * **Default:** If no number is given, default to `LIMIT 100` for both raw events and `GROUP BY` results. Notify the user when you apply this default (e.g., "I've added a `LIMIT 100` for safety."). + + 3. **Handling "All Data" Requests:** If a user asks for "all" results, apply a safety `LIMIT 250` and state that this limit was added to protect the system. + + ## Don't use tech preview features unless specified otherwise + + Using tech preview commands, functions or other features should be avoided unless specifically asked by the user. + + ## Use MATCH for full text search + + Unless specified otherwise, full text searches should always be done using MATCH in favor of other search functions. + + ## ES|QL query formatting + + - All generated ES|QL queries must be wrapped with ```esql and ``` + - Queries must be properly formatted, with a carriage return after each function + + Example: + ``` + FROM logs-* + | WHERE @timestamp <= NOW() - 24 hours + | STATS count = COUNT(*) BY log.level + | SORT count DESC + ``` + + ## Do not invent things to please the user + + If what the user is asking for is not technically achievable with ES|QL's capabilities, just inform + the user. DO NOT invent capabilities not described in the documentation just to provide + a positive answer to the user. + + When converting queries from one language to ES|QL, make sure that the functions are available + and documented in ES|QL. E.g., for SPL's LEN, use LENGTH. For IF, use CASE. + + diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/syntax.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/syntax.txt new file mode 100644 index 0000000000000..271e37015d95f --- /dev/null +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/prompts/syntax.txt @@ -0,0 +1,291 @@ + + The Elasticsearch Query Language (ES|QL) provides a powerful and flexible way to query, + filter, transform, and analyze data stored in Elasticsearch. + + An ES|QL query is composed of a source command followed by a series + of processing commands, separated by a pipe character: |. + + For example: + ```esql + [source-command] + | [processing-command1] + | [processing-command2] + ``` + + + Source commands select a data source. + + - FROM: Selects one or multiple indices, data streams or aliases to use as source. + - ROW: Produces a row with one or more columns with values that you specify. + - SHOW: returns information about the deployment. + + + + ES|QL processing commands change an input table by adding, removing, or changing rows and columns. + + - DISSECT: extracts structured data out of a string, using a dissect pattern + - DROP: drops one or more columns + - ENRICH: adds data from existing indices as new columns + - EVAL: adds a new column with calculated values, using various type of functions + - GROK: extracts structured data out of a string, using a grok pattern + - KEEP: keeps one or more columns, drop the ones that are not kept + - LIMIT: returns the first n number of rows. The maximum value for this is 10000 + - MV_EXPAND: expands multi-value columns into a single row per value + - RENAME: renames a column + - STATS ... BY: groups rows according to a common value and calculates one or more aggregated values over the grouped rows, using aggregation and grouping functions. + - SORT: sorts the row in a table by a column. Expressions are not supported. + - WHERE: Filters rows based on a boolean condition. WHERE supports the same functions as EVAL. + - [preview] RERANK: uses an inference model to compute a new relevance score for an initial set of documents + - [preview] SAMPLE: samples a fraction of the table rows + - [preview] COMPLETION: send prompts and context to an LLM + - [preview] CHANGE_POINT: detects spikes, dips, and change points in a metric. + + + + BUCKET: Create groups of values out of a datetime or numeric input + CATEGORIZE: Organize textual data into groups of similar format + + + + AVG: calculate the average of a numeric field + COUNT: return the total number of input values + COUNT_DISTINCT: return the number of distinct values in a field + MAX: calculate the maximum value of a field + MEDIAN: calculate the median value of a numeric field + MEDIAN_ABSOLUTE_DEVIATION: calculate the median absolute deviation of a numeric field + MIN: calculate the minimum value of a field + PERCENTILE: calculate a specified percentile of a numeric field + STD_DEV: calculate the standard deviation of a numeric field + SUM: calculate the total sum of a numeric expression + TOP: collect the top values for a specified field + VALUES: return all values in a group as a multivalued field + WEIGHTED_AVG: calculate the weighted average of a numeric expression + + + + CASE: accept pairs of conditions and values and return the value for the first true condition + COALESCE: return the first non-null argument from the list of provided arguments + GREATEST: return the maximum value from multiple columns + LEAST: return the smallest value from multiple columns + + + + MATCH: execute a match query on a specified field - equivalent to match query for Elasticsearch Query DSL + QSTR: perform a Lucene query string query + KQL: perform a KQL query + + + + DATE_DIFF: calculate the difference between two timestamps in a given unit + DATE_EXTRACT: extract a specific part of a date + DATE_FORMAT: return a string representation of a date using the provided format + DATE_PARSE: convert a date string into a date + DATE_TRUNC: round down a date to the nearest specified interval + NOW: return the current date and time + + + + BIT_LENGTH: calculate the bit length of a string + BYTE_LENGTH: calculate the byte length of a string + CONCAT: combine two or more strings + ENDS_WITH: check if a given string ends with a specified suffix + FROM_BASE64: decode a base64 string + HASH: compute the hash of a given input using a specified algorithm + LEFT: extract a specified number of characters from the start of a string + LENGTH: calculate the character length of a given string + LOCATE: return the position of a specified substring within a string + LTRIM: remove leading whitespaces from a string + REPEAT: generate a string by repeating a specified string a certain number of times + REPLACE: substitute any match of a regular expression within a string with a replacement string + REVERSE: reverse a string + RIGHT: extract a specified number of characters from the end of a string + RTRIM: remove trailing whitespaces from a string + SPACE: create a string composed of a specific number of spaces + SPLIT: split a single valued string into multiple strings based on a delimiter + STARTS_WITH: check if a given string begins with another specified string + SUBSTRING: extract a portion of a string + TO_BASE64: encode a string to b64 + TO_LOWER: convert a string to lowercase + TO_UPPER: convert a string to uppercase + TRIM: remove leading and trailing whitespaces from a string + + + + CIDR_MATCH: checks if an IP address falls within specified network blocks + IP_PREFIX: truncates an IP address to a specified prefix length + + + + TO_BOOLEAN + TO_CARTESIANPOINT + TO_CARTESIANSHAPE + TO_DATETIME (prefer DATE_PARSE to convert strings to datetime) + TO_DATEPERIOD + TO_DEGREES + TO_DOUBLE + TO_GEOPOINT + TO_GEOSHAPE + TO_INTEGER + TO_IP + TO_LONG + TO_RADIANS + TO_STRING + TO_TIMEDURATION + TO_UNSIGNED_LONG + TO_VERSION + + + + ABS + ACOS + ASIN + ATAN + ATAN2 + CBRT + CEIL + COS + COSH + E + EXP + FLOOR + HYPOT + LOG + LOG10 + PI + POW + ROUND + SIGNUM + SIN + SINH + SQRT + TAN + TANH + TAU + + + + Multivalue function are used to manipulate and transform multi-value fields. + + MV_APPEND: concatenates the values of two multi-value fields + MV_AVG: returns the average of all values in a multivalued field + MV_CONCAT: transforms a multivalued string expression into a single valued string + MV_COUNT: counts the total number of values in a multivalued expression + MV_DEDUPE: eliminates duplicate values from a multivalued field + MV_FIRST: returns the first value of a multivalued field + MV_LAST: returns the last value of a multivalued field + MV_MAX: returns the max value of a multivalued field + MV_MEDIAN: returns the median value of a multivalued field + MV_MEDIAN_ABSOLUTE_DEVIATION: returns the median absolute deviation of a multivalued field + MV_MIN: returns the min value of a multivalued field + MV_PERCENTILE: returns the specified percentile of a multivalued field + MV_SLIDE: extract a subset of a multivalued field using specified start and end index values + MV_SORT: sorts a multivalued field in lexicographical order. + MV_SUM: returns the sum of all values of a multivalued field + MV_ZIP: combines the values from two multivalued fields with a specified delimiter + + + + ST_CONTAINS: check if the first specified geometry encompasses the second one + ST_DISJOINT: check if two geometries or geometry columns are disjoint + ST_DISTANCE: calculate the distance between two points + ST_ENVELOPE: calculate the minimum bounding box for the provided geometry + ST_INTERSECTS: check if two geometries intersect + ST_WITHIN: check if the first geometry is located within the second geometry + ST_X/ST_Y: extract the x/y coordinate from a given point + ST_XMAX/ST_YMAX: extract the maximum value of the x/y coordinates from a geometry + ST_XMIN/ST_YMIN: extract the minimum value of the x/y coordinates from a geometry + ST_EXTENT_AGG: calculate the spatial extent over a field that has a geometry type + ST_CENTROID_AGG: calculate the spatial centroid over a spatial point geometry field + + + + Binary operators: ==, !=, <, <=, >, >=, +, -, *, /, % + Logical operators: AND, OR, NOT + Predicates: IS NULL, IS NOT NULL + Unary operators: - + IN: test if a field or expression is in a list of literals + LIKE: filter data based on string patterns using wildcards + RLIKE: filter data based on string patterns using regular expressions + Cast (`::`): provides a convenient alternative syntax to the `TO_` conversion functions + + + + + ### Identifiers + + Identifiers must be quoted with backticks (`` ` ``) if: + - They don’t start with a letter, `_`, or `@`. + - They contain characters other than letters, numbers, or `_`. + + For example: + ```esql + FROM index + | KEEP `1.field` + ``` + + ### String Literals + + String literals are enclosed in double quotes (`"`). + If the string contains quotes, escape them with `\\` or use triple quotes (`"""`): + + For example: + ```esql + ROW name = """Indiana "Indy" Jones""" + ``` + + ### Comments + + ES|QL uses C++ style comments: + - double slash // for single line comments + - /* and */ for block comments + + ### timespan literals + + Timespan literals represent datetime intervals and are expressed as a combination of a number and a temporal unit (e.g., `1 day`, `24h`, `7 weeks`). They are not whitespace-sensitive: + ```esql + 1day + 1 day + 1 day + ``` + + Supported temporal units: + + | Temporal Units | Valid Abbreviations | + |---|---| + | year | y, yr, years | + | quarter | q, quarters | + | month | mo, months | + | week | w, weeks | + | day | d, days | + | hour | h, hours | + | minute | min, minutes | + | second | s, sec, seconds | + | millisecond | ms, milliseconds | + + Example of using temporal units: + + ```esql + FROM events + | WHERE @timestamp >= NOW() - 1 day + | STATS event_count = COUNT(*) BY hour = BUCKET(@timestamp, 1 hour) + | SORT hour + ``` + + ### Named Parameters in Functions + + Some functions, like `MATCH`, support named parameters for additional options: + + ```esql + FROM library + | WHERE MATCH(author, "Frank Herbert", {"minimum_should_match": 2, "operator": "AND"}) + | LIMIT 5 + ``` + + + + - ES|QL currently does not support pagination + - A query will never return more than 10000 rows + - Some field types, such as `binary`, `nested`, and `histogram`, are not yet supported. + + + diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/system_message.txt b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/system_message.txt deleted file mode 100644 index f3262ff80eef6..0000000000000 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/system_message.txt +++ /dev/null @@ -1,381 +0,0 @@ -# Limitations - -- ES|QL currently does not support pagination. -- A query will never return more than 10000 rows. - -# Syntax - -An ES|QL query is composed of a source command followed by a series -of processing commands, separated by a pipe character: |. - -For example: - - | - | - -## Source commands - -Source commands select a data source. - -There are three source commands: -- FROM: Selects one or multiple indices, data streams or aliases to use as source. -- ROW: Produces a row with one or more columns with values that you specify. -- SHOW: returns information about the deployment. - -## Processing commands - -ES|QL processing commands change an input table by adding, removing, or -changing rows and columns. - -The following processing commands are available: - -[preview] CHANGE_POINT -- [preview] COMPLETION: send prompts and context to an LLM -- DISSECT: extracts structured data out of a string, using a dissect pattern -- DROP: drops one or more columns -- ENRICH: adds data from existing indices as new columns -- EVAL: adds a new column with calculated values, using various type of functions -- [preview] FORK: executes multiple branches of a query and combines them into a single output table -- GROK: extracts structured data out of a string, using a grok pattern -- KEEP: keeps one or more columns, drop the ones that are not kept -- LIMIT: returns the first n number of rows. The maximum value for this is 10000 -- LOOKUP JOIN: Joins data from a query results table with matching records from a specified lookup index -- [preview] MV_EXPAND: expands multi-value columns into a single row per value -- RENAME: renames a column -- [preview] RERANK: uses an inference model to compute a new relevance score for an initial set of documents -- [preview] SAMPLE: samples a fraction of the table rows -- STATS ... BY: groups rows according to a common value and calculates - one or more aggregated values over the grouped rows. STATS supports aggregation - function and can group using grouping functions. -- SORT: sorts the row in a table by a column. Expressions are not supported. -- WHERE: Filters rows based on a boolean condition. WHERE supports the same functions as EVAL. - -## Functions and operators - -### Join functions - -### Grouping functions - -BUCKET: Creates groups of values out of a datetime or numeric input -CATEGORIZE: Organize textual data into groups of similar format - -### Aggregation functions - -AVG: calculates the average of a numeric field -COUNT: returns the total number of input values -COUNT_DISTINCT: return the number of distinct values in a field -MAX: calculates the maximum value of a field -MEDIAN: calculates the median value of a numeric field -MEDIAN_ABSOLUTE_DEVIATION: calculates the median absolute deviation of a numeric field -MIN: calculates the minimum value of a field -PERCENTILE: calculates a specified percentile of a numeric field -STD_DEV: calculates the standard deviation of a numeric field -SUM: calculates the total sum of a numeric expression -TOP: collects the top values for a specified field -[preview] VALUES: returns all values in a group as a multivalued field -WEIGHTED_AVG: calculates the weighted average of a numeric expression - -### Conditional functions and expressions - -Conditional functions return one of their arguments by evaluating in an if-else manner - -CASE: accepts pairs of conditions and values and returns the value that belongs to the first condition that evaluates to true -COALESCE: returns the first non-null argument from the list of provided arguments -GREATEST: returns the maximum value from multiple columns -LEAST: returns the smallest value from multiple columns - -### Search functions - -Search functions perform full-text search against the data - -MATCH: execute a match query on a specified field (tech preview) -QSTR: performs a Lucene query string query (tech preview) - -### Date-time functions - -DATE_DIFF: calculates the difference between two timestamps in a given unit -DATE_EXTRACT: extract a specific part of a date -DATE_FORMAT: returns a string representation of a date using the provided format -DATE_PARSE: convert a date string into a date -DATE_TRUNC: rounds down a date to the nearest specified interval -NOW: returns the current date and time - -### Mathematical functions - -ABS: returns the absolute value of a number -ACOS: returns the arccosine of a number -ASIN: returns the arcsine of a number -ATAN: returns the arctangent of a number -ATAN2: returns the angle from the positive x-axis to a point (x, y) -CBRT: calculates the cube root of a given number -CEIL: rounds a number up to the nearest integer -COS: returns the cosine of a given angle -COSH: returns the hyperbolic cosine of a given angle -E: returns Euler's number -EXP: returns the value of Euler's number raised to the power of a given number -FLOOR: rounds a number down to the nearest integer -HYPOT: calculate the hypotenuse of two numbers -LOG: calculates the logarithm of a given value to a specified base -LOG10: calculates the logarithm of a value to base 10 -PI: returns the mathematical constant Pi -POW: calculates the value of a base raised to the power of an exponent -ROUND: rounds a numeric value to a specified number of decimal -SIGNUM: returns the sign of a given number -SIN: calculates the sine of a given angle -SINH: calculates the hyperbolic sine of a given angle -SQRT: calculates the square root of a given number -TAN: calculates the tangent of a given angle -TANH: calculates the hyperbolic tangent of a given angle -TAU: returns the mathematical constant τ (tau) - -### String functions - -BIT_LENGTH: calculates the bit length of a string -BYTE_LENGTH: calculates the byte length of a string -CONCAT: combines two or more strings into one -ENDS_WITH: checks if a given string ends with a specified suffix -FROM_BASE64: decodes a base64 string -HASH: computes the hash of a given input using a specified algorithm -LEFT: extracts a specified number of characters from the start of a string -LENGTH: calculates the character length of a given string -LOCATE: returns the position of a specified substring within a string -LTRIM: remove leading whitespaces from a string -REPEAT: generates a string by repeating a specified string a certain number of times -REPLACE: substitutes any match of a regular expression within a string with a replacement string -REVERSE: reverses a string -RIGHT: extracts a specified number of characters from the end of a string -RTRIM: remove trailing whitespaces from a string -SPACE: creates a string composed of a specific number of spaces -SPLIT: split a single valued string into multiple strings based on a delimiter -STARTS_WITH: checks if a given string begins with another specified string -SUBSTRING: extracts a portion of a string -TO_BASE64: encodes a string to a base64 -TO_LOWER: converts a string to lowercase -TO_UPPER: converts a string to uppercase -TRIM: removes leading and trailing whitespaces from a string - -### IP Functions - -CIDR_MATCH: checks if an IP address falls within specified network blocks -IP_PREFIX: truncates an IP address to a specified prefix length - -### Type conversion functions - -TO_BOOLEAN -TO_CARTESIANPOINT -TO_CARTESIANSHAPE -TO_DATETIME (prefer DATE_PARSE to convert strings to datetime) -TO_DATEPERIOD -TO_DEGREES -TO_DOUBLE -TO_GEOPOINT -TO_GEOSHAPE -TO_INTEGER -TO_IP -TO_LONG -TO_RADIANS -TO_STRING -TO_TIMEDURATION -TO_UNSIGNED_LONG -TO_VERSION - -### Search functions -KQL -MATCH -MATCH_PHRASE -QSTR - -### Multivalue functions - -Multivalue function are used to manipulate and transform multi-value fields. - -MV_APPEND: concatenates the values of two multi-value fields -MV_AVG: returns the average of all values in a multivalued field -MV_CONCAT: transforms a multivalued string expression into a single valued string -MV_COUNT: counts the total number of values in a multivalued expression -MV_DEDUPE: eliminates duplicate values from a multivalued field -MV_FIRST: returns the first value of a multivalued field -MV_LAST: returns the last value of a multivalued field -MV_MAX: returns the max value of a multivalued field -MV_MEDIAN: returns the median value of a multivalued field -MV_MEDIAN_ABSOLUTE_DEVIATION: returns the median absolute deviation of a multivalued field -MV_MIN: returns the min value of a multivalued field -MV_PERCENTILE: returns the specified percentile of a multivalued field -MV_SLICE: extract a subset of a multivalued field using specified start and end index values -MV_SORT: sorts a multivalued field in lexicographical order. -MV_SUM: returns the sum of all values of a multivalued field -MV_ZIP: combines the values from two multivalued fields with a specified delimiter - -### Spacial functions - -ST_CONTAINS: checks if the first specified geometry encompasses the second one -ST_DISJOINT: checks if two geometries or geometry columns are disjoint -ST_DISTANCE: calculates the distance between two points -ST_ENVELOPE: calculates the minimum bounding box for the provided geometry -ST_INTERSECTS: checks if two geometries intersect -ST_WITHIN: checks if the first geometry is located within the second geometry -ST_X: extracts the x coordinate from a given point -ST_XMAX: extracts the maximum value of the x coordinates from a geometry -ST_XMIN: extracts the minimum value of the x coordinates from a geometry -ST_Y: extracts the y coordinate from a given point -ST_YMAX: extracts the maximum value of the y coordinates from a geometry -ST_YMIN: extracts the minimum value of the y coordinates from a geometry - -### Spacial aggregations functions - -ST_EXTENT_AGG: calculates the spatial extent over a field that has a geometry type -ST_CENTROID_AGG: calculates the spatial centroid over a spatial point geometry field - -### Operators - -Binary operators: ==, !=, <, <=, >, >=, +, -, *, /, % -Logical operators: AND, OR, NOT -Predicates: IS NULL, IS NOT NULL -Unary operators: - -IN: test if a field or expression is in a list of literals -LIKE: filter data based on string patterns using wildcards -RLIKE: filter data based on string patterns using regular expressions -Cast (`::`): provides a convenient alternative syntax to the `TO_` conversion functions - -# Usage examples - -Here are some examples of ES|QL queries: - -**Returns the 10 latest errors from the logs** -```esql -FROM logs -| WHERE level == "ERROR" -| SORT @timestamp DESC -| LIMIT 10 -``` - -**Returns the title and description of last month's blog articles** -```esql -FROM blogposts -| WHERE published > NOW() - 1 month -| KEEP title, description -| SORT title -``` - -**Returns the number of employees from the "NL" country using STATS** -```esql -FROM employees -| WHERE country == "NL" -| STATS COUNT(*) -``` - -**Returns the number of order for each month over last year** -```esql -FROM orders -| WHERE order_date > NOW() - 1 year -| STATS count = COUNT(*) BY date_bucket = BUCKET(order_date, 1 month) -``` - -**Extracting structured data from logs using DISSECT** -```esql -FROM postgres-logs* -// messages are similar to "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" -| DISSECT message "%{date} - %{msg} - %{ip}" -// keep columns created by the dissect command -| KEEP date, msg, ip -// evaluate date from string representation -| EVAL date = DATE_PARSE("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", date) -``` - -**Find contributors which first name starts with "b", sort them by number of commits and -then returns their first and last names for the top 5** -```esql -FROM commits -| WHERE TO_LOWER(first_name) LIKE "b*" -| STATS doc_count = COUNT(*) by first_name, last_name -| SORT doc_count DESC -| KEEP first_name, last_name -| LIMIT 5 -``` - -**Returning average salary per hire date split in 20 buckets using BUCKET** -```esql -FROM employees -| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" -| STATS avg_salary = AVG(salary) BY date_bucket = BUCKET(hire_date, 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z") -| SORT bucket -``` - -**Returning number of employees grouped by buckets of salary** -```esql -FROM employees -| WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" -| STATS c = COUNT(1) BY b = BUCKET(salary, 5000.) -| SORT b -``` - -**returns total and recent hire counts plus ratio break down by country** -```esql -FROM employees -// insert a boolean column using case for conditional evaluation -| EVAL is_recent_hire = CASE(hire_date <= "2023-01-01T00:00:00Z", 1, 0) -// using stats with multiple grouping expressions -| STATS total_recent_hires = SUM(is_recent_hire), total_hires = COUNT(*) BY country -// evaluate the recent hiring rate by country based on the previous grouping expressions -| EVAL recent_hiring_rate = total_recent_hires / total_hires -``` - -**computes failure ratios from logs** -```esql -FROM logs-* -| WHERE @timestamp <= NOW() - 24 hours -// convert a keyword field into a numeric field to aggregate over it -| EVAL is_5xx = CASE(http.response.status_code >= 500, 1, 0) -// count total events and failed events to calculate a rate -| STATS total_events = COUNT(*), total_failures = SUM(is_5xx) BY host.hostname, bucket = BUCKET(@timestamp, 1 hour) -// evaluate the failure ratio -| EVAL failure_rate_per_host = total_failures / total_events -// drops the temporary columns -| DROP total_events, total_failures -``` - -**Returning the number of logs grouped by level over the past 24h** -```esql -FROM logs-* -| WHERE @timestamp <= NOW() - 24 hours -| STATS count = COUNT(*) BY log.level -| SORT count DESC -``` - -**Returning all first names for each first letter** -```esql -FROM employees -// evaluate first letter -| EVAL first_letter = SUBSTRING(first_name, 0, 1) -// group all first_name into a multivalued field, break down by first_letter -| STATS first_name = MV_SORT(VALUES(first_name)) BY first_letter -| SORT first_letter -``` - -**Retrieving the min, max and average value from a multivalued field** -```esql -FROM bag_of_numbers -| EVAL min = MV_MIN(numbers), max = MV_MAX(numbers), avg = MV_AVG(numbers) -| KEEP bad_id, min, max, avg -``` - -**Converts a date string into datetime using DATE_PARSE** -```esql -FROM personal_info -// birth_date is a text field storing date with the "yyyy-MM-dd" format -| EVAL birth=DATE_PARSE("yyyy-MM-dd", birth_date) -| KEEP user_name, birth -| SORT birth -``` - - -**Look up and join the `source.ip` field in `firewall_logs` with the `source.ip` field in `threat_list`, filters rows to include only those with non-null `threat_level`, sorts the results by `timestamp`, and keeps only the `source.ip`, `action`, `threat_level`, and `threat_type` fields.** - -```esql -FROM firewall_logs -| LOOKUP JOIN threat_list ON source.ip -| WHERE threat_level IS NOT NULL -| SORT timestamp -| KEEP source.ip, action, threat_level, threat_type -| LIMIT 10 -``` diff --git a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/task.ts b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/task.ts index a6bd7e5ab9877..9ec25b5a26806 100644 --- a/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/task.ts +++ b/x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/task.ts @@ -35,12 +35,6 @@ export function naturalLanguageToEsql({ }: NlToEsqlTaskParams): Observable> { return from(loadDocBase()).pipe( switchMap((docBase) => { - const systemMessage = `You are a helpful assistant for generating and executing ES|QL queries. -Your goal is to help the user construct an ES|QL query for their data. -VERY IMPORTANT: When writing ES|QL queries, make sure to ONLY use commands, functions -and operators listed in the current documentation. -${docBase.getSystemMessage()}`; - const messages: Message[] = 'input' in rest ? [{ role: MessageRole.User, content: rest.input }] : rest.messages; @@ -50,7 +44,6 @@ ${docBase.getSystemMessage()}`; messages, docBase, logger, - systemMessage, functionCalling, maxRetries, retryConfiguration, @@ -59,7 +52,7 @@ ${docBase.getSystemMessage()}`; tools, toolChoice, }, - system, + additionalSystemInstructions: system, }); return requestDocumentation({ @@ -69,7 +62,7 @@ ${docBase.getSystemMessage()}`; retryConfiguration, outputApi: client.output, messages, - system: systemMessage, + esqlPrompts: docBase.getPrompts(), metadata, toolOptions: { tools, diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/execute_query.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/execute_query.spec.ts index fc5480e853033..5a14ec3912ecb 100644 --- a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/execute_query.spec.ts +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/complete/functions/execute_query.spec.ts @@ -143,7 +143,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon it('contains user message with information about how to request ESQL documentation', () => { expect(last(secondRequestBody.messages)?.content).to.contain( - 'Based on the previous conversation, request documentation' + 'Now, based on the previous conversation, request documentation' ); }); }); @@ -157,11 +157,6 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon expect(hasToolCall).to.be(true); }); - it('contains ESQL documentation', () => { - const parsed = JSON.parse(last(thirdRequestBody.messages)?.content as string); - expect(parsed.documentation.OPERATORS).to.contain('Binary Operators'); - }); - it('allows the LLM to call the tools execute_query, visualize_query and request_documentation', () => { expect(thirdRequestBody.tools?.map((t) => t.function.name)).to.eql([ 'execute_query',