[ES|QL] METRICS_INFO support: columns_after & summary

package.json

@@ -134,12 +134,12 @@
     "@elastic/ecs": "9.3.0",
     "@elastic/elasticsearch": "9.3.4",
     "@elastic/ems-client": "8.6.3",
-    "@elastic/esql": "1.1.0",
+    "@elastic/esql": "1.3.0",
     "@elastic/eui": "113.1.0",
     "@elastic/eui-theme-borealis": "6.1.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/kibana-d3-color": "npm:@elastic/kibana-d3-color@2.0.1",
-    "@elastic/monaco-esql": "3.1.21",
+    "@elastic/monaco-esql": "3.2.0",
     "@elastic/node-crypto": "1.2.3",
     "@elastic/numeral": "2.5.1",
     "@elastic/opentelemetry-node": "1.8.0",

src/dev/license_checker/config.ts

@@ -92,7 +92,7 @@ export const LICENSE_OVERRIDES = {
   'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
   '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
   '@elastic/ems-client@8.6.3': ['Elastic License 2.0'],
-  '@elastic/esql@1.1.0': ['Elastic License 2.0'],
+  '@elastic/esql@1.3.0': ['Elastic License 2.0'],
   '@elastic/eui@113.1.0': ['Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0'],
   '@elastic/eui-theme-borealis@6.1.0': ['Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0'],
   'language-subtag-registry@0.3.21': ['CC-BY-4.0'], // retired ODC‑By license https://github.com/mattcg/language-subtag-registry

src/platform/packages/shared/kbn-esql-language/src/commands/registry/index.ts

@@ -38,6 +38,7 @@ import { promqlCommand } from './promql';
 import { mergeCommandWithGeneratedCommandData } from './elastisearch_command_data_loader';
 import { setCommand } from './set';
 import { mmrCommand } from './mmr';
+import { metricsInfoCommand } from './metrics_info';
 
 const esqlCommandRegistry = new CommandRegistry();
 
@@ -71,6 +72,7 @@ const baseCommands = [
   rerankCommand,
   promqlCommand,
   mmrCommand,
+  metricsInfoCommand,
 ];
 
 baseCommands.forEach((command) => {

src/platform/packages/shared/kbn-esql-language/src/commands/registry/metrics_info/columns_after.test.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ESQLCommand } from '@elastic/esql/types';
+import type { ESQLColumnData } from '../types';
+import { columnsAfter } from './columns_after';
+import { METRICS_INFO_COLUMNS } from './columns_after';
+
+describe('METRICS_INFO > columnsAfter', () => {
+  const mockCommand = { name: 'metrics_info' } as ESQLCommand;
+
+  it('appends METRICS_INFO columns to previous columns', () => {
+    const previousColumns: ESQLColumnData[] = [
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'field2', type: 'double', userDefined: false },
+    ];
+
+    const result = columnsAfter(mockCommand, previousColumns, '');
+
+    expect(result).toEqual<ESQLColumnData[]>([
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'field2', type: 'double', userDefined: false },
+      ...METRICS_INFO_COLUMNS,
+    ]);
+  });
+
+  it('returns only METRICS_INFO columns when previous columns is empty', () => {
+    const result = columnsAfter(mockCommand, [], '');
+
+    expect(result).toEqual(METRICS_INFO_COLUMNS);
+  });
+});

src/platform/packages/shared/kbn-esql-language/src/commands/registry/metrics_info/columns_after.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ESQLCommand } from '@elastic/esql/types';
+import type { ESQLColumnData } from '../types';
+
+export const METRICS_INFO_COLUMNS: ESQLColumnData[] = [
+  { name: 'metric_name', type: 'keyword', userDefined: false },
+  { name: 'data_stream', type: 'keyword', userDefined: false },
+  { name: 'unit', type: 'keyword', userDefined: false },
+  { name: 'metric_type', type: 'keyword', userDefined: false },
+  { name: 'field_type', type: 'keyword', userDefined: false },
+  { name: 'dimension_fields', type: 'keyword', userDefined: false },
+];
+
+export const columnsAfter = (
+  _command: ESQLCommand,
+  previousColumns: ESQLColumnData[],
+  _query: string
+): ESQLColumnData[] => {
+  return [...previousColumns, ...METRICS_INFO_COLUMNS];
+};

src/platform/packages/shared/kbn-esql-language/src/commands/registry/metrics_info/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { i18n } from '@kbn/i18n';
+import type { ICommand, ICommandMethods } from '../registry';
+import type { ICommandContext } from '../types';
+import { Commands } from '../../definitions/keywords';
+import { columnsAfter } from './columns_after';
+import { summary } from './summary';
+
+const metricsInfoCommandMethods: ICommandMethods<ICommandContext> = {
+  autocomplete: () => Promise.resolve([]),
+  columnsAfter,
+  summary,
+};
+
+export const metricsInfoCommand: ICommand = {
+  name: Commands.METRICS_INFO,
+  methods: metricsInfoCommandMethods,
+  metadata: {
+    description: i18n.translate('kbn-esql-language.esql.definitions.metricsInfoDoc', {
+      defaultMessage:
+        'The METRICS_INFO command returns information about the metrics in the query. Only available on TS commands.',
+    }),
+    declaration: 'METRICS_INFO',
+    examples: ['TS index | METRICS_INFO'],
+    preview: true,
+    requiresTimeseriesSource: true,
+    hiddenAfterCommands: [Commands.STATS, Commands.INLINE_STATS],
+  },
+};

src/platform/packages/shared/kbn-esql-language/src/commands/registry/metrics_info/summary.test.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ESQLCommand } from '@elastic/esql/types';
+import { summary } from './summary';
+import { METRICS_INFO_COLUMNS } from './columns_after';
+
+describe('METRICS_INFO > summary', () => {
+  const mockCommand = { name: 'metrics_info' } as ESQLCommand;
+
+  it('returns exactly the fixed set of metrics info columns', () => {
+    const result = summary(mockCommand);
+
+    expect(result.newColumns).toEqual(new Set(METRICS_INFO_COLUMNS.map((column) => column.name)));
+  });
+
+  it('does not return renamedColumnsPairs, metadataColumns, aggregates, or grouping', () => {
+    const result = summary(mockCommand);
+
+    expect(result.renamedColumnsPairs).toBeUndefined();
+    expect(result.metadataColumns).toBeUndefined();
+    expect(result.aggregates).toBeUndefined();
+    expect(result.grouping).toBeUndefined();
+  });
+});

src/platform/packages/shared/kbn-esql-language/src/commands/registry/metrics_info/summary.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ESQLCommand } from '@elastic/esql/types';
+import type { ESQLCommandSummary } from '../types';
+import { METRICS_INFO_COLUMNS } from './columns_after';
+
+export const summary = (_command: ESQLCommand): ESQLCommandSummary => {
+  const newColumns = METRICS_INFO_COLUMNS.map((column) => column.name);
+  return { newColumns: new Set(newColumns) };
+};

src/platform/packages/shared/kbn-esql-language/src/commands/registry/registry.ts

@@ -96,6 +96,8 @@ export interface ICommandMetadata {
   observabilityTier?: string; // Optional property indicating the observability tier availability
   type?: 'source' | 'header' | 'processing'; // Optional property to classify the command type
   isTimeseries?: boolean; // Optional property to indicate if the command is a timeseries source command
+  requiresTimeseriesSource?: boolean; // Optional property to indicate the command is only available when the source command is TS
+  hiddenAfterCommands?: string[]; // Optional list of command names; this command is not suggested when any of them appear anywhere in the pipeline
   subqueryRestrictions?: {
     hideInside: boolean; // Command is hidden inside subqueries
     hideOutside: boolean; // Command is hidden outside subqueries (at root level)

src/platform/packages/shared/kbn-esql-language/src/language/autocomplete/autocomplete.test.ts

@@ -124,6 +124,20 @@ describe('autocomplete', () => {
       .flat();
   };
 
+  /** Suggestion text for commands that are only available when source is TS (requiresTimeseriesSource) */
+  const getRequiresTimeseriesSourceSuggestionTexts = (): string[] =>
+    esqlCommandRegistry
+      .getAllCommands()
+      .filter((c) => c.metadata?.requiresTimeseriesSource === true)
+      .map((c) => c.name.toUpperCase() + ' ');
+
+  /** Suggestion text for commands that have hiddenAfterCommands (not suggested when any of those commands appear in the pipeline) */
+  const getCommandsWithHiddenAfterCommandsSuggestionTexts = (): string[] =>
+    esqlCommandRegistry
+      .getAllCommands()
+      .filter((c) => (c.metadata?.hiddenAfterCommands?.length ?? 0) > 0)
+      .map((c) => c.name.toUpperCase() + ' ');
+
   describe('New command', () => {
     const recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
       'FROM logs*',
@@ -137,11 +151,13 @@ describe('autocomplete', () => {
       ...recommendedQuerySuggestions.map((q) => q.queryString),
     ]);
     const commands = getNonSourceHeaderCommands();
+    const tsOnlySuggestionTexts = getRequiresTimeseriesSourceSuggestionTexts();
+    const commandsAfterNonTsSource = commands.filter((c) => !tsOnlySuggestionTexts.includes(c));
 
-    testSuggestions('from a | /', commands);
-    testSuggestions('from a metadata _id | /', commands);
-    testSuggestions('from a | eval col0 = a | /', commands);
-    testSuggestions('from a metadata _id | eval col0 = a | /', commands);
+    testSuggestions('from a | /', commandsAfterNonTsSource);
+    testSuggestions('from a metadata _id | /', commandsAfterNonTsSource);
+    testSuggestions('from a | eval col0 = a | /', commandsAfterNonTsSource);
+    testSuggestions('from a metadata _id | eval col0 = a | /', commandsAfterNonTsSource);
 
     const promqlPipedQueries = [
       'PROMQL index=metrics (sum by (instance) rate(http_requests_total[5m])) | /',
@@ -152,7 +168,48 @@ describe('autocomplete', () => {
     ];
 
     promqlPipedQueries.forEach((query) => {
-      testSuggestions(query, commands);
+      testSuggestions(query, commandsAfterNonTsSource);
+    });
+  });
+
+  describe('command filtering by metadata (requiresTimeseriesSource, hiddenAfterCommands)', () => {
+    const tsOnlySuggestionTexts = getRequiresTimeseriesSourceSuggestionTexts();
+    const hiddenAfterCommandsSuggestionTexts = getCommandsWithHiddenAfterCommandsSuggestionTexts();
+
+    it('does not suggest commands with requiresTimeseriesSource when source is not TS (e.g. after FROM)', async () => {
+      const { suggest: suggestFn } = await setup();
+      const suggestedTexts = (await suggestFn('FROM index | /')).map((s) => s.text);
+      for (const text of tsOnlySuggestionTexts) {
+        expect(suggestedTexts).not.toContain(text);
+      }
+    });
+
+    it('suggests commands with requiresTimeseriesSource when source is TS and cursor is after pipe', async () => {
+      const { suggest: suggestFn } = await setup();
+      const suggestedTexts = (await suggestFn('TS index | /')).map((s) => s.text);
+      for (const text of tsOnlySuggestionTexts) {
+        expect(suggestedTexts).toContain(text);
+      }
+    });
+
+    it('does not suggest commands with hiddenAfterCommands when a listed command is the previous command', async () => {
+      const { suggest: suggestFn } = await setup();
+      const suggestedTexts = (await suggestFn('TS index | STATS x = count(*) | /')).map(
+        (s) => s.text
+      );
+      for (const text of hiddenAfterCommandsSuggestionTexts) {
+        expect(suggestedTexts).not.toContain(text);
+      }
+    });
+
+    it('does not suggest commands with hiddenAfterCommands when a listed command appears anywhere in the pipeline', async () => {
+      const { suggest: suggestFn } = await setup();
+      const suggestedTexts = (
+        await suggestFn('TS index | STATS AVG(field1) | EVAL test = "hello" | /')
+      ).map((s) => s.text);
+      for (const text of hiddenAfterCommandsSuggestionTexts) {
+        expect(suggestedTexts).not.toContain(text);
+      }
     });
   });
 
@@ -270,9 +327,11 @@ describe('autocomplete', () => {
     ]);
 
     const commands = getNonSourceHeaderCommands();
+    const tsOnlySuggestionTexts = getRequiresTimeseriesSourceSuggestionTexts();
+    const commandsAfterNonTsSource = commands.filter((c) => !tsOnlySuggestionTexts.includes(c));
 
     // pipe command
-    testSuggestions('FROM k | E/', commands);
+    testSuggestions('FROM k | E/', commandsAfterNonTsSource);
 
     describe('function arguments', () => {
       // function argument
@@ -498,11 +557,12 @@ describe('autocomplete', () => {
     ]);
 
     const commands = getNonSourceHeaderCommands();
-
+    const tsOnlySuggestionTexts = getRequiresTimeseriesSourceSuggestionTexts();
+    const commandsAfterNonTsSource = commands.filter((c) => !tsOnlySuggestionTexts.includes(c));
     // Pipe command
     testSuggestions(
       'FROM a | E/',
-      commands.map((name) => attachTriggerCommand(name))
+      commandsAfterNonTsSource.map((name) => attachTriggerCommand(name))
     );
 
     describe('function arguments', () => {

src/platform/packages/shared/kbn-esql-language/src/language/autocomplete/autocomplete.ts

@@ -36,6 +36,7 @@ import { getQueryForFields } from '../shared/get_query_for_fields';
 import type { GetColumnMapFn } from '../shared/columns_retrieval_helpers';
 import { getColumnsByTypeRetriever } from '../shared/columns_retrieval_helpers';
 import { getUnmappedFieldsStrategy } from '../../commands/definitions/utils/settings';
+import { isTimeseriesSourceCommand } from '../../commands/definitions/utils/timeseries_check';
 
 function isSourceCommandSuggestion({ label }: { label: string }) {
   const sourceCommands = esqlCommandRegistry
@@ -113,6 +114,32 @@ export async function suggest(
 
         return hasLicenseAccess && hasObservabilityAccess;
       })
+      .filter((command) => {
+        // Commands that require a TS source are only suggested when the source command is TS
+        if (command.metadata?.requiresTimeseriesSource) {
+          return (
+            astContext.astForContext.commands.length > 0 &&
+            isTimeseriesSourceCommand(astContext.astForContext.commands)
+          );
+        }
+        return true;
+      })
+      .filter((command) => {
+        // Commands with hiddenAfterCommands are not suggested when any command in the pipeline is in that list
+        const hiddenAfter = command.metadata?.hiddenAfterCommands;
+        if (hiddenAfter?.length && astContext.astForContext.commands.length > 0) {
+          const commandNamesInPipeline = new Set(
+            astContext.astForContext.commands.map((cmd) => cmd.name).filter(Boolean)
+          );
+          const hasHiddenCommandInPipeline = hiddenAfter.some((name) =>
+            commandNamesInPipeline.has(name)
+          );
+          if (hasHiddenCommandInPipeline) {
+            return false;
+          }
+        }
+        return true;
+      })
       .map((command) => command.name);
 
     const suggestions = getCommandAutocompleteDefinitions(commands);

yarn.lock

@@ -2436,10 +2436,10 @@
   resolved "https://registry.yarnpkg.com/@elastic/eslint-plugin-eui/-/eslint-plugin-eui-2.9.0.tgz#0901bb3c98b4a1006473d2d4e54b6e49cd537095"
   integrity sha512-aIW2iGyeL36w8nXK3huGKBd68sskSr0NdrvXh7Ldx02jz/ASnyZ0PlK7LireueNa8hjd7pG+VWa5b3YHdgQb3w==
 
-"@elastic/esql@1.1.0":
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/@elastic/esql/-/esql-1.1.0.tgz#704dbfe5f314f4669e7fb672463ba9cad1d77679"
-  integrity sha512-7uxePub21pVUVjhR+N09FpHeSP/9/p6eyib9mEQqOFSKVRRadfEhtks2LwTs1unn52NC/G6U4o06x1RtVtg8nw==
+"@elastic/esql@1.3.0":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@elastic/esql/-/esql-1.3.0.tgz#91cbf200ce123ded7eaca260857f4334237d93f5"
+  integrity sha512-ZOZjlII7+vL2fH8RR0R0cF5jWu+zWX0Jt8tEoCC0lTjMQ6Z3DjgzbdaRCpbnpt9Cmik9y1HFzGArL9EeSiOpmQ==
   dependencies:
     antlr4 "4.13.2"
     tree-dump "1.1.0"
@@ -2527,10 +2527,10 @@
     progress "^1.1.8"
     through2 "^2.0.0"
 
-"@elastic/monaco-esql@3.1.21":
-  version "3.1.21"
-  resolved "https://registry.yarnpkg.com/@elastic/monaco-esql/-/monaco-esql-3.1.21.tgz#abeec7bcd84405b618ad03aab522943c87007717"
-  integrity sha512-1GQY8L34DLrOdZKcHdURBy0Ao7NWc5HSot8roikXyhWY8i0dWB09sJdJF5dyZ9Tgu6lbLXVAFtlLkp83BYxM2w==
+"@elastic/monaco-esql@3.2.0":
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@elastic/monaco-esql/-/monaco-esql-3.2.0.tgz#328d4ca591ae1fd29b6c48d89622158b25617d70"
+  integrity sha512-ZFopjEoNU87wIFtHKT75kqvtSFtb3ye4X9foyJccABOIEQXR20dJkUJR74nDW09xDPr0AywrKwP2zTNXfr6n4g==
 
 "@elastic/node-crypto@1.2.3", "@elastic/node-crypto@^1.2.3":
   version "1.2.3"