[ES|QL] improve available column lists

src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx

@@ -493,6 +493,7 @@ const ESQLEditorInternal = function ESQLEditor({
                   name: c.name,
                   type: c.meta.esType as FieldType,
                   hasConflict: c.meta.type === KBN_FIELD_TYPES.CONFLICT,
+                  userDefined: false,
                 };
               }) || [];
 

src/platform/packages/shared/kbn-esql-ast/src/__tests__/autocomplete.ts

@@ -15,11 +15,10 @@
 import { uniq } from 'lodash';
 import type { LicenseType } from '@kbn/licensing-types';
 import type {
-  ESQLUserDefinedColumn,
-  ESQLFieldWithMetadata,
   ICommandCallbacks,
   ISuggestionItem,
   Location,
+  ESQLColumnData,
 } from '../commands_registry/types';
 import { getLocationFromCommandOrOptionName } from '../commands_registry/types';
 import { aggFunctionDefinitions } from '../definitions/generated/aggregation_functions';
@@ -50,8 +49,7 @@ export const suggest = (
     arg1: ESQLCommand,
     arg2: ICommandCallbacks,
     arg3: {
-      userDefinedColumns: Map<string, ESQLUserDefinedColumn[]>;
-      fields: Map<string, ESQLFieldWithMetadata>;
+      columns: Map<string, ESQLColumnData>;
     },
     arg4?: number
   ) => Promise<ISuggestionItem[]>,
@@ -80,8 +78,7 @@ export const expectSuggestions = async (
     arg1: ESQLCommand,
     arg2: ICommandCallbacks,
     arg3: {
-      userDefinedColumns: Map<string, ESQLUserDefinedColumn[]>;
-      fields: Map<string, ESQLFieldWithMetadata>;
+      columns: Map<string, ESQLColumnData>;
     },
     arg4?: number
   ) => Promise<ISuggestionItem[]>,
@@ -105,12 +102,10 @@ export function getFieldNamesByType(
   _requestedType: Readonly<FieldType | 'any' | Array<FieldType | 'any'>>,
   excludeUserDefined: boolean = false
 ) {
-  const fieldsMap = mockContext.fields;
-  const userDefinedColumnsMap = mockContext.userDefinedColumns;
-  const fields = Array.from(fieldsMap.values());
-  const userDefinedColumns = Array.from(userDefinedColumnsMap.values()).flat();
+  const columnMap = mockContext.columns;
+  const columns = Array.from(columnMap.values());
   const requestedType = Array.isArray(_requestedType) ? _requestedType : [_requestedType];
-  const finalArray = excludeUserDefined ? fields : [...fields, ...userDefinedColumns];
+  const finalArray = excludeUserDefined ? columns.filter((col) => !col.userDefined) : columns;
   return finalArray
     .filter(
       ({ type }) =>

src/platform/packages/shared/kbn-esql-ast/src/__tests__/context_fixtures.ts

@@ -8,9 +8,8 @@
  */
 import type { IndexAutocompleteItem, InferenceEndpointAutocompleteItem } from '@kbn/esql-types';
 import type {
-  ESQLFieldWithMetadata,
+  ESQLColumnData,
   ESQLPolicy,
-  ESQLUserDefinedColumn,
   ICommandCallbacks,
   ICommandContext,
 } from '../commands_registry/types';
@@ -115,84 +114,79 @@ export const editorExtensions = {
 };
 
 export const mockContext: ICommandContext = {
-  userDefinedColumns: new Map<string, ESQLUserDefinedColumn[]>([
+  columns: new Map<string, ESQLColumnData>([
     [
       'var0',
-      [
-        {
-          name: 'var0',
-          type: 'double',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'var0',
+        type: 'double',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
     [
       'col0',
-      [
-        {
-          name: 'col0',
-          type: 'double',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'col0',
+        type: 'double',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
     [
       'prompt',
-      [
-        {
-          name: 'prompt',
-          type: 'keyword',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'prompt',
+        type: 'keyword',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
     [
       'integerPrompt',
-      [
-        {
-          name: 'integerPrompt',
-          type: 'integer',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'integerPrompt',
+        type: 'integer',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
     [
       'ipPrompt',
-      [
-        {
-          name: 'ipPrompt',
-          type: 'ip',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'ipPrompt',
+        type: 'ip',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
     [
       'renamedField',
-      [
-        {
-          name: 'renamedField',
-          type: 'keyword',
-          location: { min: 0, max: 10 },
-        },
-      ],
+      {
+        name: 'renamedField',
+        type: 'keyword',
+        location: { min: 0, max: 10 },
+        userDefined: true,
+      },
     ],
-  ]),
-  fields: new Map<string, ESQLFieldWithMetadata>([
-    ['keywordField', { name: 'keywordField', type: 'keyword' }],
-    ['any#Char$Field', { name: 'any#Char$Field', type: 'keyword' }],
-    ['textField', { name: 'textField', type: 'text' }],
-    ['doubleField', { name: 'doubleField', type: 'double' }],
-    ['integerField', { name: 'integerField', type: 'integer' }],
-    ['counterIntegerField', { name: 'counterIntegerField', type: 'counter_integer' }],
-    ['dateField', { name: 'dateField', type: 'date' }],
-    ['dateNanosField', { name: 'dateNanosField', type: 'date_nanos' }],
-    ['@timestamp', { name: '@timestamp', type: 'date' }],
-    ['ipField', { name: 'ipField', type: 'ip' }],
-    ['booleanField', { name: 'booleanField', type: 'boolean' }],
-    ['geoPointField', { name: 'geoPointField', type: 'geo_point' }],
-    ['geoShapeField', { name: 'geoShapeField', type: 'geo_shape' }],
-    ['versionField', { name: 'versionField', type: 'version' }],
-    ['longField', { name: 'longField', type: 'long' }],
+    ['keywordField', { name: 'keywordField', type: 'keyword', userDefined: false }],
+    ['any#Char$Field', { name: 'any#Char$Field', type: 'keyword', userDefined: false }],
+    ['textField', { name: 'textField', type: 'text', userDefined: false }],
+    ['doubleField', { name: 'doubleField', type: 'double', userDefined: false }],
+    ['integerField', { name: 'integerField', type: 'integer', userDefined: false }],
+    [
+      'counterIntegerField',
+      { name: 'counterIntegerField', type: 'counter_integer', userDefined: false },
+    ],
+    ['dateField', { name: 'dateField', type: 'date', userDefined: false }],
+    ['dateNanosField', { name: 'dateNanosField', type: 'date_nanos', userDefined: false }],
+    ['@timestamp', { name: '@timestamp', type: 'date', userDefined: false }],
+    ['ipField', { name: 'ipField', type: 'ip', userDefined: false }],
+    ['booleanField', { name: 'booleanField', type: 'boolean', userDefined: false }],
+    ['geoPointField', { name: 'geoPointField', type: 'geo_point', userDefined: false }],
+    ['geoShapeField', { name: 'geoShapeField', type: 'geo_shape', userDefined: false }],
+    ['versionField', { name: 'versionField', type: 'version', userDefined: false }],
+    ['longField', { name: 'longField', type: 'long', userDefined: false }],
   ]),
   policies: new Map<string, ESQLPolicy>(policies.map((policy) => [policy.name, policy])),
   sources: indexes.map((name) => ({

src/platform/packages/shared/kbn-esql-ast/src/__tests__/validation.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type { ESQLUserDefinedColumn, ESQLFieldWithMetadata } from '../commands_registry/types';
+import type { ESQLColumnData } from '../commands_registry/types';
 import { Parser } from '../parser';
 import type { ESQLCommand, ESQLMessage } from '../types';
 import { mockContext } from './context_fixtures';
@@ -28,8 +28,7 @@ export const expectErrors = (
     arg0: ESQLCommand,
     arg1: ESQLCommand[],
     arg2: {
-      userDefinedColumns: Map<string, ESQLUserDefinedColumn[]>;
-      fields: Map<string, ESQLFieldWithMetadata>;
+      columns: Map<string, ESQLColumnData>;
     }
   ) => any
 ) => {

src/platform/packages/shared/kbn-esql-ast/src/commands_registry/commands/change_point/columns_after.test.ts

@@ -7,54 +7,48 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 import { synth } from '../../../..';
-import type { ESQLFieldWithMetadata, ESQLUserDefinedColumn } from '../../types';
+import type { ESQLColumnData } from '../../types';
 import { columnsAfter } from './columns_after';
 
 describe('CHANGE_POINT > columnsAfter', () => {
-  const context = {
-    userDefinedColumns: new Map<string, ESQLUserDefinedColumn[]>([]),
-    fields: new Map<string, ESQLFieldWithMetadata>([
-      ['field1', { name: 'field1', type: 'keyword' }],
-      ['count', { name: 'count', type: 'double' }],
-    ]),
-  };
   it('adds "type" and "pvalue" fields, when AS option not specified', () => {
-    const previousCommandFields = [
-      { name: 'field1', type: 'keyword' },
-      { name: 'count', type: 'double' },
-    ] as ESQLFieldWithMetadata[];
+    const previousCommandFields: ESQLColumnData[] = [
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'count', type: 'double', userDefined: false },
+    ];
 
-    const result = columnsAfter(
-      synth.cmd`CHANGE_POINT count ON field1`,
-      previousCommandFields,
-      context
-    );
+    const result = columnsAfter(synth.cmd`CHANGE_POINT count ON field1`, previousCommandFields, '');
 
-    expect(result).toEqual([
-      { name: 'field1', type: 'keyword' },
-      { name: 'count', type: 'double' },
-      { name: 'type', type: 'keyword' },
-      { name: 'pvalue', type: 'double' },
+    expect(result).toEqual<ESQLColumnData[]>([
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'count', type: 'double', userDefined: false },
+      { name: 'type', type: 'keyword', userDefined: false },
+      { name: 'pvalue', type: 'double', userDefined: false },
     ]);
   });
 
   it('adds the given names as fields, when AS option is specified', () => {
-    const previousCommandFields = [
-      { name: 'field1', type: 'keyword' },
-      { name: 'count', type: 'double' },
-    ] as ESQLFieldWithMetadata[];
+    const previousCommandFields: ESQLColumnData[] = [
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'count', type: 'double', userDefined: false },
+    ];
 
     const result = columnsAfter(
       synth.cmd`CHANGE_POINT count ON field1 AS changePointType, pValue`,
       previousCommandFields,
-      context
+      ''
     );
 
-    expect(result).toEqual([
-      { name: 'field1', type: 'keyword' },
-      { name: 'count', type: 'double' },
-      { name: 'changePointType', type: 'keyword' },
-      { name: 'pValue', type: 'double' },
+    expect(result).toEqual<ESQLColumnData[]>([
+      { name: 'field1', type: 'keyword', userDefined: false },
+      { name: 'count', type: 'double', userDefined: false },
+      {
+        name: 'changePointType',
+        type: 'keyword',
+        userDefined: true,
+        location: { min: 0, max: 0 },
+      },
+      { name: 'pValue', type: 'double', userDefined: true, location: { min: 0, max: 0 } },
     ]);
   });
 });

src/platform/packages/shared/kbn-esql-ast/src/commands_registry/commands/change_point/columns_after.ts

@@ -7,30 +7,49 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 import uniqBy from 'lodash/uniqBy';
-import { type ESQLCommand, type ESQLAstChangePointCommand } from '../../../types';
 import { LeafPrinter } from '../../../pretty_print/leaf_printer';
-import type { ESQLFieldWithMetadata } from '../../types';
-import type { ICommandContext } from '../../types';
+import { type ESQLAstChangePointCommand, type ESQLCommand } from '../../../types';
+import type { ESQLColumnData } from '../../types';
 
 export const columnsAfter = (
   command: ESQLCommand,
-  previousColumns: ESQLFieldWithMetadata[],
-  context?: ICommandContext
+  previousColumns: ESQLColumnData[],
+  query: string
 ) => {
   const { target } = command as ESQLAstChangePointCommand;
 
-  return uniqBy(
-    [
-      ...previousColumns,
-      {
-        name: target ? LeafPrinter.column(target.type) : 'type',
-        type: 'keyword' as const,
-      },
-      {
-        name: target ? LeafPrinter.column(target.pvalue) : 'pvalue',
-        type: 'double' as const,
-      },
-    ],
-    'name'
-  );
+  let typeField: ESQLColumnData;
+  let pvalueField: ESQLColumnData;
+
+  if (target?.type) {
+    typeField = {
+      name: LeafPrinter.column(target.type),
+      type: 'keyword' as const,
+      userDefined: true,
+      location: target.type.location,
+    };
+  } else {
+    typeField = {
+      name: 'type',
+      type: 'keyword' as const,
+      userDefined: false,
+    };
+  }
+
+  if (target?.pvalue) {
+    pvalueField = {
+      name: LeafPrinter.column(target.pvalue),
+      type: 'double' as const,
+      userDefined: true,
+      location: target.pvalue.location,
+    };
+  } else {
+    pvalueField = {
+      name: 'pvalue',
+      type: 'double' as const,
+      userDefined: false,
+    };
+  }
+
+  return uniqBy([...previousColumns, typeField, pvalueField], 'name');
 };

src/platform/packages/shared/kbn-esql-ast/src/commands_registry/commands/change_point/validate.test.ts

@@ -50,11 +50,11 @@ describe('CHANGE_POINT Validation', () => {
 
       test('raises error when the default @timestamp field is missing', () => {
         // make sure that @timestamp field is not present
-        const newFields = new Map(mockContext.fields);
-        newFields.delete('@timestamp');
+        const newColumns = new Map(mockContext.columns);
+        newColumns.delete('@timestamp');
         const context = {
           ...mockContext,
-          fields: newFields,
+          columns: newColumns,
         };
         changePointExpectErrors(
           'FROM a_index | CHANGE_POINT doubleField',