[ES|QL] Supports remote cluster lookup mode indices in the editor

src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx

@@ -36,7 +36,7 @@ import type { ILicense } from '@kbn/licensing-types';
 import { ESQLLang, ESQL_LANG_ID, monaco, type ESQLCallbacks } from '@kbn/monaco';
 import type { ComponentProps } from 'react';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-import { fixESQLQueryWithVariables } from '@kbn/esql-utils';
+import { fixESQLQueryWithVariables, getRemoteClustersFromESQLQuery } from '@kbn/esql-utils';
 import { createPortal } from 'react-dom';
 import { css } from '@emotion/react';
 import { ESQLVariableType, type ESQLControlVariable } from '@kbn/esql-types';
@@ -527,7 +527,13 @@ const ESQLEditorInternal = function ESQLEditor({
       canSuggestVariables: () => {
         return variablesService?.areSuggestionsEnabled ?? false;
       },
-      getJoinIndices: kibana.services?.esql?.getJoinIndicesAutocomplete,
+      getJoinIndices: async () => {
+        const remoteClusters = getRemoteClustersFromESQLQuery(code);
+        const result = await kibana.services?.esql?.getJoinIndicesAutocomplete?.(
+          remoteClusters?.join(',')
+        );
+        return result ?? { indices: [] };
+      },
       getTimeseriesIndices: kibana.services?.esql?.getTimeseriesIndicesAutocomplete,
       getEditorExtensions: async (queryString: string) => {
         if (activeSolutionId) {
@@ -560,6 +566,7 @@ const ESQLEditorInternal = function ESQLEditor({
     };
     return callbacks;
   }, [
+    code,
     fieldsMetadata,
     kibana.services?.esql,
     dataSourcesCache,

src/platform/packages/private/kbn-esql-editor/src/types.ts

@@ -112,7 +112,7 @@ interface ESQLVariableService {
 }
 
 export interface EsqlPluginStartBase {
-  getJoinIndicesAutocomplete: () => Promise<IndicesAutocompleteResult>;
+  getJoinIndicesAutocomplete: (remoteClusters?: string) => Promise<IndicesAutocompleteResult>;
   getTimeseriesIndicesAutocomplete: () => Promise<IndicesAutocompleteResult>;
   getEditorExtensionsAutocomplete: (
     queryString: string,

src/platform/packages/shared/kbn-esql-types/index.ts

@@ -23,14 +23,11 @@ export {
 export {
   type IndicesAutocompleteResult,
   type IndexAutocompleteItem,
+  type ResolveIndexResponse,
   type ESQLSourceResult,
 } from './src/sources_autocomplete_types';
 
-export {
-  type RecommendedQuery,
-  type RecommendedField,
-  type ResolveIndexResponse,
-} from './src/extensions_autocomplete_types';
+export { type RecommendedQuery, type RecommendedField } from './src/extensions_autocomplete_types';
 
 export {
   type InferenceEndpointsAutocompleteResult,

src/platform/packages/shared/kbn-esql-types/src/extensions_autocomplete_types.ts

@@ -22,15 +22,3 @@ export interface RecommendedField {
   // The associated index pattern for the field, used to match the field with the current query's index pattern
   pattern: string;
 }
-
-interface ResolveIndexResponseItem {
-  name: string;
-  mode?: 'lookup' | 'time_series' | string;
-  aliases?: string[];
-}
-
-export interface ResolveIndexResponse {
-  indices?: ResolveIndexResponseItem[];
-  aliases?: ResolveIndexResponseItem[];
-  data_streams?: ResolveIndexResponseItem[];
-}

src/platform/packages/shared/kbn-esql-types/src/sources_autocomplete_types.ts

@@ -23,3 +23,17 @@ export interface ESQLSourceResult {
   dataStreams?: Array<{ name: string; title?: string }>;
   type?: string;
 }
+
+// response from resolve_index api
+interface ResolveIndexResponseItem {
+  name: string;
+  mode?: 'lookup' | 'time_series' | string;
+  indices?: string[];
+  aliases?: string[];
+}
+
+export interface ResolveIndexResponse {
+  indices?: ResolveIndexResponseItem[];
+  aliases?: ResolveIndexResponseItem[];
+  data_streams?: ResolveIndexResponseItem[];
+}

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -45,6 +45,7 @@ export {
   extractCategorizeTokens,
   getArgsFromRenameFunction,
   getCategorizeField,
+  getRemoteClustersFromESQLQuery,
 } from './src';
 
 export { ENABLE_ESQL, FEEDBACK_LINK } from './constants';

src/platform/packages/shared/kbn-esql-utils/src/index.ts

@@ -27,6 +27,7 @@ export {
   getCategorizeColumns,
   getArgsFromRenameFunction,
   getCategorizeField,
+  getRemoteClustersFromESQLQuery,
 } from './utils/query_parsing_helpers';
 export { queryCannotBeSampled } from './utils/query_cannot_be_sampled';
 export {

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -10,6 +10,7 @@ import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 import { ESQLVariableType, type ESQLControlVariable } from '@kbn/esql-types';
 import {
   getIndexPatternFromESQLQuery,
+  getRemoteClustersFromESQLQuery,
   getLimitFromESQLQuery,
   removeDropCommandsFromESQLQuery,
   hasTransformationalCommand,
@@ -963,4 +964,60 @@ describe('esql query helpers', () => {
       expect(getCategorizeField(esql)).toEqual(expected);
     });
   });
+
+  describe('getRemoteClustersFromESQLQuery', () => {
+    it('should return undefined for queries without remote clusters', () => {
+      expect(getRemoteClustersFromESQLQuery('FROM foo')).toBeUndefined();
+      expect(getRemoteClustersFromESQLQuery('FROM foo-1,foo-2')).toBeUndefined();
+      expect(getRemoteClustersFromESQLQuery('FROM foo | STATS COUNT(*)')).toBeUndefined();
+    });
+
+    it('should return undefined for empty or undefined queries', () => {
+      expect(getRemoteClustersFromESQLQuery('')).toBeUndefined();
+      expect(getRemoteClustersFromESQLQuery()).toBeUndefined();
+    });
+
+    it('should extract remote clusters from FROM command', () => {
+      expect(getRemoteClustersFromESQLQuery('FROM cluster1:index1')).toEqual(['cluster1']);
+      expect(getRemoteClustersFromESQLQuery('FROM remote_cluster:foo-2')).toEqual([
+        'remote_cluster',
+      ]);
+    });
+
+    it('should extract multiple remote clusters from mixed indices', () => {
+      expect(
+        getRemoteClustersFromESQLQuery(
+          'FROM local-index, cluster1:remote-index1, cluster2:remote-index2'
+        )
+      ).toEqual(['cluster1', 'cluster2']);
+      expect(
+        getRemoteClustersFromESQLQuery('FROM cluster1:index1, local-index, cluster2:index2')
+      ).toEqual(['cluster1', 'cluster2']);
+    });
+
+    it('should extract remote clusters from TS command', () => {
+      expect(getRemoteClustersFromESQLQuery('TS cluster1:tsdb')).toEqual(['cluster1']);
+      expect(
+        getRemoteClustersFromESQLQuery('TS remote_cluster:timeseries | STATS max(cpu) BY host')
+      ).toEqual(['remote_cluster']);
+    });
+
+    it('should handle duplicate remote clusters', () => {
+      expect(getRemoteClustersFromESQLQuery('FROM cluster1:index1, cluster1:index2')).toEqual([
+        'cluster1',
+      ]);
+    });
+
+    it('should handle wrapped in quotes', () => {
+      expect(getRemoteClustersFromESQLQuery('FROM "cluster1:index1,cluster1:index2"')).toEqual([
+        'cluster1',
+      ]);
+
+      expect(
+        getRemoteClustersFromESQLQuery(
+          'FROM "cluster1:index1,cluster1:index2", "cluster2:index3", cluster3:index3, index4'
+        )
+      ).toEqual(['cluster3', 'cluster1', 'cluster2']);
+    });
+  });
 });

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -39,6 +39,44 @@ export function getIndexPatternFromESQLQuery(esql?: string) {
   return indices?.map((index) => index.name).join(',');
 }
 
+export function getRemoteClustersFromESQLQuery(esql?: string): string[] | undefined {
+  if (!esql) return undefined;
+  const { root } = Parser.parse(esql);
+  const sourceCommand = root.commands.find(({ name }) => ['from', 'ts'].includes(name));
+  const args = (sourceCommand?.args ?? []) as ESQLSource[];
+
+  const clustersSet = new Set<string>();
+
+  // Handle sources with explicit prefix (e.g., FROM cluster1:index1)
+  args
+    .filter((arg) => arg.prefix)
+    .forEach((arg) => {
+      if (arg.prefix?.value) {
+        clustersSet.add(arg.prefix.value);
+      }
+    });
+
+  // Handle sources without prefix that might contain cluster:index patterns
+  // This includes quoted sources like "cluster1:index1,cluster2:index2"
+  args
+    .filter((arg) => !arg.prefix)
+    .forEach((arg) => {
+      // Split by comma to handle cases like "cluster1:index1,cluster2:index2"
+      const indices = arg.name.split(',');
+      indices.forEach((index) => {
+        const trimmedIndex = index.trim();
+        const colonIndex = trimmedIndex.indexOf(':');
+        // Only add if there's a valid cluster:index pattern
+        if (colonIndex > 0 && colonIndex < trimmedIndex.length - 1) {
+          const clusterName = trimmedIndex.substring(0, colonIndex);
+          clustersSet.add(clusterName);
+        }
+      });
+    });
+
+  return clustersSet.size > 0 ? [...clustersSet] : undefined;
+}
+
 // For ES|QL we consider stats and keep transformational command
 // The metrics command too but only if it aggregates
 export function hasTransformationalCommand(esql?: string) {

src/platform/plugins/shared/esql/public/plugin.ts

@@ -49,7 +49,7 @@ interface EsqlPluginStartDependencies {
 }
 
 export interface EsqlPluginStart {
-  getJoinIndicesAutocomplete: () => Promise<IndicesAutocompleteResult>;
+  getJoinIndicesAutocomplete: (remoteClusters?: string) => Promise<IndicesAutocompleteResult>;
   getTimeseriesIndicesAutocomplete: () => Promise<IndicesAutocompleteResult>;
   getInferenceEndpointsAutocomplete?: (
     taskType: InferenceTaskType
@@ -106,14 +106,18 @@ export class EsqlPlugin implements Plugin<{}, EsqlPluginStart> {
 
     const variablesService = new EsqlVariablesService();
 
-    const getJoinIndicesAutocomplete = cacheNonParametrizedAsyncFunction(
-      async () => {
+    const getJoinIndicesAutocomplete = cacheParametrizedAsyncFunction(
+      async (remoteClusters?: string) => {
+        const query = remoteClusters ? { remoteClusters } : {};
+
         const result = await core.http.get<IndicesAutocompleteResult>(
-          '/internal/esql/autocomplete/join/indices'
+          '/internal/esql/autocomplete/join/indices',
+          { query }
         );
 
         return result;
       },
+      (remoteClusters?: string) => remoteClusters || '',
       1000 * 60 * 5, // Keep the value in cache for 5 minutes
       1000 * 15 // Refresh the cache in the background only if 15 seconds passed since the last call
     );

src/platform/plugins/shared/esql/server/README.md

@@ -8,7 +8,7 @@ This directory contains the server-side components and API routes for the ES|QL
 
 The ES|QL server exposes the following internal API routes:
 
-* **`/internal/esql/autocomplete/join/indices`**: Used by the ES|QL editor to retrieve a list of indices suitable for **`JOIN`** autocompletion.
+* **`/internal/esql/autocomplete/join/indices`**: Used by the ES|QL editor to retrieve a list of indices suitable for **`JOIN`** autocompletion. If the remoteClusters array is passed it will also retrieve the list of remote indices.
 * **`/internal/esql/autocomplete/timeseries/indices`**: Provides index suggestions specifically for **time-series analysis** contexts within the ES|QL editor.
 * **`/internal/esql/autocomplete/inference_endpoints/{taskType}`**: This endpoint is used to fetch LLM inference endpoints by task type.
 * **`/internal/esql_registry/extensions/{query}`**: This is the primary endpoint for fetching **registered ES|QL extensions**, which enhance the editor's capabilities by providing contextual suggestions.

src/platform/plugins/shared/esql/server/lookup/utils.test.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { getListOfCCSIndices } from './utils';
+import type { IndexAutocompleteItem } from '@kbn/esql-types';
+
+describe('getListOfCCSIndices', () => {
+  const createLookupItem = (indexName: string, aliases?: string[]): IndexAutocompleteItem => ({
+    name: indexName,
+    mode: 'lookup',
+    aliases: aliases || [],
+  });
+
+  it('should return empty array when no clusters are provided', () => {
+    const clusters: string[] = [];
+    const lookupIndices = [
+      createLookupItem('cluster1:index1'),
+      createLookupItem('cluster2:index2'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([]);
+  });
+
+  it('should return empty array when no lookup indices are provided', () => {
+    const clusters = ['cluster1', 'cluster2'];
+    const lookupIndices: IndexAutocompleteItem[] = [];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([]);
+  });
+
+  it('should return empty array when no indices match the specified clusters', () => {
+    const clusters = ['cluster1', 'cluster2'];
+    const lookupIndices = [
+      createLookupItem('cluster3:index1'),
+      createLookupItem('cluster4:index2'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([]);
+  });
+
+  it('should handle indices without cluster prefix', () => {
+    const clusters = ['cluster1'];
+    const lookupIndices = [
+      createLookupItem('index1'),
+      createLookupItem('cluster1:index2'),
+      createLookupItem('index3'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([createLookupItem('index2')]);
+  });
+
+  it('should handle multiple indices in the same cluster', () => {
+    const clusters = ['cluster1'];
+    const lookupIndices = [
+      createLookupItem('cluster1:index1', ['alias1']),
+      createLookupItem('cluster1:index2'),
+      createLookupItem('cluster1:index3'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([
+      createLookupItem('index1', ['alias1']),
+      createLookupItem('index2'),
+      createLookupItem('index3'),
+    ]);
+  });
+
+  it('should return empty array when no common indices exist across all clusters', () => {
+    const clusters = ['cluster1', 'cluster2', 'cluster3'];
+    const lookupIndices = [
+      createLookupItem('cluster1:index1'),
+      createLookupItem('cluster2:index2'),
+      createLookupItem('cluster3:index3'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([]);
+  });
+
+  it('should find common indices across multiple clusters', () => {
+    const clusters = ['cluster1', 'cluster2', 'cluster3'];
+    const lookupIndices = [
+      createLookupItem('cluster1:index1', ['alias1']),
+      createLookupItem('cluster1:index2'),
+      createLookupItem('cluster2:index1', ['alias2']),
+      createLookupItem('cluster2:index3'),
+      createLookupItem('cluster3:index1', ['alias1']), // alias1 is duplicated to test Set uniqueness
+      createLookupItem('cluster3:index4'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([createLookupItem('index1', ['alias1', 'alias2'])]);
+  });
+
+  it('should handle malformed cluster:index patterns', () => {
+    const clusters = ['cluster1'];
+    const lookupIndices = [
+      createLookupItem('cluster1:'),
+      createLookupItem(':index1'),
+      createLookupItem('cluster1:index2'),
+      createLookupItem('notacluster'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([createLookupItem('index2')]);
+  });
+
+  it('should handle indices with complex names containing special characters', () => {
+    const clusters = ['cluster1', 'cluster2'];
+    const lookupIndices = [
+      createLookupItem('cluster1:logs-2023.01.01'),
+      createLookupItem('cluster1:metrics_system'),
+      createLookupItem('cluster2:logs-2023.01.01'),
+      createLookupItem('cluster2:traces.apm'),
+    ];
+    const result = getListOfCCSIndices(clusters, lookupIndices);
+    expect(result).toEqual([createLookupItem('logs-2023.01.01')]);
+  });
+});