Add CCS Remote Views Detection

docs/changelog/143384.yaml

@@ -0,0 +1,5 @@
+area: ES|QL
+issues: []
+pr: 143384
+summary: Add CCS Remote Views Detection
+type: enhancement

server/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -2075,6 +2075,12 @@ private enum ElasticsearchExceptionHandle {
             UpdateNotSupportedException::new,
             190,
             OCC_NOT_SUPPORTED_EXCEPTION_VERSION
+        ),
+        REMOTE_VIEW_NOT_SUPPORTED_EXCEPTION(
+            org.elasticsearch.action.fieldcaps.RemoteViewNotSupportedException.class,
+            org.elasticsearch.action.fieldcaps.RemoteViewNotSupportedException::new,
+            191,
+            org.elasticsearch.action.support.IndicesOptions.INDICES_OPTIONS_RESOLVE_VIEWS
         );
 
         final Class<? extends ElasticsearchException> exceptionClass;

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -128,7 +128,7 @@ void clusterAlias(String clusterAlias) {
         this.clusterAlias = clusterAlias;
     }
 
-    String clusterAlias() {
+    public String clusterAlias() {
         return clusterAlias;
     }
 

server/src/main/java/org/elasticsearch/action/fieldcaps/RemoteViewNotSupportedException.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.fieldcaps;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.RemoteClusterAware;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Thrown when ES|QL detects views during cross-cluster search field resolution.
+ * Views are not supported in CCS and the query must fail.
+ */
+public class RemoteViewNotSupportedException extends ElasticsearchException {
+
+    private static final String VIEW_NAMES_KEY = "es.esql.view.names";
+
+    @SuppressWarnings("this-escape")
+    public RemoteViewNotSupportedException(List<String> views) {
+        super(message(views));
+        addMetadata(VIEW_NAMES_KEY, views);
+    }
+
+    public RemoteViewNotSupportedException(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    /**
+     * Merge two exceptions into one that reports all matched views.
+     */
+    public static RemoteViewNotSupportedException merge(RemoteViewNotSupportedException a, RemoteViewNotSupportedException b) {
+        return new RemoteViewNotSupportedException(
+            Stream.concat(a.getMetadata(VIEW_NAMES_KEY).stream(), b.getMetadata(VIEW_NAMES_KEY).stream()).toList()
+        );
+    }
+
+    private static String message(List<String> views) {
+        String exclusions = views.stream().map(v -> {
+            String[] clusterAndIndex = RemoteClusterAware.splitIndexName(v);
+            return clusterAndIndex[0] + ":-" + clusterAndIndex[1];
+        }).collect(Collectors.joining(","));
+        return "ES|QL queries with remote views are not supported. Matched "
+            + views
+            + ". Remove them from the query pattern or exclude them with ["
+            + exclusions
+            + "] if matched by a wildcard.";
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+}

server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@@ -530,9 +530,12 @@ private enum Option {
 
         ALLOW_FAILURE_INDICES,  // Added in 8.14, Removed in 8.18
         ALLOW_SELECTORS,        // Added in 8.18
-        INCLUDE_FAILURE_INDICES // Added in 8.18
+        INCLUDE_FAILURE_INDICES, // Added in 8.18
+        RESOLVE_VIEWS
     }
 
+    public static final TransportVersion INDICES_OPTIONS_RESOLVE_VIEWS = TransportVersion.fromName("esql_resolve_fields_response_views");
+
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IndicesOptions.class);
     private static final String IGNORE_THROTTLED_DEPRECATION_MESSAGE = "[ignore_throttled] parameter is deprecated "
         + "because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices.";
@@ -1012,6 +1015,9 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
         if (gatekeeperOptions.includeFailureIndices()) {
             backwardsCompatibleOptions.add(Option.INCLUDE_FAILURE_INDICES);
         }
+        if (indexAbstractionOptions.resolveViews() && out.getTransportVersion().supports(INDICES_OPTIONS_RESOLVE_VIEWS)) {
+            backwardsCompatibleOptions.add(Option.RESOLVE_VIEWS);
+        }
         out.writeEnumSet(backwardsCompatibleOptions);
 
         EnumSet<WildcardStates> states = EnumSet.noneOf(WildcardStates.class);
@@ -1044,14 +1050,15 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
             .includeFailureIndices(includeFailureIndices)
             .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED))
             .build();
+        IndexAbstractionOptions indexAbstractionOptions = new IndexAbstractionOptions(options.contains(Option.RESOLVE_VIEWS));
         return new IndicesOptions(
             options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS)
                 ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS
                 : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcardOptions,
             gatekeeperOptions,
             CrossProjectModeOptions.readFrom(in),
-            IndexAbstractionOptions.DEFAULT
+            indexAbstractionOptions
         );
     }
 

server/src/main/resources/transport/definitions/referable/esql_resolve_fields_response_views.csv

@@ -0,0 +1 @@
+9315000

server/src/main/resources/transport/upper_bounds/9.4.csv

@@ -1 +1 @@
-inference_endpoint_metadata_display_model_creator_added,9314000
+esql_resolve_fields_response_views,9315000

server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java

@@ -880,6 +880,7 @@ public void testIds() {
         ids.put(188, SearchContextMissingNodesException.class);
         ids.put(189, OCCNotSupportedException.class);
         ids.put(190, UpdateNotSupportedException.class);
+        ids.put(191, org.elasticsearch.action.fieldcaps.RemoteViewNotSupportedException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {

server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -69,13 +70,27 @@ public void testSerialization() throws Exception {
             StreamInput streamInput = output.bytes().streamInput();
             IndicesOptions indicesOptions2 = IndicesOptions.readIndicesOptions(streamInput);
 
-            IndicesOptions expected = IndicesOptions.builder(indicesOptions)
-                .indexAbstractionOptions(IndexAbstractionOptions.builder(indicesOptions.indexAbstractionOptions()).resolveViews(false))
-                .build();
-            assertThat(indicesOptions2, equalTo(expected));
+            assertThat(indicesOptions2, equalTo(indicesOptions));
         }
     }
 
+    public void testSerializationResolveViewsDroppedBeforeSupportVersion() throws Exception {
+        IndicesOptions indicesOptions = IndicesOptions.builder()
+            .wildcardOptions(WildcardOptions.builder().matchOpen(true).matchClosed(false))
+            .indexAbstractionOptions(IndexAbstractionOptions.builder().resolveViews(true))
+            .build();
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        output.setTransportVersion(TransportVersionUtils.getPreviousVersion(IndicesOptions.INDICES_OPTIONS_RESOLVE_VIEWS));
+        indicesOptions.writeIndicesOptions(output);
+
+        StreamInput streamInput = output.bytes().streamInput();
+        streamInput.setTransportVersion(output.getTransportVersion());
+        IndicesOptions deserialized = IndicesOptions.readIndicesOptions(streamInput);
+
+        assertFalse(deserialized.indexAbstractionOptions().resolveViews());
+    }
+
     public void testFromOptions() {
         final boolean ignoreUnavailable = randomBoolean();
         final boolean allowNoIndices = randomBoolean();

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.ccq;
 
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.Version;
 
@@ -30,6 +31,7 @@ static ElasticsearchCluster remoteCluster(Map<String, String> additionalSettings
             .setting("node.roles", "[data,ingest,master]")
             .setting("xpack.security.enabled", "false")
             .setting("xpack.license.self_generated.type", "trial")
+            .feature(FeatureFlag.ESQL_VIEWS)
             .shared(true);
         if (supportRetryOnShardFailures(version) == false) {
             cluster.setting("cluster.routing.rebalance.enable", "none");
@@ -73,6 +75,7 @@ public static ElasticsearchCluster localCluster(
             .setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"")
             .setting("cluster.remote.connections_per_cluster", "1")
             .setting("cluster.remote." + REMOTE_CLUSTER_NAME + ".skip_unavailable", skipUnavailable.toString())
+            .feature(FeatureFlag.ESQL_VIEWS)
             .shared(true);
         if (supportRetryOnShardFailures(version) == false) {
             cluster.setting("cluster.routing.rebalance.enable", "none");

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
@@ -765,6 +766,38 @@ private static boolean includeCCSMetadata() {
         return randomBoolean();
     }
 
+    public void testRemoteViewFailsQuery() throws IOException {
+        assumeTrue("views not supported on remote cluster", capabilitiesSupportedNewAndOld(List.of("views_crud_as_index_actions")));
+        try (RestClient remoteClient = remoteClusterClient()) {
+            Request putView = new Request("PUT", "/_query/view/test-remote-view");
+            putView.setJsonEntity("{\"query\":\"FROM test-remote-index | LIMIT 10\"}");
+            assertOK(remoteClient.performRequest(putView));
+        }
+        try {
+            ResponseException e = expectThrows(
+                ResponseException.class,
+                () -> runEsql(new RestEsqlTestCase.RequestObjectBuilder().query("FROM remote_cluster:test-remote-*").build())
+            );
+            assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+            @SuppressWarnings("unchecked")
+            Map<String, Object> error = (Map<String, Object>) entityAsMap(e.getResponse()).get("error");
+            assertThat(error.get("type"), equalTo("remote_view_not_supported_exception"));
+            assertThat(
+                (String) error.get("reason"),
+                equalTo(
+                    "ES|QL queries with remote views are not supported. Matched [remote_cluster:test-remote-view]."
+                        + " Remove them from the query pattern or exclude them with"
+                        + " [remote_cluster:-test-remote-view] if matched by a wildcard."
+                )
+            );
+        } finally {
+            try (RestClient remoteClient = remoteClusterClient()) {
+                Request deleteView = new Request("DELETE", "/_query/view/test-remote-view");
+                remoteClient.performRequest(deleteView);
+            }
+        }
+    }
+
     public static class ClusterSettingToggle implements AutoCloseable {
         private final RestClient client;
         private final String settingKey;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterViewIT.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.metadata.View;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.esql.view.PutViewAction;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+
+public class CrossClusterViewIT extends AbstractCrossClusterTestCase {
+
+    @Before
+    public void setupClustersAndViews() throws IOException {
+        setupClusters(3);
+        createViewOnCluster(REMOTE_CLUSTER_1, "logs-web", "FROM logs-2 | LIMIT 10");
+        createViewOnCluster(REMOTE_CLUSTER_1, "logs-mobile", "FROM logs-2 | LIMIT 10");
+    }
+
+    public void testRemoteViewWildcardMatchFailsQuery() {
+        Exception e = expectThrows(Exception.class, () -> runQuery("FROM cluster-a:logs-*", null));
+        Throwable cause = ExceptionsHelper.unwrapCause(e);
+        assertThat(
+            cause.getMessage(),
+            containsString(
+                "ES|QL queries with remote views are not supported. Matched [cluster-a:logs-mobile, cluster-a:logs-web]."
+                    + " Remove them from the query pattern or exclude them with"
+                    + " [cluster-a:-logs-mobile,cluster-a:-logs-web] if matched by a wildcard."
+            )
+        );
+    }
+
+    public void testRemoteViewConcreteMatchFailsQuery() {
+        Exception e = expectThrows(Exception.class, () -> runQuery("FROM cluster-a:logs-web", null));
+        Throwable cause = ExceptionsHelper.unwrapCause(e);
+        assertThat(
+            cause.getMessage(),
+            containsString(
+                "ES|QL queries with remote views are not supported. Matched [cluster-a:logs-web]."
+                    + " Remove them from the query pattern or exclude them with [cluster-a:-logs-web] if matched by a wildcard."
+            )
+        );
+    }
+
+    public void testRemoteViewExcludedSucceeds() {
+        try (var resp = runQuery("FROM cluster-a:logs-*,cluster-a:-logs-web,cluster-a:-logs-mobile", null)) {
+            assertNotNull(resp);
+        }
+    }
+
+    public void testAllViewsOnRemoteExcludedSucceeds() {
+        try (var resp = runQuery("FROM cluster*:logs-*,-cluster-a:*,remote-b:*", null)) {
+            assertNotNull(resp);
+        }
+    }
+
+    public void testRemoteViewFailsOnOneCluster() {
+        Exception e = expectThrows(Exception.class, () -> runQuery("FROM cluster-a:logs-*,remote-b:logs-*", null));
+        Throwable cause = ExceptionsHelper.unwrapCause(e);
+        assertThat(
+            cause.getMessage(),
+            containsString(
+                "ES|QL queries with remote views are not supported. Matched [cluster-a:logs-mobile, cluster-a:logs-web]."
+                    + " Remove them from the query pattern or exclude them with"
+                    + " [cluster-a:-logs-mobile,cluster-a:-logs-web] if matched by a wildcard."
+            )
+        );
+    }
+
+    public void testNoRemoteViewsQuerySucceeds() {
+        try (var resp = runQuery("FROM remote-b:logs-*", null)) {
+            assertNotNull(resp);
+        }
+    }
+
+    private void createViewOnCluster(String clusterAlias, String viewName, String query) {
+        assertAcked(
+            client(clusterAlias).execute(
+                PutViewAction.INSTANCE,
+                new PutViewAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, new View(viewName, query))
+            ).actionGet(30, java.util.concurrent.TimeUnit.SECONDS)
+        );
+    }
+}