diff --git a/docs/changelog/139009.yaml b/docs/changelog/139009.yaml new file mode 100644 index 0000000000000..71a68deb7ff90 --- /dev/null +++ b/docs/changelog/139009.yaml @@ -0,0 +1,5 @@ +pr: 139009 +summary: Always error out if CCS expression shows up when CCS is not supported +area: CCS +type: bug +issues: [ 138987 ] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/70_remote.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/70_remote.yml new file mode 100644 index 0000000000000..d3500aaf0aed4 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/70_remote.yml @@ -0,0 +1,37 @@ +--- +setup: + - do: + indices.create: + index: test_1 + +--- +"Remote index mapping, should fail": + - requires: + cluster_features: [ "search.indices_boost_remote_index_fix" ] + reason: "indices_boost remote index fix must be present" + - do: + catch: /Cross-cluster calls are not supported in this context but remote indices were requested:/ + indices.get_mapping: + index: remote1:test_1 + +--- +"Remote and local index mapping, should fail": + - requires: + cluster_features: [ "search.indices_boost_remote_index_fix" ] + reason: "indices_boost remote index fix must be present" + - do: + catch: /Cross-cluster calls are not supported in this context but remote indices were requested:/ + indices.get_mapping: + index: test_1,remote1:test_1 + +--- +"Remote index mapping, ignore_unavailable=true, should fail": + - requires: + cluster_features: [ "search.indices_boost_remote_index_fix" ] + reason: "indices_boost remote index fix must be present" + - do: + catch: /Cross-cluster calls are not supported in this context but remote indices were requested:/ + indices.get_mapping: + ignore_unavailable: true + index: remote1:test_1 + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/40_indices_boost.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/40_indices_boost.yml index 9e6b4582d8f22..6cb30d66e6a3b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/40_indices_boost.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/40_indices_boost.yml @@ -151,3 +151,17 @@ setup: - match: { hits.total: 2} - match: { hits.hits.0._index: test_2 } - match: { hits.hits.1._index: test_1 } + +--- +"Remote expressions not supported in indices boost": + - requires: + cluster_features: ["search.indices_boost_remote_index_fix"] + reason: "indices_boost remote index fix must be present" + - do: + catch: /Cross-cluster calls are not supported in this context but remote indices were requested:/ + search: + rest_total_hits_as_int: true + ignore_unavailable: true + index: _all + body: + indices_boost: [{nonexistent: 2.0}, {remote1:test_1: 1.0}, {test_2: 2.0}] diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 78e6b5da9230c..833b356862cc7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -101,7 +101,10 @@ protected void masterOperation( request.indices() ); Map indicesAndFilters = new HashMap<>(); - Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(project.metadata(), request.indices()); + Set indicesAndAliases = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project.metadata(), + request.indices() + ); for (String index : concreteIndices) { final AliasFilter aliasFilter = indicesService.buildAliasFilter(project, index, indicesAndAliases); final String[] aliases = indexNameExpressionResolver.allIndexAliases(project.metadata(), index, indicesAndAliases); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 5de102c35b104..edf40783d3dc8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -783,6 +783,7 @@ static void resolveIndices( projectState.metadata(), indicesOptions, true, + false, names ); for (ResolvedExpression s : resolvedIndexAbstractions) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 6b5a4ffeaf14e..3227be0133595 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -152,7 +152,7 @@ private ProjectState getProjectState() { @Override protected ShardValidateQueryRequest newShardRequest(int numShards, ShardRouting shard, ValidateQueryRequest request) { final ProjectState projectState = getProjectState(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( projectState.metadata(), request.indices() ); diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 1a97373afbcaa..6fea177b888c5 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -128,7 +128,7 @@ protected boolean resolveIndex(ExplainRequest request) { @Override protected void resolveRequest(ProjectState state, InternalRequest request) { - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( state.metadata(), request.request().index() ); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 8134021f4962f..0f472fd9fe4d3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -1722,7 +1722,7 @@ private void executeSearch( } else { final Index[] indices = resolvedIndices.getConcreteLocalIndices(); concreteLocalIndices = Arrays.stream(indices).map(Index::getName).toArray(String[]::new); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( projectState.metadata(), searchRequest.indices() ); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java index 1a98d1d91c5ca..5d9b8801a9d29 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java @@ -144,7 +144,7 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act ), listener.delegateFailureAndWrap((delegate, searchRequest) -> { Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( project.metadata(), searchRequest.indices() ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 9417612491229..28bc7d1eb1f77 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -235,6 +235,12 @@ public String[] concreteIndexNames(ProjectMetadata project, IndicesOptions optio Context context = new Context( project, options, + System.currentTimeMillis(), + false, + false, + false, + false, + false, getSystemIndexAccessLevel(), getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() @@ -299,10 +305,12 @@ public List dataStreams(ProjectMetadata project, IndicesOpti Context context = new Context( project, options, + System.currentTimeMillis(), false, false, true, true, + false, getSystemIndexAccessLevel(), getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() @@ -463,7 +471,7 @@ public static InvalidIndexNameException invalidExclusion() { * Validates the requested expression by performing the following checks: * - Ensure it's not empty * - Ensure it doesn't start with `_` - * - Ensure it's not a remote expression unless the allow unavailable targets is enabled. + * - Ensure it's not a remote expression unless remotes are allowed by the context. */ private static void validateResourceExpression(Context context, boolean isExclusion, String current, String[] expressions) { if (Strings.isEmpty(current)) { @@ -476,17 +484,14 @@ private static void validateResourceExpression(Context context, boolean isExclus if (current.charAt(0) == '_') { throw new InvalidIndexNameException(current, "must not start with '_'."); } - ensureRemoteExpressionRequireIgnoreUnavailable(context.getOptions(), current, expressions); + checkForRemoteIndexExpression(context, current, expressions); } /** - * Throws an exception if the expression is a remote expression and we do not allow unavailable targets + * Throws an exception if the expression is a remote expression and we do not allow it. */ - private static void ensureRemoteExpressionRequireIgnoreUnavailable(IndicesOptions options, String current, String[] expressions) { - if (options.ignoreUnavailable()) { - return; - } - if (RemoteClusterAware.isRemoteIndexName(current)) { + private static void checkForRemoteIndexExpression(Context context, String current, String[] expressions) { + if (context.isAllowRemoteIndices() == false && RemoteClusterAware.isRemoteIndexName(current)) { List crossClusterIndices = RemoteClusterAware.getRemoteIndexExpressions(expressions); throw new IllegalArgumentException( "Cross-cluster calls are not supported in this context but remote indices were requested: " + crossClusterIndices @@ -588,6 +593,7 @@ public Index[] concreteIndices(ProjectMetadata project, IndicesRequest request, false, request.includeDataStreams(), false, + false, getSystemIndexAccessLevel(), getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() @@ -1006,8 +1012,8 @@ public static void assertExpressionHasNullOrDataSelector(String expression) { /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ - public Set resolveExpressions(ProjectMetadata project, String... expressions) { - return resolveExpressions(project, IndicesOptions.lenientExpandOpen(), false, expressions); + public Set resolveExpressionsIgnoringRemotes(ProjectMetadata project, String... expressions) { + return resolveExpressions(project, IndicesOptions.lenientExpandOpen(), false, true, expressions); } /** @@ -1019,15 +1025,18 @@ public Set resolveExpressions( ProjectMetadata project, IndicesOptions indicesOptions, boolean preserveDataStreams, + boolean allowRemoteIndices, String... expressions ) { Context context = new Context( project, indicesOptions, + System.currentTimeMillis(), true, false, true, preserveDataStreams, + allowRemoteIndices, getSystemIndexAccessLevel(), getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() @@ -1226,7 +1235,10 @@ public Map> resolveSearchRouting(ProjectMetadata project, @N Context context = new Context( project, IndicesOptions.lenientExpandOpen(), + System.currentTimeMillis(), + false, false, + true, false, true, getSystemIndexAccessLevel(), @@ -1534,52 +1546,24 @@ public static class Context { private final boolean resolveToWriteIndex; private final boolean includeDataStreams; private final boolean preserveDataStreams; + private final boolean allowRemoteIndices; private final SystemIndexAccessLevel systemIndexAccessLevel; private final Predicate systemIndexAccessPredicate; private final Predicate netNewSystemIndexPredicate; Context(ProjectMetadata project, IndicesOptions options, SystemIndexAccessLevel systemIndexAccessLevel) { - this(project, options, systemIndexAccessLevel, Predicates.always(), Predicates.never()); - } - - Context( - ProjectMetadata project, - IndicesOptions options, - SystemIndexAccessLevel systemIndexAccessLevel, - Predicate systemIndexAccessPredicate, - Predicate netNewSystemIndexPredicate - ) { - this( - project, - options, - System.currentTimeMillis(), - systemIndexAccessLevel, - systemIndexAccessPredicate, - netNewSystemIndexPredicate - ); - } - - Context( - ProjectMetadata project, - IndicesOptions options, - boolean preserveAliases, - boolean resolveToWriteIndex, - boolean includeDataStreams, - SystemIndexAccessLevel systemIndexAccessLevel, - Predicate systemIndexAccessPredicate, - Predicate netNewSystemIndexPredicate - ) { this( project, options, System.currentTimeMillis(), - preserveAliases, - resolveToWriteIndex, - includeDataStreams, + false, + false, + false, + false, false, systemIndexAccessLevel, - systemIndexAccessPredicate, - netNewSystemIndexPredicate + Predicates.always(), + Predicates.never() ); } @@ -1589,7 +1573,6 @@ public static class Context { boolean preserveAliases, boolean resolveToWriteIndex, boolean includeDataStreams, - boolean preserveDataStreams, SystemIndexAccessLevel systemIndexAccessLevel, Predicate systemIndexAccessPredicate, Predicate netNewSystemIndexPredicate @@ -1601,27 +1584,6 @@ public static class Context { preserveAliases, resolveToWriteIndex, includeDataStreams, - preserveDataStreams, - systemIndexAccessLevel, - systemIndexAccessPredicate, - netNewSystemIndexPredicate - ); - } - - Context( - ProjectMetadata project, - IndicesOptions options, - long startTime, - SystemIndexAccessLevel systemIndexAccessLevel, - Predicate systemIndexAccessPredicate, - Predicate netNewSystemIndexPredicate - ) { - this( - project, - options, - startTime, - false, - false, false, false, systemIndexAccessLevel, @@ -1638,6 +1600,7 @@ protected Context( boolean resolveToWriteIndex, boolean includeDataStreams, boolean preserveDataStreams, + boolean allowRemoteIndices, SystemIndexAccessLevel systemIndexAccessLevel, Predicate systemIndexAccessPredicate, Predicate netNewSystemIndexPredicate @@ -1649,6 +1612,7 @@ protected Context( this.resolveToWriteIndex = resolveToWriteIndex; this.includeDataStreams = includeDataStreams; this.preserveDataStreams = preserveDataStreams; + this.allowRemoteIndices = allowRemoteIndices; this.systemIndexAccessLevel = systemIndexAccessLevel; this.systemIndexAccessPredicate = systemIndexAccessPredicate; this.netNewSystemIndexPredicate = netNewSystemIndexPredicate; @@ -1691,6 +1655,10 @@ public boolean isPreserveDataStreams() { return preserveDataStreams; } + public boolean isAllowRemoteIndices() { + return allowRemoteIndices; + } + /** * Used to determine system index access is allowed in this context (e.g. for this request). */ diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java index 3cc3d2d15553b..ca832fea3b859 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java @@ -35,6 +35,7 @@ public Set getFeatures() { public static final NodeFeature SEARCH_WITH_NO_DIMENSIONS_BUGFIX = new NodeFeature("search.vectors.no_dimensions_bugfix"); public static final NodeFeature SEARCH_RESCORE_SCRIPT = new NodeFeature("search.rescore.script"); public static final NodeFeature NEGATIVE_FUNCTION_SCORE_BAD_REQUEST = new NodeFeature("search.negative.function.score.bad.request"); + public static final NodeFeature INDICES_BOOST_REMOTE_INDEX_FIX = new NodeFeature("search.indices_boost_remote_index_fix"); @Override public Set getTestFeatures() { @@ -47,7 +48,8 @@ public Set getTestFeatures() { BBQ_HNSW_DEFAULT_INDEXING, SEARCH_WITH_NO_DIMENSIONS_BUGFIX, SEARCH_RESCORE_SCRIPT, - NEGATIVE_FUNCTION_SCORE_BAD_REQUEST + NEGATIVE_FUNCTION_SCORE_BAD_REQUEST, + INDICES_BOOST_REMOTE_INDEX_FIX ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index d9a2d885b82a6..197371404a31b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -330,6 +330,7 @@ public void testResolveHiddenProperlyWithDateMath() { project, IndicesOptions.LENIENT_EXPAND_OPEN, true, + false, requestedIndex ); assertThat(resolvedIndices.size(), is(1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 840083f1a6261..1a599683c8052 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -1610,7 +1610,7 @@ public void testFilterClosedIndicesOnAliases() { assertArrayEquals(new String[] { "test-0" }, strings); } - public void testResolveExpressions() { + public void testResolveExpressionsIgnoringRemotes() { ProjectMetadata project = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID) .put(indexBuilder("test-0").state(State.OPEN).putAlias(AliasMetadata.builder("alias-0").filter("{ \"term\": \"foo\"}"))) .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1"))) @@ -1618,7 +1618,7 @@ public void testResolveExpressions() { assertEquals( Set.of(new ResolvedExpression("alias-0", DATA), new ResolvedExpression("alias-1", DATA)), - indexNameExpressionResolver.resolveExpressions(project, "alias-*") + indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "alias-*") ); assertEquals( Set.of( @@ -1626,7 +1626,7 @@ public void testResolveExpressions() { new ResolvedExpression("alias-0", DATA), new ResolvedExpression("alias-1", DATA) ), - indexNameExpressionResolver.resolveExpressions(project, "test-0", "alias-*") + indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "test-0", "alias-*") ); assertEquals( Set.of( @@ -1635,11 +1635,11 @@ public void testResolveExpressions() { new ResolvedExpression("alias-0", DATA), new ResolvedExpression("alias-1", DATA) ), - indexNameExpressionResolver.resolveExpressions(project, "test-*", "alias-*") + indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "test-*", "alias-*") ); assertEquals( Set.of(new ResolvedExpression("test-1", DATA), new ResolvedExpression("alias-1", DATA)), - indexNameExpressionResolver.resolveExpressions(project, "*-1") + indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "*-1") ); } @@ -1681,7 +1681,7 @@ public void testIndexAliases() { .putAlias(AliasMetadata.builder("test-alias-non-filtering")) ) .build(); - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "test-*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "test-*"); String[] strings = indexNameExpressionResolver.allIndexAliases(project, "test-0", resolvedExpressions); Arrays.sort(strings); @@ -1716,21 +1716,21 @@ public void testIndexAliasesDataStreamAliases() { ProjectMetadata project = projectBuilder.build(); { // Only resolve aliases that refer to dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_foo", "logs", "logs_bar")); } { // Only resolve aliases that refer to dataStreamName2 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_baz", "logs_baz2")); } { // Null is returned, because skipping identity check and resolvedExpressions contains the backing index name - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( project, @@ -1744,28 +1744,40 @@ public void testIndexAliasesDataStreamAliases() { } { // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(project, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned, because an unfiltered alias is targeting the same data stream - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "logs_bar", "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "logs_bar", + "logs" + ); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned because we target the data stream name and skipIdentity is false - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + dataStreamName1, + "logs" + ); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // The filtered alias is returned because although we target the data stream name, skipIdentity is true - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + dataStreamName1, + "logs" + ); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( project, @@ -1799,21 +1811,30 @@ public void testIndexAliasesDataStreamFailureStoreAndAliases() { ProjectMetadata project = projectBuilder.build(); { // Resolving the failure component with a backing index should return null - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "l*::failures" + ); String index = randomBoolean() ? backingIndex1.getIndex().getName() : backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // Only resolve aliases that refer to dataStreamName1 failure store - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "l*::failures" + ); String index = failureIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.allIndexAliases(project, index, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_foo::failures", "logs::failures", "logs_bar::failures")); } { // Null is returned, because we perform the identity check and resolvedExpressions contains the failure index name - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "l*::failures" + ); String index = failureIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( project, @@ -1827,14 +1848,20 @@ public void testIndexAliasesDataStreamFailureStoreAndAliases() { } { // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "l*::failures"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "l*::failures" + ); String index = failureIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned because we target the failure store of the data stream - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(project, "logs::failures"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressionsIgnoringRemotes( + project, + "logs::failures" + ); String index = failureIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.filteringAliases(project, index, resolvedExpressions); assertThat(result, nullValue()); @@ -3335,6 +3362,11 @@ public void testDateMathMixedArray() { .build(), IndicesOptions.strictExpand(), now, + false, + false, + false, + false, + false, SystemIndexAccessLevel.NONE, Predicates.never(), Predicates.never() @@ -3374,16 +3406,29 @@ public void testMathExpressionSupportWithOlderDate() { assertEquals(resolved, "older-date-2020-12"); } + private IndexNameExpressionResolver.Context makeRandomContext(IndicesOptions options, boolean allowRemotes) { + ProjectMetadata project = ProjectMetadata.builder(randomUniqueProjectId()).build(); + return new IndexNameExpressionResolver.Context( + project, + options, + System.currentTimeMillis(), + false, + false, + false, + false, + allowRemotes, + SystemIndexAccessLevel.NONE, + Predicates.always(), + Predicates.never() + ); + } + public void testRemoteIndex() { ProjectMetadata project = ProjectMetadata.builder(randomUniqueProjectId()).build(); { IndicesOptions options = IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); - IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context( - project, - options, - SystemIndexAccessLevel.NONE - ); + IndexNameExpressionResolver.Context context = makeRandomContext(options, false); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> indexNameExpressionResolver.concreteIndexNames(context, "cluster:index", "local") @@ -3401,14 +3446,22 @@ public void testRemoteIndex() { } { IndicesOptions options = IndicesOptions.fromOptions(true, true, randomBoolean(), randomBoolean(), randomBoolean()); - IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context( - project, - options, - SystemIndexAccessLevel.NONE - ); + IndexNameExpressionResolver.Context context = makeRandomContext(options, true); String[] indexNames = indexNameExpressionResolver.concreteIndexNames(context, "cluster:index", "local"); assertEquals(0, indexNames.length); } + { + IndicesOptions options = IndicesOptions.fromOptions(true, true, randomBoolean(), randomBoolean(), randomBoolean()); + IndexNameExpressionResolver.Context context = makeRandomContext(options, false); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteIndexNames(context, "cluster:index", "local") + ); + assertEquals( + "Cross-cluster calls are not supported in this context but remote indices were requested: [cluster:index]", + iae.getMessage() + ); + } } public void testResolveWriteIndexAbstraction() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index dcfdd03fed9c1..984af52d2594c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -290,7 +290,7 @@ private void doLookup(T request, CancellableTask task, ActionListener AliasFilter aliasFilter = indicesService.buildAliasFilter( projectState, request.shardId.getIndex().getName(), - indexNameExpressionResolver.resolveExpressions(projectState.metadata(), request.indexPattern) + indexNameExpressionResolver.resolveExpressionsIgnoringRemotes(projectState.metadata(), request.indexPattern) ); LookupShardContext shardContext = lookupShardContextFactory.create(request.shardId); diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/search/ccs/AbstractSemanticCrossClusterSearchTestCase.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/search/ccs/AbstractSemanticCrossClusterSearchTestCase.java index 613eabb3d50b8..48c67ab746dda 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/search/ccs/AbstractSemanticCrossClusterSearchTestCase.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/search/ccs/AbstractSemanticCrossClusterSearchTestCase.java @@ -37,6 +37,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteConnectionInfo; import org.elasticsearch.xpack.inference.FakeMlPlugin; import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; @@ -275,7 +276,7 @@ protected static Map sparseVectorMapping() { } protected static String fullyQualifiedIndexName(String clusterAlias, String indexName) { - return clusterAlias + ":" + indexName; + return RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexName); } protected static float[] generateDenseVectorFieldValue(int dimensions, DenseVectorFieldMapper.ElementType elementType, float value) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index ceb868aa74ba9..d43c5e8a01243 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.search.crossproject.CrossProjectIndexResolutionValidator; import org.elasticsearch.transport.NoSuchRemoteClusterException; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; @@ -742,22 +743,36 @@ public void resolveAsSeparateMappings( } } client.fieldCaps(fieldRequest, listener.delegateFailureAndWrap((delegate, response) -> { - client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> { - delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); - }, ex -> { - if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); - } else { - delegate.onFailure(ex); - } - })); + GetAliasesRequest request = createGetAliasesRequest(response, includeFrozen); + if (request.indices().length == 0) { + delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); + } else { + client.admin().indices().getAliases(request, wrap(aliases -> { + delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); + }, ex -> { + if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { + delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); + } else { + delegate.onFailure(ex); + } + })); + } })); } + /** + * Filter out remote index expressions. + * TODO: SQL Metadata commands currently do not support remote indices. + * See also: showTablesIdentifierPatternOnAliases-Ignore + */ + private static String[] onlyLocalIndices(String[] indices) { + return Arrays.stream(indices).filter(i -> RemoteClusterAware.isRemoteIndexName(i) == false).toArray(String[]::new); + } + private static GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse response, boolean includeFrozen) { return new GetAliasesRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT).aliases("*") - .indices(response.getIndices()) + .indices(onlyLocalIndices(response.getIndices())) .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java index f5a6f510765b5..85e2e81799969 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java @@ -21,6 +21,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.SecurityContext; @@ -36,6 +37,7 @@ import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.persistence.TransformIndex; +import java.util.Arrays; import java.util.Map; /** @@ -335,17 +337,19 @@ private static void updateTransformConfiguration( final String destinationIndex = config.getDestination().getIndex(); String[] dest = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), destinationIndex); - String[] src = indexNameExpressionResolver.concreteIndexNames( - clusterState, - IndicesOptions.lenientExpandOpen(), - true, - config.getSource().getIndex() - ); + // FIXME: what do we do with remote indices? + String[] sourceIndices = Arrays.stream(config.getSource().getIndex()) + .filter(ind -> RemoteClusterAware.isRemoteIndexName(ind) == false) + .toArray(String[]::new); + String[] src = sourceIndices.length > 0 + ? indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), true, sourceIndices) + : null; // If we are running, we should verify that the destination index exists and create it if it does not if (PersistentTasksCustomMetadata.getTaskWithId(clusterState, config.getId()) != null && dest.length == 0 // Verify we have source indices. The user could defer_validations and if the task is already running // we allow source indices to disappear. If the source and destination indices do not exist, don't do anything // the transform will just have to dynamically create the destination index without special mapping. + && src != null && src.length > 0) { TransformIndex.createDestinationIndex( client,