From c92b0cfa8beb7eb4c58740ef6084e5d913031bca Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 19 Feb 2020 17:45:49 -0700 Subject: [PATCH 01/14] Implement hidden aliases This commit introduces hidden aliases. These are similar to hidden indices, in that they are not visible by default, unless explicitly specified by name or by indicating that hidden indices/aliases are desired. The new alias property, `is_hidden` is implemented similarly to `is_write_index`, except that it must be consistent across all indices with a given alias - that is, all indices with a given alias must specify the alias as either hidden, or all specify it as non-hidden, either explicitly or by omitting the `is_hidden` property. --- .../action/admin/indices/alias/Alias.java | 29 ++++ .../cluster/metadata/AliasMetaData.java | 52 +++++-- .../cluster/metadata/AliasOrIndex.java | 42 +++++- .../cluster/metadata/MetaData.java | 2 +- .../metadata/MetaDataCreateIndexService.java | 3 +- .../cluster/metadata/AliasMetaDataTests.java | 6 + .../cluster/metadata/AliasOrIndexTests.java | 130 ++++++++++++++++++ 7 files changed, 253 insertions(+), 11 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java index f118441771839..93749f92e9da9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -49,6 +50,7 @@ public class Alias implements Writeable, ToXContentFragment { private static final ParseField INDEX_ROUTING = new ParseField("index_routing", "indexRouting", "index-routing"); private static final ParseField SEARCH_ROUTING = new ParseField("search_routing", "searchRouting", "search-routing"); private static final ParseField IS_WRITE_INDEX = new ParseField("is_write_index"); + private static final ParseField IS_HIDDEN = new ParseField("is_hidden"); private String name; @@ -64,12 +66,18 @@ public class Alias implements Writeable, ToXContentFragment { @Nullable private Boolean writeIndex; + @Nullable + private Boolean isHidden; + public Alias(StreamInput in) throws IOException { name = in.readString(); filter = in.readOptionalString(); indexRouting = in.readOptionalString(); searchRouting = in.readOptionalString(); writeIndex = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + isHidden = in.readOptionalBoolean(); + } } public Alias(String name) { @@ -189,6 +197,21 @@ public Alias writeIndex(@Nullable Boolean writeIndex) { return this; } + /** + * @return whether this alias is hidden or not + */ + public Boolean isHidden() { + return isHidden; + } + + /** + * Sets whether this alias is hidden + */ + public Alias isHidden(@Nullable Boolean isHidden) { + this.isHidden = isHidden; + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); @@ -196,6 +219,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(indexRouting); out.writeOptionalString(searchRouting); out.writeOptionalBoolean(writeIndex); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalBoolean(isHidden); + } } /** @@ -228,6 +254,8 @@ public static Alias fromXContent(XContentParser parser) throws IOException { } else if (token == XContentParser.Token.VALUE_BOOLEAN) { if (IS_WRITE_INDEX.match(currentFieldName, parser.getDeprecationHandler())) { alias.writeIndex(parser.booleanValue()); + } else if (IS_HIDDEN.match(currentFieldName, parser.getDeprecationHandler())) { + alias.isHidden(parser.booleanValue()); } } } @@ -256,6 +284,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(IS_WRITE_INDEX.getPreferredName(), writeIndex); + builder.field(IS_HIDDEN.getPreferredName(), isHidden); builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 478e76b525610..f533e67508350 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; @@ -40,6 +41,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import java.util.Objects; import java.util.Set; import static java.util.Collections.emptySet; @@ -59,7 +61,11 @@ public class AliasMetaData extends AbstractDiffable implements To @Nullable private final Boolean writeIndex; - private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex) { + @Nullable + private final Boolean isHidden; + + private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex, + Boolean isHidden) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -70,10 +76,12 @@ private AliasMetaData(String alias, CompressedXContent filter, String indexRouti searchRoutingValues = emptySet(); } this.writeIndex = writeIndex; + this.isHidden = isHidden; } private AliasMetaData(AliasMetaData aliasMetaData, String alias) { - this(alias, aliasMetaData.filter(), aliasMetaData.indexRouting(), aliasMetaData.searchRouting(), aliasMetaData.writeIndex()); + this(alias, aliasMetaData.filter(), aliasMetaData.indexRouting(), aliasMetaData.searchRouting(), aliasMetaData.writeIndex(), + aliasMetaData.isHidden); } public String alias() { @@ -120,6 +128,10 @@ public Boolean writeIndex() { return writeIndex; } + public Boolean isHidden() { + return isHidden; + } + public static Builder builder(String alias) { return new Builder(alias); } @@ -142,11 +154,12 @@ public boolean equals(Object o) { final AliasMetaData that = (AliasMetaData) o; - if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false; - if (filter != null ? !filter.equals(that.filter) : that.filter != null) return false; - if (indexRouting != null ? !indexRouting.equals(that.indexRouting) : that.indexRouting != null) return false; - if (searchRouting != null ? !searchRouting.equals(that.searchRouting) : that.searchRouting != null) return false; - if (writeIndex != null ? writeIndex != that.writeIndex : that.writeIndex != null) return false; + if (Objects.equals(alias, that.alias) == false) return false; + if (Objects.equals(filter, that.filter) == false) return false; + if (Objects.equals(indexRouting, that.indexRouting) == false) return false; + if (Objects.equals(searchRouting, that.searchRouting) == false) return false; + if (Objects.equals(writeIndex, that.writeIndex) == false) return false; + if (Objects.equals(isHidden, that.isHidden) == false) return false; return true; } @@ -183,6 +196,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeOptionalBoolean(writeIndex()); + + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalBoolean(isHidden); + } } public AliasMetaData(StreamInput in) throws IOException { @@ -205,6 +222,12 @@ public AliasMetaData(StreamInput in) throws IOException { searchRoutingValues = emptySet(); } writeIndex = in.readOptionalBoolean(); + + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + isHidden = in.readOptionalBoolean(); + } else { + isHidden = null; + } } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -235,6 +258,8 @@ public static class Builder { @Nullable private Boolean writeIndex; + @Nullable + private Boolean isHidden; public Builder(String alias) { this.alias = alias; @@ -292,8 +317,13 @@ public Builder writeIndex(@Nullable Boolean writeIndex) { return this; } + public Builder isHidden(@Nullable Boolean isHidden) { + this.isHidden = isHidden; + return this; + } + public AliasMetaData build() { - return new AliasMetaData(alias, filter, indexRouting, searchRouting, writeIndex); + return new AliasMetaData(alias, filter, indexRouting, searchRouting, writeIndex, isHidden); } public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -319,6 +349,10 @@ public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder build builder.field("is_write_index", aliasMetaData.writeIndex()); } + if (aliasMetaData.isHidden != null) { + builder.field("is_hidden", aliasMetaData.isHidden()); + } + builder.endObject(); } @@ -358,6 +392,8 @@ public static AliasMetaData fromXContent(XContentParser parser) throws IOExcepti } else if (token == XContentParser.Token.VALUE_BOOLEAN) { if ("is_write_index".equals(currentFieldName)) { builder.writeIndex(parser.booleanValue()); + } else if ("is_hidden".equals(currentFieldName)) { + builder.isHidden(parser.booleanValue()); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java index 22266b8dbfcbd..166df92a0a838 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java @@ -28,8 +28,12 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_HIDDEN_SETTING; + /** * Encapsulates the {@link IndexMetaData} instances of a concrete index or indices an alias is pointing to. */ @@ -46,6 +50,11 @@ public interface AliasOrIndex { */ List getIndices(); + /** + * @return whether this alias/index is hidden or not + */ + Boolean isHidden(); + /** * Represents an concrete index and encapsulates its {@link IndexMetaData} */ @@ -66,6 +75,11 @@ public boolean isAlias() { public List getIndices() { return Collections.singletonList(concreteIndex); } + + @Override @Nullable + public Boolean isHidden() { + return concreteIndex.getSettings().getAsBoolean(INDEX_HIDDEN_SETTING.getKey(), null); + } } /** @@ -76,11 +90,13 @@ class Alias implements AliasOrIndex { private final String aliasName; private final List referenceIndexMetaDatas; private final SetOnce writeIndex = new SetOnce<>(); + private final Boolean isHidden; public Alias(AliasMetaData aliasMetaData, IndexMetaData indexMetaData) { this.aliasName = aliasMetaData.getAlias(); this.referenceIndexMetaDatas = new ArrayList<>(); this.referenceIndexMetaDatas.add(indexMetaData); + this.isHidden = aliasMetaData.isHidden(); } @Override @@ -103,6 +119,11 @@ public IndexMetaData getWriteIndex() { return writeIndex.get(); } + @Override @Nullable + public Boolean isHidden() { + return isHidden; + } + /** * Returns the unique alias metadata per concrete index. * @@ -135,7 +156,8 @@ void addIndex(IndexMetaData indexMetaData) { this.referenceIndexMetaDatas.add(indexMetaData); } - public void computeAndValidateWriteIndex() { + public void computeAndValidateAliasProperties() { + // Validate write indices List writeIndices = referenceIndexMetaDatas.stream() .filter(idxMeta -> Boolean.TRUE.equals(idxMeta.getAliases().get(aliasName).writeIndex())) .collect(Collectors.toList()); @@ -153,6 +175,24 @@ public void computeAndValidateWriteIndex() { throw new IllegalStateException("alias [" + aliasName + "] has more than one write index [" + Strings.collectionToCommaDelimitedString(writeIndicesStrings) + "]"); } + + // Validate hidden status + final Map> groupedByHiddenStatus = referenceIndexMetaDatas.stream() + .collect(Collectors.groupingBy(idxMeta -> Boolean.TRUE.equals(idxMeta.getAliases().get(aliasName).isHidden()))); + if (isNonEmpty(groupedByHiddenStatus.get(true)) && isNonEmpty(groupedByHiddenStatus.get(false))) { + List hiddenOn = groupedByHiddenStatus.get(true).stream() + .map(idx -> idx.getIndex().getName()).collect(Collectors.toList()); + List nonHiddenOn = groupedByHiddenStatus.get(false).stream() + .map(idx -> idx.getIndex().getName()).collect(Collectors.toList()); + throw new IllegalStateException("alias [" + aliasName + "] is hidden set as on [" + + Strings.collectionToCommaDelimitedString(hiddenOn) + "] but as NOT hidden on [" + + Strings.collectionToCommaDelimitedString(nonHiddenOn) + "]; alias must have same hidden setting " + + "on all indices"); + } + } + + private boolean isNonEmpty(List idxMetas) { + return (Objects.isNull(idxMetas) || idxMetas.isEmpty()) == false; } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 02dca032a11c2..00d3a8d708d62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -1266,7 +1266,7 @@ private SortedMap buildAliasAndIndexLookup() { } } aliasAndIndexLookup.values().stream().filter(AliasOrIndex::isAlias) - .forEach(alias -> ((AliasOrIndex.Alias) alias).computeAndValidateWriteIndex()); + .forEach(alias -> ((AliasOrIndex.Alias) alias).computeAndValidateAliasProperties()); return aliasAndIndexLookup; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 9bac4976be212..030d29f80699e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -553,7 +553,8 @@ static List resolveAndValidateAliases(String index, Set al aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext, xContentRegistry); } AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter()) - .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).writeIndex(alias.writeIndex()).build(); + .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).writeIndex(alias.writeIndex()) + .isHidden(alias.isHidden()).build(); resolvedAliases.add(aliasMetaData); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java index de23c560eb9af..321a8070b1613 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java @@ -42,6 +42,7 @@ public void testSerialization() throws IOException { .routing("routing") .searchRouting("trim,tw , ltw , lw") .writeIndex(randomBoolean() ? null : randomBoolean()) + .isHidden(randomBoolean() ? null : randomBoolean()) .build(); assertThat(before.searchRoutingValues(), equalTo(Sets.newHashSet("trim", "tw ", " ltw ", " lw"))); @@ -64,6 +65,7 @@ protected void assertEqualInstances(AliasMetaData expectedInstance, AliasMetaDat .indexRouting(expectedInstance.indexRouting()) .searchRouting(expectedInstance.searchRouting()) .writeIndex(randomBoolean() ? null : randomBoolean()) + .isHidden(randomBoolean() ? null : randomBoolean()) .build(); } assertEquals(expectedInstance, newInstance); @@ -112,6 +114,10 @@ private static AliasMetaData createTestItem() { builder.filter("{\"term\":{\"year\":2016}}"); } builder.writeIndex(randomBoolean()); + + if (randomBoolean()) { + builder.isHidden(randomBoolean()); + } return builder.build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java new file mode 100644 index 0000000000000..11044476e0173 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java @@ -0,0 +1,130 @@ +/* + * + * * Licensed to Elasticsearch under one or more contributor + * * license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright + * * ownership. Elasticsearch licenses this file to you under + * * the Apache License, Version 2.0 (the "License"); you may + * * not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.test.ESTestCase; + +import java.util.Objects; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; + +public class AliasOrIndexTests extends ESTestCase { + + public void testHiddenAliasValidation() { + final String hiddenAliasName = "hidden_alias"; + AliasMetaData hiddenAliasMetadata = new AliasMetaData.Builder(hiddenAliasName).isHidden(true).build(); + + IndexMetaData hidden1 = buildIndexWithAlias("hidden1", hiddenAliasName, true); + IndexMetaData hidden2 = buildIndexWithAlias("hidden2", hiddenAliasName, true); + IndexMetaData hidden3 = buildIndexWithAlias("hidden3", hiddenAliasName, true); + + IndexMetaData indexWithNonHiddenAlias = buildIndexWithAlias("nonhidden1", hiddenAliasName, false); + IndexMetaData indexWithUnspecifiedAlias = buildIndexWithAlias("nonhidden2", hiddenAliasName, null); + + { + AliasOrIndex.Alias allHidden = new AliasOrIndex.Alias(hiddenAliasMetadata, hidden1); + allHidden.addIndex(hidden2); + allHidden.addIndex(hidden3); + allHidden.computeAndValidateAliasProperties(); // Should be ok + } + + { + AliasOrIndex.Alias allVisible; + if (randomBoolean()) { + allVisible = new AliasOrIndex.Alias(hiddenAliasMetadata, indexWithNonHiddenAlias); + allVisible.addIndex(indexWithUnspecifiedAlias); + } else { + allVisible = new AliasOrIndex.Alias(hiddenAliasMetadata, indexWithUnspecifiedAlias); + allVisible.addIndex(indexWithNonHiddenAlias); + } + + allVisible.computeAndValidateAliasProperties(); // Should be ok + } + + { + AliasOrIndex.Alias oneNonHidden = new AliasOrIndex.Alias(hiddenAliasMetadata, hidden1); + oneNonHidden.addIndex(hidden2); + oneNonHidden.addIndex(hidden3); + oneNonHidden.addIndex(indexWithNonHiddenAlias); + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> oneNonHidden.computeAndValidateAliasProperties()); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [")); + assertThat(exception.getMessage(), allOf(containsString(hidden1.getIndex().getName()), + containsString(hidden2.getIndex().getName()), + containsString(hidden3.getIndex().getName()))); + assertThat(exception.getMessage(), containsString("but as NOT hidden on [" + + indexWithNonHiddenAlias.getIndex().getName() + "]; alias must have same hidden setting on all indices")); + } + + { + AliasOrIndex.Alias oneUnspecified = new AliasOrIndex.Alias(hiddenAliasMetadata, hidden1); + oneUnspecified.addIndex(hidden2); + oneUnspecified.addIndex(hidden3); + oneUnspecified.addIndex(indexWithUnspecifiedAlias); + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> oneUnspecified.computeAndValidateAliasProperties()); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [")); + assertThat(exception.getMessage(), allOf(containsString(hidden1.getIndex().getName()), + containsString(hidden2.getIndex().getName()), + containsString(hidden3.getIndex().getName()))); + assertThat(exception.getMessage(), containsString("but as NOT hidden on [" + + indexWithUnspecifiedAlias.getIndex().getName() + "]; alias must have same hidden setting on all indices")); + } + + { + AliasOrIndex.Alias mostlyVisibleOneHidden; + if (randomBoolean()) { + mostlyVisibleOneHidden = new AliasOrIndex.Alias(hiddenAliasMetadata, indexWithNonHiddenAlias); + mostlyVisibleOneHidden.addIndex(indexWithUnspecifiedAlias); + } else { + mostlyVisibleOneHidden = new AliasOrIndex.Alias(hiddenAliasMetadata, indexWithUnspecifiedAlias); + mostlyVisibleOneHidden.addIndex(indexWithNonHiddenAlias); + } + final IndexMetaData hiddenIndex = randomFrom(hidden1, hidden2, hidden3); + mostlyVisibleOneHidden.addIndex(hiddenIndex); + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> mostlyVisibleOneHidden.computeAndValidateAliasProperties()); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [" + + hiddenIndex.getIndex().getName() + "] but as NOT hidden on [")); + assertThat(exception.getMessage(), allOf(containsString(indexWithUnspecifiedAlias.getIndex().getName()), + containsString(indexWithNonHiddenAlias.getIndex().getName()))); + assertThat(exception.getMessage(), containsString("but as NOT hidden on [")); + } + } + + private IndexMetaData buildIndexWithAlias(String indexName, String aliasName, @Nullable Boolean aliasIsHidden) { + final AliasMetaData.Builder aliasMetaData = new AliasMetaData.Builder(aliasName); + if (Objects.nonNull(aliasIsHidden) || randomBoolean()) { + aliasMetaData.isHidden(aliasIsHidden); + } + return new IndexMetaData.Builder(indexName) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasMetaData) + .build(); + } + +} From 522bba68dcab4b657dea27eb615abc8c09862cd1 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 24 Feb 2020 17:18:31 -0700 Subject: [PATCH 02/14] AliasOrIndex.isHidden -> boolean --- .../cluster/metadata/AliasOrIndex.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java index 166df92a0a838..c81e159e7b27c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java @@ -53,7 +53,7 @@ public interface AliasOrIndex { /** * @return whether this alias/index is hidden or not */ - Boolean isHidden(); + boolean isHidden(); /** * Represents an concrete index and encapsulates its {@link IndexMetaData} @@ -76,9 +76,9 @@ public List getIndices() { return Collections.singletonList(concreteIndex); } - @Override @Nullable - public Boolean isHidden() { - return concreteIndex.getSettings().getAsBoolean(INDEX_HIDDEN_SETTING.getKey(), null); + @Override + public boolean isHidden() { + return INDEX_HIDDEN_SETTING.get(concreteIndex.getSettings()); } } @@ -90,13 +90,13 @@ class Alias implements AliasOrIndex { private final String aliasName; private final List referenceIndexMetaDatas; private final SetOnce writeIndex = new SetOnce<>(); - private final Boolean isHidden; + private final boolean isHidden; public Alias(AliasMetaData aliasMetaData, IndexMetaData indexMetaData) { this.aliasName = aliasMetaData.getAlias(); this.referenceIndexMetaDatas = new ArrayList<>(); this.referenceIndexMetaDatas.add(indexMetaData); - this.isHidden = aliasMetaData.isHidden(); + this.isHidden = aliasMetaData.isHidden() == null ? false : aliasMetaData.isHidden(); } @Override @@ -119,8 +119,8 @@ public IndexMetaData getWriteIndex() { return writeIndex.get(); } - @Override @Nullable - public Boolean isHidden() { + @Override + public boolean isHidden() { return isHidden; } From b36f53eeb045f427b339aa1c0d161cdf2975d9fd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 25 Feb 2020 16:55:26 -0700 Subject: [PATCH 03/14] Only include isHidden in alias XContent if present --- .../org/elasticsearch/action/admin/indices/alias/Alias.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java index 93749f92e9da9..62ac8ea4dc9fe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java @@ -284,7 +284,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(IS_WRITE_INDEX.getPreferredName(), writeIndex); - builder.field(IS_HIDDEN.getPreferredName(), isHidden); + + if (isHidden != null) { + builder.field(IS_HIDDEN.getPreferredName(), isHidden); + } builder.endObject(); return builder; From bf118884c6ea42a72964a1c52bd624e834699ea1 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 25 Feb 2020 17:02:43 -0700 Subject: [PATCH 04/14] Update alias docs --- docs/reference/indices/aliases.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 6899fb2581c17..b39bbaef43639 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -106,6 +106,13 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=index-alias-filter] + See <> for an example. +`is_hidden`:: +(Optional, boolean) +If `true`, the alias will be excluded from wildcard expressions by default, +unless overriden in the request using the `expand_wildcards` parameter, +similar to <>. This property must be set to the +same value on all indices that share an alias. Defaults to `false`. + `is_write_index`:: (Optional, boolean) If `true`, assigns the index as an alias's write index. From d3985605fb6689a67a38fc3eced952b04525e8af Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 25 Feb 2020 17:37:02 -0700 Subject: [PATCH 05/14] Handle hidden indices in standard resolver --- .../metadata/IndexNameExpressionResolver.java | 26 ++-- .../elasticsearch/aliases/IndexAliasesIT.java | 21 ++- .../IndexNameExpressionResolverTests.java | 139 ++++++++++++++++++ 3 files changed, 171 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index d68d69bf24e8c..b795f662eb1ef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -721,7 +721,7 @@ private Set innerResolve(Context context, List expressions, Indi // add all the previous ones... result = new HashSet<>(expressions.subList(0, i)); } - if (!Regex.isSimpleMatchPattern(expression)) { + if (Regex.isSimpleMatchPattern(expression) == false) { //TODO why does wildcard resolver throw exceptions regarding non wildcarded expressions? This should not be done here. if (options.ignoreUnavailable() == false) { AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression); @@ -840,27 +840,29 @@ private static Set expand(Context context, IndexMetaData.State excludeSt String expression, boolean includeHidden) { Set expand = new HashSet<>(); for (Map.Entry entry : matches.entrySet()) { + String aliasOrIndexName = entry.getKey(); AliasOrIndex aliasOrIndex = entry.getValue(); - if (context.isPreserveAliases() && aliasOrIndex.isAlias()) { - expand.add(entry.getKey()); - } else { - for (IndexMetaData meta : aliasOrIndex.getIndices()) { - if (excludeState == null || meta.getState() != excludeState) { - if (includeHidden) { - expand.add(meta.getIndex().getName()); - } else if (IndexMetaData.INDEX_HIDDEN_SETTING.get(meta.getSettings()) == false) { - expand.add(meta.getIndex().getName()); - } else if (meta.getIndex().getName().startsWith(".") && - expression.startsWith(".") && Regex.isSimpleMatchPattern(expression)) { + + if (aliasOrIndex.isHidden() == false || includeHidden || implicitHiddenMatch(aliasOrIndexName, expression)) { + if (context.isPreserveAliases() && aliasOrIndex.isAlias()) { + expand.add(aliasOrIndexName); + } else { + for (IndexMetaData meta : aliasOrIndex.getIndices()) { + if (excludeState == null || meta.getState() != excludeState) { expand.add(meta.getIndex().getName()); } } + } } } return expand; } + private static boolean implicitHiddenMatch(String itemName, String expression) { + return itemName.startsWith(".") && expression.startsWith(".") && Regex.isSimpleMatchPattern(expression); + } + private boolean isEmptyOrTrivialWildcard(List expressions) { return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0)) || Regex.isMatchAllPattern(expressions.get(0)))); diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index aa28c27d1a21a..557e7e09f7db2 100644 --- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -909,7 +909,8 @@ public void testCreateIndexWithAliases() { .setMapping("field", "type=text") .addAlias(new Alias("alias1")) .addAlias(new Alias("alias2").filter(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("field")))) - .addAlias(new Alias("alias3").indexRouting("index").searchRouting("search"))); + .addAlias(new Alias("alias3").indexRouting("index").searchRouting("search")) + .addAlias(new Alias("alias4").isHidden(true))); checkAliases(); } @@ -919,7 +920,8 @@ public void testCreateIndexWithAliasesInSource() throws Exception { " \"aliases\" : {\n" + " \"alias1\" : {},\n" + " \"alias2\" : {\"filter\" : {\"match_all\": {}}},\n" + - " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" + + " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"},\n" + + " \"alias4\" : {\"is_hidden\": true}\n" + " }\n" + "}", XContentType.JSON)); @@ -932,7 +934,8 @@ public void testCreateIndexWithAliasesSource() { .setAliases("{\n" + " \"alias1\" : {},\n" + " \"alias2\" : {\"filter\" : {\"term\": {\"field\":\"value\"}}},\n" + - " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" + + " \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"},\n" + + " \"alias4\" : {\"is_hidden\": true}\n" + "}")); checkAliases(); @@ -1127,6 +1130,7 @@ private void checkAliases() { assertThat(aliasMetaData.filter(), nullValue()); assertThat(aliasMetaData.indexRouting(), nullValue()); assertThat(aliasMetaData.searchRouting(), nullValue()); + assertThat(aliasMetaData.isHidden(), nullValue()); getAliasesResponse = admin().indices().prepareGetAliases("alias2").get(); assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1)); @@ -1135,6 +1139,7 @@ private void checkAliases() { assertThat(aliasMetaData.filter(), notNullValue()); assertThat(aliasMetaData.indexRouting(), nullValue()); assertThat(aliasMetaData.searchRouting(), nullValue()); + assertThat(aliasMetaData.isHidden(), nullValue()); getAliasesResponse = admin().indices().prepareGetAliases("alias3").get(); assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1)); @@ -1143,6 +1148,16 @@ private void checkAliases() { assertThat(aliasMetaData.filter(), nullValue()); assertThat(aliasMetaData.indexRouting(), equalTo("index")); assertThat(aliasMetaData.searchRouting(), equalTo("search")); + assertThat(aliasMetaData.isHidden(), nullValue()); + + getAliasesResponse = admin().indices().prepareGetAliases("alias4").get(); + assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1)); + aliasMetaData = getAliasesResponse.getAliases().get("test").get(0); + assertThat(aliasMetaData.alias(), equalTo("alias4")); + assertThat(aliasMetaData.filter(), nullValue()); + assertThat(aliasMetaData.indexRouting(), nullValue()); + assertThat(aliasMetaData.searchRouting(), nullValue()); + assertThat(aliasMetaData.isHidden(), equalTo(true)); } private void assertHits(SearchHits hits, String... ids) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index ab2177b51d684..009b4df329517 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -49,10 +49,12 @@ import java.util.Set; import java.util.function.Function; +import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_HIDDEN_SETTING; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.endsWith; @@ -879,6 +881,143 @@ public void testConcreteIndicesWildcardAndAliases() { assertTrue(indexNames.contains("bar_bar")); } + public void testHiddenAliasAndHiddenIndexResolution() { + final String visibleIndex = "visible_index"; + final String hiddenIndex = "hidden_index"; + final String visibleAlias = "visible_alias"; + final String hiddenAlias = "hidden_alias"; + final String dottedHiddenAlias = ".hidden_alias"; + final String dottedHiddenIndex = ".hidden_index"; + + IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, false, true, false, false, false); + IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, false, true, false, true, true, false, false, false); + + { + // A visible index with a visible alias and a hidden index with a hidden alias + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder(visibleIndex).state(State.OPEN).putAlias(AliasMetaData.builder(visibleAlias))) + .put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(hiddenAlias).isHidden(true))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + + // A total wildcard should only be resolved to visible indices + String[] indexNames; + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex)); + + // Unless hidden is specified in the options + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // Both hidden indices and hidden aliases should not be included in wildcard resolution + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "hidden*", "visible*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex)); + + // unless it's specified in the options + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "hidden*", "visible*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // Only visible aliases should be included in wildcard resolution + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex)); + + // unless, again, it's specified in the options + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // If we specify a hidden alias by name, the options shouldn't matter. + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, hiddenAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(hiddenIndex)); + + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, hiddenAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(hiddenIndex)); + } + + { + // A visible alias that points to one hidden and one visible index + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder(visibleIndex).state(State.OPEN).putAlias(AliasMetaData.builder(visibleAlias))) + .put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(visibleAlias))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + + // If the alias is resolved to concrete indices, it should resolve to all the indices it points to, hidden or not. + String[] indexNames; + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, visibleAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, visibleAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // A total wildcards does not resolve the hidden index in this case + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex)); + } + + { + // A hidden alias that points to one hidden and one visible index + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder(visibleIndex).state(State.OPEN).putAlias(AliasMetaData.builder(hiddenAlias).isHidden(true))) + .put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(hiddenAlias).isHidden(true))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + + String[] indexNames; + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // A query that only matches the hidden alias should throw + expectThrows(IndexNotFoundException.class, + () -> indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*_alias")); + + // But if we include hidden it should be resolved to both indices + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + + // If we specify the alias by name it should resolve to both indices, regardless of if the options specify hidden + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, hiddenAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, hiddenAlias); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex)); + } + + { + // A hidden alias with a dot-prefixed name that points to one hidden index with a dot prefix, and one hidden index without + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder(dottedHiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(dottedHiddenAlias).isHidden(true))) + .put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(dottedHiddenAlias).isHidden(true))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + + String[] indexNames; + // A dot-prefixed pattern that includes only the hidden alias should resolve to both, regardless of the options + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, ".hidden_a*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(dottedHiddenIndex, hiddenIndex)); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, ".hidden_a*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(dottedHiddenIndex, hiddenIndex)); + + // A query that doesn't include the dot should fail if the options don't include hidden + expectThrows(IndexNotFoundException.class, + () -> indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*_alias")); + + // But should include both indices if the options do include hidden + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "*_alias"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(dottedHiddenIndex, hiddenIndex)); + + } + } + /** * test resolving _all pattern (null, empty array or "_all") for random IndicesOptions */ From 7cceb0c09c24b0494a1a1be8b52e6d9a5947cfcb Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 25 Feb 2020 17:43:21 -0700 Subject: [PATCH 06/14] Improve is_hidden consistency error message --- .../cluster/metadata/AliasOrIndex.java | 6 +++--- .../cluster/metadata/AliasOrIndexTests.java | 20 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java index c81e159e7b27c..9d458663c2f62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java @@ -184,9 +184,9 @@ public void computeAndValidateAliasProperties() { .map(idx -> idx.getIndex().getName()).collect(Collectors.toList()); List nonHiddenOn = groupedByHiddenStatus.get(false).stream() .map(idx -> idx.getIndex().getName()).collect(Collectors.toList()); - throw new IllegalStateException("alias [" + aliasName + "] is hidden set as on [" + - Strings.collectionToCommaDelimitedString(hiddenOn) + "] but as NOT hidden on [" + - Strings.collectionToCommaDelimitedString(nonHiddenOn) + "]; alias must have same hidden setting " + + throw new IllegalStateException("alias [" + aliasName + "] has is_hidden set to true on indices [" + + Strings.collectionToCommaDelimitedString(hiddenOn) + "] but does not have is_hidden set to true on indices [" + + Strings.collectionToCommaDelimitedString(nonHiddenOn) + "]; alias must have the same is_hidden setting " + "on all indices"); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java index 11044476e0173..355c1cce42df7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasOrIndexTests.java @@ -70,12 +70,13 @@ public void testHiddenAliasValidation() { oneNonHidden.addIndex(indexWithNonHiddenAlias); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> oneNonHidden.computeAndValidateAliasProperties()); - assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [")); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + + "] has is_hidden set to true on indices [")); assertThat(exception.getMessage(), allOf(containsString(hidden1.getIndex().getName()), containsString(hidden2.getIndex().getName()), containsString(hidden3.getIndex().getName()))); - assertThat(exception.getMessage(), containsString("but as NOT hidden on [" + - indexWithNonHiddenAlias.getIndex().getName() + "]; alias must have same hidden setting on all indices")); + assertThat(exception.getMessage(), containsString("but does not have is_hidden set to true on indices [" + + indexWithNonHiddenAlias.getIndex().getName() + "]; alias must have the same is_hidden setting on all indices")); } { @@ -85,12 +86,13 @@ public void testHiddenAliasValidation() { oneUnspecified.addIndex(indexWithUnspecifiedAlias); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> oneUnspecified.computeAndValidateAliasProperties()); - assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [")); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + + "] has is_hidden set to true on indices [")); assertThat(exception.getMessage(), allOf(containsString(hidden1.getIndex().getName()), containsString(hidden2.getIndex().getName()), containsString(hidden3.getIndex().getName()))); - assertThat(exception.getMessage(), containsString("but as NOT hidden on [" + - indexWithUnspecifiedAlias.getIndex().getName() + "]; alias must have same hidden setting on all indices")); + assertThat(exception.getMessage(), containsString("but does not have is_hidden set to true on indices [" + + indexWithUnspecifiedAlias.getIndex().getName() + "]; alias must have the same is_hidden setting on all indices")); } { @@ -106,11 +108,11 @@ public void testHiddenAliasValidation() { mostlyVisibleOneHidden.addIndex(hiddenIndex); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> mostlyVisibleOneHidden.computeAndValidateAliasProperties()); - assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] is hidden set as on [" + - hiddenIndex.getIndex().getName() + "] but as NOT hidden on [")); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] has is_hidden set to true on " + + "indices [" + hiddenIndex.getIndex().getName() + "] but does not have is_hidden set to true on indices [")); assertThat(exception.getMessage(), allOf(containsString(indexWithUnspecifiedAlias.getIndex().getName()), containsString(indexWithNonHiddenAlias.getIndex().getName()))); - assertThat(exception.getMessage(), containsString("but as NOT hidden on [")); + assertThat(exception.getMessage(), containsString("but does not have is_hidden set to true on indices [")); } } From c8c7472a3ace39bd87d0c22c412fb08f91df7e60 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 27 Feb 2020 10:37:59 -0700 Subject: [PATCH 07/14] Add isHidden to AliasAction + unit tests --- .../indices/alias/IndicesAliasesRequest.java | 24 +++ .../alias/TransportIndicesAliasesAction.java | 2 +- .../rollover/TransportRolloverAction.java | 6 +- .../cluster/metadata/AliasAction.java | 13 +- .../MetaDataIndexAliasesServiceTests.java | 141 +++++++++++++++--- .../cluster/metadata/MetaDataTests.java | 33 ++++ 6 files changed, 190 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index eede7ecf4a408..08987ce1507ec 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -97,6 +97,7 @@ public static class AliasActions implements AliasesRequest, Writeable, ToXConten private static final ParseField INDEX_ROUTING = new ParseField("index_routing", "indexRouting", "index-routing"); private static final ParseField SEARCH_ROUTING = new ParseField("search_routing", "searchRouting", "search-routing"); private static final ParseField IS_WRITE_INDEX = new ParseField("is_write_index"); + private static final ParseField IS_HIDDEN = new ParseField("is_hidden"); private static final ParseField ADD = new ParseField("add"); private static final ParseField REMOVE = new ParseField("remove"); @@ -193,6 +194,7 @@ private static ObjectParser parser(String name, Supplier REMOVE_PARSER = parser(REMOVE.getPreferredName(), AliasActions::remove); private static final ObjectParser REMOVE_INDEX_PARSER = parser(REMOVE_INDEX.getPreferredName(), @@ -231,6 +233,7 @@ private static ObjectParser parser(String name, Supplier rolloverAliasToNewIndex(String oldIndex, String newInde boolean explicitWriteIndex) { if (explicitWriteIndex) { return List.of( - new AliasAction.Add(newIndex, request.getAlias(), null, null, null, true), - new AliasAction.Add(oldIndex, request.getAlias(), null, null, null, false)); + new AliasAction.Add(newIndex, request.getAlias(), null, null, null, true, null), + new AliasAction.Add(oldIndex, request.getAlias(), null, null, null, false, null)); } else { return List.of( - new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null), + new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null, null), new AliasAction.Remove(oldIndex, request.getAlias())); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index 436ae79c10319..ca3dfb1e53707 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -85,11 +85,13 @@ public static class Add extends AliasAction { @Nullable private final Boolean writeIndex; + @Nullable final Boolean isHidden; + /** * Build the operation. */ - public Add(String index, String alias, @Nullable String filter, @Nullable String indexRouting, - @Nullable String searchRouting, @Nullable Boolean writeIndex) { + public Add(String index, String alias, @Nullable String filter, @Nullable String indexRouting, @Nullable String searchRouting, + @Nullable Boolean writeIndex, Boolean isHidden) { super(index); if (false == Strings.hasText(alias)) { throw new IllegalArgumentException("[alias] is required"); @@ -99,6 +101,7 @@ public Add(String index, String alias, @Nullable String filter, @Nullable String this.indexRouting = indexRouting; this.searchRouting = searchRouting; this.writeIndex = writeIndex; + this.isHidden = isHidden; } /** @@ -112,6 +115,10 @@ public Boolean writeIndex() { return writeIndex; } + public Boolean isHidden() { + return isHidden; + } + @Override boolean removeIndex() { return false; @@ -122,7 +129,7 @@ boolean apply(NewAliasValidator aliasValidator, MetaData.Builder metadata, Index aliasValidator.validate(alias, indexRouting, filter, writeIndex); AliasMetaData newAliasMd = AliasMetaData.newAliasMetaDataBuilder(alias).filter(filter).indexRouting(indexRouting) - .searchRouting(searchRouting).writeIndex(writeIndex).build(); + .searchRouting(searchRouting).writeIndex(writeIndex).isHidden(isHidden).build(); // Check if this alias already exists AliasMetaData currentAliasMd = index.getAliases().get(alias); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java index 9b320ae9f27e7..4221d14a87d64 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java @@ -38,6 +38,7 @@ import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; @@ -72,7 +73,8 @@ public void testAddAndRemove() { ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index); // Add an alias to it - ClusterState after = service.applyAliasActions(before, singletonList(new AliasAction.Add(index, "test", null, null, null, null))); + ClusterState after = service.applyAliasActions(before, singletonList(new AliasAction.Add(index, "test", null, null, null, null, + null))); AliasOrIndex alias = after.metaData().getAliasAndIndexLookup().get("test"); assertNotNull(alias); assertTrue(alias.isAlias()); @@ -83,7 +85,7 @@ public void testAddAndRemove() { before = after; after = service.applyAliasActions(before, Arrays.asList( new AliasAction.Remove(index, "test"), - new AliasAction.Add(index, "test_2", null, null, null, null))); + new AliasAction.Add(index, "test_2", null, null, null, null, null))); assertNull(after.metaData().getAliasAndIndexLookup().get("test")); alias = after.metaData().getAliasAndIndexLookup().get("test_2"); assertNotNull(alias); @@ -107,7 +109,7 @@ public void testMultipleIndices() { for (int i = 0; i < length; i++) { final String index = randomValueOtherThanMany(v -> indices.add(v) == false, () -> randomAlphaOfLength(8)); before = createIndex(before, index); - addActions.add(new AliasAction.Add(index, "alias-" + index, null, null, null, null)); + addActions.add(new AliasAction.Add(index, "alias-" + index, null, null, null, null, null)); } final ClusterState afterAddingAliasesToAll = service.applyAliasActions(before, addActions); assertAliasesVersionIncreased(indices.toArray(new String[0]), before, afterAddingAliasesToAll); @@ -117,7 +119,7 @@ public void testMultipleIndices() { final var randomAddActions = new ArrayList(length); for (var index : indices) { if (randomBoolean()) { - randomAddActions.add(new AliasAction.Add(index, "random-alias-" + index, null, null, null, null)); + randomAddActions.add(new AliasAction.Add(index, "random-alias-" + index, null, null, null, null, null)); randomIndices.add(index); } } @@ -134,17 +136,18 @@ public void testChangingWriteAliasStateIncreasesAliasesVersion() { final ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index); final ClusterState afterAddWriteAlias = - service.applyAliasActions(before, singletonList(new AliasAction.Add(index, "test", null, null, null, true))); + service.applyAliasActions(before, singletonList(new AliasAction.Add(index, "test", null, null, null, true, null))); assertAliasesVersionIncreased(index, before, afterAddWriteAlias); final ClusterState afterChangeWriteAliasToNonWriteAlias = - service.applyAliasActions(afterAddWriteAlias, singletonList(new AliasAction.Add(index, "test", null, null, null, false))); + service.applyAliasActions(afterAddWriteAlias, singletonList(new AliasAction.Add(index, "test", null, null, null, false, + null))); assertAliasesVersionIncreased(index, afterAddWriteAlias, afterChangeWriteAliasToNonWriteAlias); final ClusterState afterChangeNonWriteAliasToWriteAlias = service.applyAliasActions( afterChangeWriteAliasToNonWriteAlias, - singletonList(new AliasAction.Add(index, "test", null, null, null, true))); + singletonList(new AliasAction.Add(index, "test", null, null, null, true, null))); assertAliasesVersionIncreased(index, afterChangeWriteAliasToNonWriteAlias, afterChangeNonWriteAliasToWriteAlias); } @@ -156,7 +159,7 @@ public void testAddingAliasMoreThanOnceShouldOnlyIncreaseAliasesVersionByOne() { final int length = randomIntBetween(2, 8); final var addActions = new ArrayList(length); for (int i = 0; i < length; i++) { - addActions.add(new AliasAction.Add(index, "test", null, null, null, null)); + addActions.add(new AliasAction.Add(index, "test", null, null, null, null, null)); } final ClusterState afterAddingAliases = service.applyAliasActions(before, addActions); @@ -173,7 +176,7 @@ public void testAliasesVersionUnchangedWhenActionsAreIdempotent() { final var addActions = new ArrayList(length); for (int i = 0; i < length; i++) { final String aliasName = randomValueOtherThanMany(v -> aliasNames.add(v) == false, () -> randomAlphaOfLength(8)); - addActions.add(new AliasAction.Add(index, aliasName, null, null, null, null)); + addActions.add(new AliasAction.Add(index, aliasName, null, null, null, null, null)); } final ClusterState afterAddingAlias = service.applyAliasActions(before, addActions); @@ -181,7 +184,7 @@ public void testAliasesVersionUnchangedWhenActionsAreIdempotent() { final var removeAndAddActions = new ArrayList(2 * length); for (final var aliasName : aliasNames) { removeAndAddActions.add(new AliasAction.Remove(index, aliasName)); - removeAndAddActions.add(new AliasAction.Add(index, aliasName, null, null, null, null)); + removeAndAddActions.add(new AliasAction.Add(index, aliasName, null, null, null, null, null)); } final ClusterState afterRemoveAndAddAlias = service.applyAliasActions(afterAddingAlias, removeAndAddActions); assertAliasesVersionUnchanged(index, afterAddingAlias, afterRemoveAndAddAlias); @@ -194,7 +197,7 @@ public void testSwapIndexWithAlias() { // Now remove "test" and add an alias to "test" to "test_2" in one go ClusterState after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test_2", "test", null, null, null, null), + new AliasAction.Add("test_2", "test", null, null, null, null, null), new AliasAction.RemoveIndex("test"))); AliasOrIndex alias = after.metaData().getAliasAndIndexLookup().get("test"); assertNotNull(alias); @@ -209,7 +212,7 @@ public void testAddAliasToRemovedIndex() { // Attempt to add an alias to "test" at the same time as we remove it IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, null), + new AliasAction.Add("test", "alias", null, null, null, null, null), new AliasAction.RemoveIndex("test")))); assertEquals("test", e.getIndex().getName()); } @@ -229,20 +232,20 @@ public void testAddWriteOnlyWithNoExistingAliases() { ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), "test"); ClusterState after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, false))); + new AliasAction.Add("test", "alias", null, null, null, false, null))); assertFalse(after.metaData().index("test").getAliases().get("alias").writeIndex()); assertNull(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex()); assertAliasesVersionIncreased("test", before, after); after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, null))); + new AliasAction.Add("test", "alias", null, null, null, null, null))); assertNull(after.metaData().index("test").getAliases().get("alias").writeIndex()); assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), equalTo(after.metaData().index("test"))); assertAliasesVersionIncreased("test", before, after); after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, true))); + new AliasAction.Add("test", "alias", null, null, null, true, null))); assertTrue(after.metaData().index("test").getAliases().get("alias").writeIndex()); assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), equalTo(after.metaData().index("test"))); @@ -259,7 +262,7 @@ public void testAddWriteOnlyWithExistingWriteIndex() { .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); ClusterState after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, null))); + new AliasAction.Add("test", "alias", null, null, null, null, null))); assertNull(after.metaData().index("test").getAliases().get("alias").writeIndex()); assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), equalTo(after.metaData().index("test2"))); @@ -267,7 +270,7 @@ public void testAddWriteOnlyWithExistingWriteIndex() { assertAliasesVersionUnchanged("test2", before, after); Exception exception = expectThrows(IllegalStateException.class, () -> service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, true)))); + new AliasAction.Add("test", "alias", null, null, null, true, null)))); assertThat(exception.getMessage(), startsWith("alias [alias] has more than one write index [")); } @@ -282,8 +285,8 @@ public void testSwapWriteOnlyIndex() { Boolean unsetValue = randomBoolean() ? null : false; List swapActions = Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, unsetValue), - new AliasAction.Add("test2", "alias", null, null, null, true) + new AliasAction.Add("test", "alias", null, null, null, unsetValue, null), + new AliasAction.Add("test2", "alias", null, null, null, true, null) ); Collections.shuffle(swapActions, random()); ClusterState after = service.applyAliasActions(before, swapActions); @@ -310,7 +313,7 @@ public void testAddWriteOnlyWithExistingNonWriteIndices() { assertNull(((AliasOrIndex.Alias) before.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex()); ClusterState after = service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test3", "alias", null, null, null, true))); + new AliasAction.Add("test3", "alias", null, null, null, true, null))); assertTrue(after.metaData().index("test3").getAliases().get("alias").writeIndex()); assertThat(((AliasOrIndex.Alias) after.metaData().getAliasAndIndexLookup().get("alias")).getWriteIndex(), equalTo(after.metaData().index("test3"))); @@ -349,12 +352,106 @@ public void testAddWriteOnlyValidatesAgainstMetaDataBuilder() { .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); Exception exception = expectThrows(IllegalStateException.class, () -> service.applyAliasActions(before, Arrays.asList( - new AliasAction.Add("test", "alias", null, null, null, true), - new AliasAction.Add("test2", "alias", null, null, null, true) + new AliasAction.Add("test", "alias", null, null, null, true, null), + new AliasAction.Add("test2", "alias", null, null, null, true, null) ))); assertThat(exception.getMessage(), startsWith("alias [alias] has more than one write index [")); } + public void testHiddenPropertyValidation() { + ClusterState originalState = ClusterState.EMPTY_STATE; + originalState = createIndex(originalState, "test1"); + originalState = createIndex(originalState, "test2"); + + { + // Add a non-hidden alias to one index + ClusterState testState = service.applyAliasActions(originalState, Collections.singletonList( + new AliasAction.Add("test1", "alias", null, null, null, null, randomFrom(false, null)) + )); + + // Adding the same alias as hidden to another index should throw + Exception ex = expectThrows(IllegalStateException.class, () -> // Add a non-hidden alias to one index + service.applyAliasActions(testState, Collections.singletonList( + new AliasAction.Add("test2", "alias", null, null, null, null, true) + ))); + assertThat(ex.getMessage(), containsString("alias [alias] has is_hidden set to true on indices")); + } + + { + // Add a hidden alias to one index + ClusterState testState = service.applyAliasActions(originalState, Collections.singletonList( + new AliasAction.Add("test1", "alias", null, null, null, null, true) + )); + + // Adding the same alias as non-hidden to another index should throw + Exception ex = expectThrows(IllegalStateException.class, () -> // Add a non-hidden alias to one index + service.applyAliasActions(testState, Collections.singletonList( + new AliasAction.Add("test2", "alias", null, null, null, null, randomFrom(false, null)) + ))); + assertThat(ex.getMessage(), containsString("alias [alias] has is_hidden set to true on indices")); + } + + { + // Add a non-hidden alias to one index + ClusterState testState = service.applyAliasActions(originalState, Collections.singletonList( + new AliasAction.Add("test1", "alias", null, null, null, null, randomFrom(false, null)) + )); + + // Adding the same alias as non-hidden should be OK + service.applyAliasActions(testState, Collections.singletonList( + new AliasAction.Add("test2", "alias", null, null, null, null, randomFrom(false, null)) + )); + } + + { + // Add a hidden alias to one index + ClusterState testState = service.applyAliasActions(originalState, Collections.singletonList( + new AliasAction.Add("test1", "alias", null, null, null, null, true) + )); + + // Adding the same alias as hidden should be OK + service.applyAliasActions(testState, Collections.singletonList( + new AliasAction.Add("test2", "alias", null, null, null, null, true) + )); + } + } + + public void testSimultaneousHiddenPropertyValidation() { + IndexMetaData.Builder indexMetaData = IndexMetaData.builder("test") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + IndexMetaData.Builder indexMetaData2 = IndexMetaData.builder("test2") + .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1); + ClusterState before = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder().put(indexMetaData).put(indexMetaData2)).build(); + + { + // These should all be fine + applyHiddenAliasMix(before, null, null); + applyHiddenAliasMix(before, false, false); + applyHiddenAliasMix(before, false, null); + applyHiddenAliasMix(before, null, false); + + applyHiddenAliasMix(before, true, true); + } + + { + Exception exception = expectThrows(IllegalStateException.class, () -> applyHiddenAliasMix(before, true, randomFrom(false, null))); + assertThat(exception.getMessage(), startsWith("alias [alias] has is_hidden set to true on indices [")); + } + + { + Exception exception = expectThrows(IllegalStateException.class, () -> applyHiddenAliasMix(before, randomFrom(false, null), true)); + assertThat(exception.getMessage(), startsWith("alias [alias] has is_hidden set to true on indices [")); + } + } + + private ClusterState applyHiddenAliasMix(ClusterState before, Boolean isHidden1, Boolean isHidden2) { + return service.applyAliasActions(before, Arrays.asList( + new AliasAction.Add("test", "alias", null, null, null, null, isHidden1), + new AliasAction.Add("test2", "alias", null, null, null, null, isHidden2) + )); + } + private ClusterState createIndex(ClusterState state, String index) { IndexMetaData indexMetaData = IndexMetaData.builder(index) .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 7d2b10beb3279..5bd981f246a56 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -226,6 +226,39 @@ public void testValidateAliasWriteOnly() { assertThat(exception.getMessage(), startsWith("alias [" + alias + "] has more than one write index [")); } + public void testValidateHiddenAliasConsistency() { + String alias = randomAlphaOfLength(5); + String indexA = randomAlphaOfLength(6); + String indexB = randomAlphaOfLength(7); + + { + Exception ex = expectThrows(IllegalStateException.class, + () -> buildMetadataWithHiddenIndexMix(alias, indexA, true, indexB, randomFrom(false, null)).build()); + assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); + } + + { + Exception ex = expectThrows(IllegalStateException.class, + () -> buildMetadataWithHiddenIndexMix(alias, indexA, randomFrom(false, null), indexB, true).build()); + assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); + } + } + + private MetaData.Builder buildMetadataWithHiddenIndexMix(String aliasName, String indexAName, Boolean indexAHidden, + String indexBName, Boolean indexBHidden) { + IndexMetaData.Builder indexAMeta = IndexMetaData.builder(indexAName) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder(aliasName).isHidden(indexAHidden).build()); + IndexMetaData.Builder indexBMeta = IndexMetaData.builder(indexBName) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder(aliasName).isHidden(indexBHidden).build()); + return MetaData.builder().put(indexAMeta).put(indexBMeta); + } + public void testResolveIndexRouting() { IndexMetaData.Builder builder = IndexMetaData.builder("index") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) From e7c1af77ca2f6d5954b2b6bcc2b866c7fd1b5670 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 27 Feb 2020 11:55:05 -0700 Subject: [PATCH 08/14] Yet more unit tests --- .../action/admin/indices/alias/AliasActionsTests.java | 3 +++ .../elasticsearch/index/alias/RandomAliasActionsGenerator.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java index f2ae67e1fc1ab..73029b5e0ecbc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java @@ -115,6 +115,7 @@ public void testParseAdd() throws IOException { Object searchRouting = randomBoolean() ? randomRouting() : null; Object indexRouting = randomBoolean() ? randomBoolean() ? searchRouting : randomRouting() : null; boolean writeIndex = randomBoolean(); + boolean isHidden = randomBoolean(); XContentBuilder b = XContentBuilder.builder(randomFrom(XContentType.values()).xContent()); b.startObject(); { @@ -144,6 +145,7 @@ public void testParseAdd() throws IOException { b.field("index_routing", indexRouting); } b.field("is_write_index", writeIndex); + b.field("is_hidden", isHidden); } b.endObject(); } @@ -162,6 +164,7 @@ public void testParseAdd() throws IOException { assertEquals(Objects.toString(searchRouting, null), action.searchRouting()); assertEquals(Objects.toString(indexRouting, null), action.indexRouting()); assertEquals(writeIndex, action.writeIndex()); + assertEquals(isHidden, action.isHidden()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java index e8a554ca4aaae..31c93cbc1ddd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java @@ -85,6 +85,9 @@ public static AliasActions randomAliasAction(boolean useStringAsFilter) { if (randomBoolean()) { action.writeIndex(randomBoolean()); } + if (randomBoolean()) { + action.isHidden(randomBoolean()); + } } return action; } From 1e77c224a4814c0f378d84beeb4938667474d77e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 27 Feb 2020 15:31:56 -0700 Subject: [PATCH 09/14] Integration tests for hidden aliases --- .../elasticsearch/aliases/IndexAliasesIT.java | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 557e7e09f7db2..6803a463651fc 100644 --- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -27,7 +27,9 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; @@ -52,6 +54,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -76,6 +79,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; @@ -1122,6 +1126,103 @@ public void testRemoveIndexAndReplaceWithAlias() throws InterruptedException, Ex assertHitCount(client().prepareSearch("test").get(), 1); } + public void testHiddenAliasesMustBeConsistent() { + final String index1 = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + final String index2 = randomAlphaOfLength(6).toLowerCase(Locale.ROOT); + final String alias = randomAlphaOfLength(7).toLowerCase(Locale.ROOT); + createIndex(index1, index2); + + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias))); + + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { + AcknowledgedResponse res = admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true)).get(); + }); + logger.error("exception: {}", ex.getMessage()); + assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); + + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.remove().index(index1).alias(alias))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias).isHidden(false))); + expectThrows(IllegalStateException.class, + () -> admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true)).get()); + + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.remove().index(index1).alias(alias))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias).isHidden(true))); + expectThrows(IllegalStateException.class, + () -> admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(false)).get()); + expectThrows(IllegalStateException.class, + () -> admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias)).get()); + + // Both visible + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.remove().index(index1).alias(alias))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias).isHidden(false))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(false))); + + // Both hidden + assertAcked(admin().indices().prepareAliases() + .addAliasAction(AliasActions.remove().index(index1).alias(alias)) + .addAliasAction(AliasActions.remove().index(index2).alias(alias))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias).isHidden(true))); + assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true))); + + // Visible on one, then update it to hidden & add to a second as hidden simultaneously + assertAcked(admin().indices().prepareAliases() + .addAliasAction(AliasActions.remove().index(index1).alias(alias)) + .addAliasAction(AliasActions.remove().index(index2).alias(alias))); + assertAcked(admin().indices().prepareAliases() + .addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(false))); + assertAcked(admin().indices().prepareAliases() + .addAliasAction(AliasActions.add().index(index1).alias(alias).isHidden(true)) + .addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true))); + } + + public void testIndexingAndQueryingHiddenAliases() throws Exception { + final String writeIndex = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + final String nonWriteIndex = randomAlphaOfLength(6).toLowerCase(Locale.ROOT); + final String alias = "alias-" + randomAlphaOfLength(7).toLowerCase(Locale.ROOT); + createIndex(writeIndex, nonWriteIndex); + + assertAcked(admin().indices().prepareAliases() + .addAliasAction(AliasActions.add().index(writeIndex).alias(alias).isHidden(true).writeIndex(true)) + .addAliasAction(AliasActions.add().index(nonWriteIndex).alias(alias).isHidden(true))); + + ensureGreen(); + + // Put a couple docs in each index directly + IndexResponse res = client().index(indexRequest(nonWriteIndex).id("1").source(source("1", "nonwrite"), XContentType.JSON)).get(); + assertThat(res.status().getStatus(), equalTo(201)); + res = client().index(indexRequest(writeIndex).id("2").source(source("2", "writeindex"), XContentType.JSON)).get(); + assertThat(res.status().getStatus(), equalTo(201)); + // And through the alias + res = client().index(indexRequest(alias).id("3").source(source("3", "through alias"), XContentType.JSON)).get(); + assertThat(res.status().getStatus(), equalTo(201)); + + refresh(writeIndex, nonWriteIndex); + + // Make sure that the doc written to the alias made it + SearchResponse searchResponse = client().prepareSearch(writeIndex).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHits(searchResponse.getHits(), "2", "3"); + + // Ensure that all docs can be gotten through the alias + searchResponse = client().prepareSearch(alias).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHits(searchResponse.getHits(), "1", "2", "3"); + + // And querying using a wildcard with indices options set to expand hidden + searchResponse = client().prepareSearch("alias*") + .setQuery(QueryBuilders.matchAllQuery()) + .setIndicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, true, false, false)).get(); + assertHits(searchResponse.getHits(), "1", "2", "3"); + + // And that querying the alias with a wildcard and no expand options fails + searchResponse = client().prepareSearch("alias*").setQuery(QueryBuilders.matchAllQuery()).get(); + assertThat(searchResponse.getHits().getHits(), emptyArray()); + } + + public void testGetAliasAndAliasExistsForHiddenAliases() { + final String writeIndex = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + final String nonWriteIndex = randomAlphaOfLength(6).toLowerCase(Locale.ROOT); + final String alias = "alias-" + randomAlphaOfLength(7).toLowerCase(Locale.ROOT); + } + private void checkAliases() { GetAliasesResponse getAliasesResponse = admin().indices().prepareGetAliases("alias1").get(); assertThat(getAliasesResponse.getAliases().get("test").size(), equalTo(1)); From 29a6a5d9e2adb03282d43b7d5c1ce6c7d0379d1f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 27 Feb 2020 15:41:13 -0700 Subject: [PATCH 10/14] More unit tests --- .../IndexNameExpressionResolverTests.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 009b4df329517..fc78813fb83b3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -56,6 +56,7 @@ import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -1018,6 +1019,33 @@ public void testHiddenAliasAndHiddenIndexResolution() { } } + public void testHiddenIndexWithVisibleAliasOverlappingNameResolution() { + final String hiddenIndex = "my-hidden-index"; + final String hiddenAlias = "my-hidden-alias"; + final String visibleAlias = "my-visible-alias"; + + IndicesOptions excludeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, false, true, false, false, false); + IndicesOptions includeHiddenOptions = IndicesOptions.fromOptions(false, true, true, false, true, true, false, false, false); + + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder(hiddenIndex, Settings.builder().put(INDEX_HIDDEN_SETTING.getKey(), true).build()) + .state(State.OPEN) + .putAlias(AliasMetaData.builder(hiddenAlias).isHidden(true)) + .putAlias(AliasMetaData.builder(visibleAlias).build())); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + + String[] indexNames; + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "my-*"); + assertThat(Arrays.asList(indexNames), containsInAnyOrder(hiddenIndex)); + + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "my-hidden*"); + assertThat(Arrays.asList(indexNames), empty()); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "my-*", "-my-visible*"); + assertThat(Arrays.asList(indexNames), empty()); + indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, "my-hidden*", "-my-hidden-a*"); + assertThat(Arrays.asList(indexNames), empty()); + } + /** * test resolving _all pattern (null, empty array or "_all") for random IndicesOptions */ From 0dad3570319ded9a4dc39ca7402ad7accca7922a Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 27 Feb 2020 15:57:45 -0700 Subject: [PATCH 11/14] Line lengths --- .../test/java/org/elasticsearch/aliases/IndexAliasesIT.java | 3 ++- .../cluster/metadata/MetaDataIndexAliasesServiceTests.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 6803a463651fc..fa8b4d0a45140 100644 --- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -1135,7 +1135,8 @@ public void testHiddenAliasesMustBeConsistent() { assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index1).alias(alias))); IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { - AcknowledgedResponse res = admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true)).get(); + AcknowledgedResponse res = admin().indices().prepareAliases() + .addAliasAction(AliasActions.add().index(index2).alias(alias).isHidden(true)).get(); }); logger.error("exception: {}", ex.getMessage()); assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java index 4221d14a87d64..c010927bf5c83 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java @@ -435,12 +435,14 @@ public void testSimultaneousHiddenPropertyValidation() { } { - Exception exception = expectThrows(IllegalStateException.class, () -> applyHiddenAliasMix(before, true, randomFrom(false, null))); + Exception exception = expectThrows(IllegalStateException.class, + () -> applyHiddenAliasMix(before, true, randomFrom(false, null))); assertThat(exception.getMessage(), startsWith("alias [alias] has is_hidden set to true on indices [")); } { - Exception exception = expectThrows(IllegalStateException.class, () -> applyHiddenAliasMix(before, randomFrom(false, null), true)); + Exception exception = expectThrows(IllegalStateException.class, + () -> applyHiddenAliasMix(before, randomFrom(false, null), true)); assertThat(exception.getMessage(), startsWith("alias [alias] has is_hidden set to true on indices [")); } } From 2faec6314d8c74ecf513549dc2ac65cc8f8a7e4b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 28 Feb 2020 17:35:59 -0700 Subject: [PATCH 12/14] Plumb is_hidden into Security's resolver --- .../authz/IndicesAndAliasesResolver.java | 10 ++- .../authz/IndicesAndAliasesResolverTests.java | 65 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 28b97359a8091..2a974e6469758 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -417,15 +417,21 @@ private static boolean isIndexVisible(String expression, String index, IndicesOp private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, MetaData metaData, boolean dateMathExpression) { AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(index); + final boolean isHidden = aliasOrIndex.isHidden(); if (aliasOrIndex.isAlias()) { //it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed. //complicated to support those options with aliases pointing to multiple indices... //TODO investigate supporting expandWildcards option for aliases too, like es core does. - return indicesOptions.ignoreAliases() == false; + if (indicesOptions.ignoreAliases()) { + return false; + } else if (isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index)) { + return true; + } else { + return false; + } } assert aliasOrIndex.getIndices().size() == 1 : "concrete index must point to a single index"; IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0); - final boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(indexMetaData.getSettings()); if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) { return false; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 95ceda9264ecf..3ca8f9e1ae35b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -146,6 +146,15 @@ public void setup() { .settings(Settings.builder().put(settings).put("index.hidden", true).build())) .put(indexBuilder("hidden-closed").state(State.CLOSE) .settings(Settings.builder().put(settings).put("index.hidden", true).build())) + .put(indexBuilder("hidden-w-aliases").settings(Settings.builder().put(settings).put("index.hidden", true).build()) + .putAlias(AliasMetaData.builder("alias-hidden").isHidden(true).build()) + .putAlias(AliasMetaData.builder(".alias-hidden").isHidden(true).build()) + .putAlias(AliasMetaData.builder("alias-visible-mixed").isHidden(false).build())) + .put(indexBuilder("hidden-w-visible-alias").settings(Settings.builder().put(settings).put("index.hidden", true).build()) + .putAlias(AliasMetaData.builder("alias-visible").build())) + .put(indexBuilder("visible-w-aliases").settings(Settings.builder().put(settings).build()) + .putAlias(AliasMetaData.builder("alias-visible").build()) + .putAlias(AliasMetaData.builder("alias-visible-mixed").isHidden(false).build())) .put(indexBuilder(securityIndexName).settings(settings)).build(); if (withAlias) { @@ -169,6 +178,13 @@ public void setup() { roleMap.put("alias_read_write", new RoleDescriptor("alias_read_write", null, new IndicesPrivileges[] { IndicesPrivileges.builder().indices("barbaz", "foofoobar").privileges("read", "write").build() }, null)); + roleMap.put("hidden_alias_test", new RoleDescriptor("hidden_alias_test", null, + new IndicesPrivileges[] { + IndicesPrivileges.builder() + .indices("alias-visible", "alias-visible-mixed", "alias-hidden", ".alias-hidden", "hidden-open") + .privileges("all") + .build() + }, null)); roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR); final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); doAnswer((i) -> { @@ -1388,7 +1404,6 @@ public void testHiddenIndicesResolution() { // open + hidden searchRequest = new SearchRequest(); searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true)); - authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME); resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open")); @@ -1427,6 +1442,54 @@ public void testHiddenIndicesResolution() { assertThat(resolvedIndices.getRemote(), emptyIterable()); } + public void testHiddenAliasesResolution() { + final User user = new User("hidden-alias-tester", "hidden_alias_test"); + final List authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME); + + // Visible only + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false)); + ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), containsInAnyOrder("alias-visible", "alias-visible-mixed")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + + // Include hidden explicitly + searchRequest = new SearchRequest(); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true)); + resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), + containsInAnyOrder("alias-visible", "alias-visible-mixed", "alias-hidden", ".alias-hidden", "hidden-open")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + + // Include hidden with a wildcard + searchRequest = new SearchRequest("alias-h*"); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true)); + resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), containsInAnyOrder("alias-hidden")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + + // Dot prefix, implicitly including hidden + searchRequest = new SearchRequest(".a*"); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false)); + resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), containsInAnyOrder(".alias-hidden")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + + // Make sure ignoring aliases works (visible only) + searchRequest = new SearchRequest(); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, true, false, false, true, false, true, false)); + resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), contains("-*")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + + // Make sure ignoring aliases works (including hidden) + searchRequest = new SearchRequest(); + searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, false, true, false)); + resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices); + assertThat(resolvedIndices.getLocal(), containsInAnyOrder("hidden-open")); + assertThat(resolvedIndices.getRemote(), emptyIterable()); + } + private List buildAuthorizedIndices(User user, String action) { PlainActionFuture rolesListener = new PlainActionFuture<>(); final Authentication authentication = From 88a3d75ecab45f7c93b87b7ef65d979e16189a07 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 28 Feb 2020 18:10:50 -0700 Subject: [PATCH 13/14] Update streaming version TODOS --- .../org/elasticsearch/action/admin/indices/alias/Alias.java | 4 ++-- .../action/admin/indices/alias/IndicesAliasesRequest.java | 4 ++-- .../org/elasticsearch/cluster/metadata/AliasMetaData.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java index 62ac8ea4dc9fe..a1fa218713aa5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java @@ -75,7 +75,7 @@ public Alias(StreamInput in) throws IOException { indexRouting = in.readOptionalString(); searchRouting = in.readOptionalString(); writeIndex = in.readOptionalBoolean(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO fix for backport of https://github.com/elastic/elasticsearch/pull/52547 isHidden = in.readOptionalBoolean(); } } @@ -219,7 +219,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(indexRouting); out.writeOptionalString(searchRouting); out.writeOptionalBoolean(writeIndex); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO fix for backport of https://github.com/elastic/elasticsearch/pull/52547 out.writeOptionalBoolean(isHidden); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index 08987ce1507ec..50a63a1e5227e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -251,7 +251,7 @@ public AliasActions(StreamInput in) throws IOException { searchRouting = in.readOptionalString(); indexRouting = in.readOptionalString(); writeIndex = in.readOptionalBoolean(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: Update on backport + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { //TODO fix for backport of https://github.com/elastic/elasticsearch/pull/52547 isHidden = in.readOptionalBoolean(); } originalAliases = in.readStringArray(); @@ -267,7 +267,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(searchRouting); out.writeOptionalString(indexRouting); out.writeOptionalBoolean(writeIndex); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: Update on backport + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { //TODO fix for backport https://github.com/elastic/elasticsearch/pull/52547 out.writeOptionalBoolean(isHidden); } out.writeStringArray(originalAliases); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index f533e67508350..93b3e95507c40 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -197,7 +197,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalBoolean(writeIndex()); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { //TODO fix for backport of https://github.com/elastic/elasticsearch/pull/52547 out.writeOptionalBoolean(isHidden); } } @@ -223,7 +223,7 @@ public AliasMetaData(StreamInput in) throws IOException { } writeIndex = in.readOptionalBoolean(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { //TODO fix for backport of https://github.com/elastic/elasticsearch/pull/52547 isHidden = in.readOptionalBoolean(); } else { isHidden = null; From 9285a57e55b60975da635b7f4e9089bf1197748f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 3 Mar 2020 15:18:10 -0700 Subject: [PATCH 14/14] @Nullable consistency --- .../java/org/elasticsearch/cluster/metadata/AliasAction.java | 3 ++- .../java/org/elasticsearch/cluster/metadata/AliasMetaData.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index ca3dfb1e53707..2067589f74ded 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -91,7 +91,7 @@ public static class Add extends AliasAction { * Build the operation. */ public Add(String index, String alias, @Nullable String filter, @Nullable String indexRouting, @Nullable String searchRouting, - @Nullable Boolean writeIndex, Boolean isHidden) { + @Nullable Boolean writeIndex, @Nullable Boolean isHidden) { super(index); if (false == Strings.hasText(alias)) { throw new IllegalArgumentException("[alias] is required"); @@ -115,6 +115,7 @@ public Boolean writeIndex() { return writeIndex; } + @Nullable public Boolean isHidden() { return isHidden; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 93b3e95507c40..f5cdbc479e66e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -65,7 +65,7 @@ public class AliasMetaData extends AbstractDiffable implements To private final Boolean isHidden; private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex, - Boolean isHidden) { + @Nullable Boolean isHidden) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -128,6 +128,7 @@ public Boolean writeIndex() { return writeIndex; } + @Nullable public Boolean isHidden() { return isHidden; }