diff --git a/build.gradle b/build.gradle index 44f3d4559ec0a..b22c447df5d23 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/60819" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc index 86f8d579a1789..98c43d7ab6762 100644 --- a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc +++ b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc @@ -12,6 +12,10 @@ This action makes the index <>. To use the `forcemerge` action in the `hot` phase, the `rollover` action *must* be present. If no rollover action is configured, {ilm-init} will reject the policy. +[NOTE] +The `forcemerge` action is best effort. It might happen that some of the +shards are relocating, in which case they will not be merged. + [[ilm-forcemerge-options]] ==== Options diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 20aa7ae2f1d6f..ea5869696d6c7 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -30,6 +30,20 @@ To keep the snapshot, set `delete_searchable_snapshot` to `false` in the delete Specifies where to store the snapshot. See <> for more information. +`force_merge_index`:: +(Optional, boolean) +Force merges the managed index to one segment. +Defaults to `true`. +If the managed index was already force merged using the +<> in a previous action +the `searchable snapshot` action force merge step will be a no-op. + +[NOTE] +The `forcemerge` action is best effort. It might happen that some of +the shards are relocating, in which case they will not be merged. +The `searchable-snapshot` action will continue executing even if not all shards +are force merged. + [[ilm-searchable-snapshot-ex]] ==== Examples [source,console] diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 80ea7f439e66c..a772d33d685dd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1027,7 +1027,7 @@ protected static void assertOK(Response response) { * in an non green state * @param index index to test for **/ - protected static void ensureGreen(String index) throws IOException { + public static void ensureGreen(String index) throws IOException { ensureHealth(index, (request) -> { request.addParameter("wait_for_status", "green"); request.addParameter("wait_for_no_relocating_shards", "true"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 36716fc6941ed..43093eb269684 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; @@ -18,7 +19,7 @@ import org.elasticsearch.xpack.core.ilm.Step.StepKey; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -31,15 +32,17 @@ public class SearchableSnapshotAction implements LifecycleAction { public static final String NAME = "searchable_snapshot"; public static final ParseField SNAPSHOT_REPOSITORY = new ParseField("snapshot_repository"); + public static final ParseField FORCE_MERGE_INDEX = new ParseField("force_merge_index"); public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; public static final String RESTORED_INDEX_PREFIX = "restored-"; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, - a -> new SearchableSnapshotAction((String) a[0])); + a -> new SearchableSnapshotAction((String) a[0], a[1] == null || (boolean) a[1])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), SNAPSHOT_REPOSITORY); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), FORCE_MERGE_INDEX); } public static SearchableSnapshotAction parse(XContentParser parser) { @@ -47,22 +50,34 @@ public static SearchableSnapshotAction parse(XContentParser parser) { } private final String snapshotRepository; + private final boolean forceMergeIndex; - public SearchableSnapshotAction(String snapshotRepository) { + public SearchableSnapshotAction(String snapshotRepository, boolean forceMergeIndex) { if (Strings.hasText(snapshotRepository) == false) { throw new IllegalArgumentException("the snapshot repository must be specified"); } this.snapshotRepository = snapshotRepository; + this.forceMergeIndex = forceMergeIndex; + } + + public SearchableSnapshotAction(String snapshotRepository) { + this(snapshotRepository, true); } public SearchableSnapshotAction(StreamInput in) throws IOException { - this(in.readString()); + this(in.readString(), in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : true); + } + + boolean isForceMergeIndex() { + return forceMergeIndex; } @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey checkNoWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); + StepKey forceMergeStepKey = new StepKey(phase, NAME, ForceMergeStep.NAME); + StepKey waitForSegmentCountKey = new StepKey(phase, NAME, SegmentCountStep.NAME); StepKey generateSnapshotNameKey = new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME); StepKey cleanSnapshotKey = new StepKey(phase, NAME, CleanupSnapshotStep.NAME); StepKey createSnapshotKey = new StepKey(phase, NAME, CreateSnapshotStep.NAME); @@ -77,8 +92,14 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex, waitForNoFollowerStepKey); - WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, generateSnapshotNameKey, - client); + final WaitForNoFollowersStep waitForNoFollowersStep; + if (forceMergeIndex) { + waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, forceMergeStepKey, client); + } else { + waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, generateSnapshotNameKey, client); + } + ForceMergeStep forceMergeStep = new ForceMergeStep(forceMergeStepKey, waitForSegmentCountKey, client, 1); + SegmentCountStep segmentCountStep = new SegmentCountStep(waitForSegmentCountKey, generateSnapshotNameKey, client, 1); GenerateSnapshotNameStep generateSnapshotNameStep = new GenerateSnapshotNameStep(generateSnapshotNameKey, cleanSnapshotKey, snapshotRepository); CleanupSnapshotStep cleanupSnapshotStep = new CleanupSnapshotStep(cleanSnapshotKey, createSnapshotKey, client); @@ -108,9 +129,25 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { SwapAliasesAndDeleteSourceIndexStep swapAliasesAndDeleteSourceIndexStep = new SwapAliasesAndDeleteSourceIndexStep(swapAliasesKey, null, client, RESTORED_INDEX_PREFIX); - return Arrays.asList(checkNoWriteIndexStep, waitForNoFollowersStep, generateSnapshotNameStep, cleanupSnapshotStep, - createSnapshotBranchingStep, mountSnapshotStep, waitForGreenIndexHealthStep, copyMetadataStep, copySettingsStep, - isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep, swapAliasesAndDeleteSourceIndexStep); + List steps = new ArrayList<>(); + steps.add(checkNoWriteIndexStep); + steps.add(waitForNoFollowersStep); + if (forceMergeIndex) { + steps.add(forceMergeStep); + steps.add(segmentCountStep); + } + steps.add(generateSnapshotNameStep); + steps.add(cleanupSnapshotStep); + steps.add(createSnapshotBranchingStep); + steps.add(mountSnapshotStep); + steps.add(waitForGreenIndexHealthStep); + steps.add(copyMetadataStep); + steps.add(copySettingsStep); + steps.add(isDataStreamBranchingStep); + steps.add(replaceDataStreamBackingIndex); + steps.add(deleteSourceIndexStep); + steps.add(swapAliasesAndDeleteSourceIndexStep); + return steps; } @Override @@ -126,12 +163,16 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(snapshotRepository); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(forceMergeIndex); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(SNAPSHOT_REPOSITORY.getPreferredName(), snapshotRepository); + builder.field(FORCE_MERGE_INDEX.getPreferredName(), forceMergeIndex); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java index 9978d4cec21ba..81c66238c2e7f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java @@ -57,8 +57,7 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener, if (idxSegments == null || (response.getShardFailures() != null && response.getShardFailures().length > 0)) { final DefaultShardOperationFailedException[] failures = response.getShardFailures(); logger.info("[{}] retrieval of segment counts after force merge did not succeed, " + - "there were {} shard failures. " + - "failures: {}", + "there were {} shard failures. failures: {}", index.getName(), response.getFailedShards(), failures == null ? "n/a" : Strings.collectionToDelimitedString(Arrays.stream(failures) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index 375f5d07ae297..48757fc483064 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -20,42 +20,73 @@ public class SearchableSnapshotActionTests extends AbstractActionTestCase steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), is(13)); - - assertThat(steps.get(0).getKey(), is(expectedFirstStep)); - assertThat(steps.get(1).getKey(), is(expectedSecondStep)); - assertThat(steps.get(2).getKey(), is(expectedThirdStep)); - assertThat(steps.get(3).getKey(), is(expectedFourthStep)); - assertThat(steps.get(4).getKey(), is(expectedFifthStep)); - assertThat(steps.get(5).getKey(), is(expectedSixthStep)); - assertThat(steps.get(6).getKey(), is(expectedSeventhStep)); - assertThat(steps.get(7).getKey(), is(expectedEighthStep)); - assertThat(steps.get(8).getKey(), is(expectedNinthStep)); - assertThat(steps.get(9).getKey(), is(expectedTenthStep)); - assertThat(steps.get(10).getKey(), is(expectedElevenStep)); - assertThat(steps.get(11).getKey(), is(expectedTwelveStep)); - assertThat(steps.get(12).getKey(), is(expectedThirteenStep)); - - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedFourthStep)); + assertThat(steps.size(), is(action.isForceMergeIndex() ? 15 : 13)); + + List expectedSteps = action.isForceMergeIndex() ? expectedStepKeysWithForceMerge(phase) : + expectedStepKeysNoForceMerge(phase); + + assertThat(steps.get(0).getKey(), is(expectedSteps.get(0))); + assertThat(steps.get(1).getKey(), is(expectedSteps.get(1))); + assertThat(steps.get(2).getKey(), is(expectedSteps.get(2))); + assertThat(steps.get(3).getKey(), is(expectedSteps.get(3))); + assertThat(steps.get(4).getKey(), is(expectedSteps.get(4))); + assertThat(steps.get(5).getKey(), is(expectedSteps.get(5))); + assertThat(steps.get(6).getKey(), is(expectedSteps.get(6))); + assertThat(steps.get(7).getKey(), is(expectedSteps.get(7))); + assertThat(steps.get(8).getKey(), is(expectedSteps.get(8))); + assertThat(steps.get(9).getKey(), is(expectedSteps.get(9))); + assertThat(steps.get(10).getKey(), is(expectedSteps.get(10))); + assertThat(steps.get(11).getKey(), is(expectedSteps.get(11))); + assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); + + if (action.isForceMergeIndex()) { + assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(6); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(5))); + } else { + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(3))); + } + } + + private List expectedStepKeysWithForceMerge(String phase) { + return List.of( + new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), + new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), + new StepKey(phase, NAME, ForceMergeStep.NAME), + new StepKey(phase, NAME, SegmentCountStep.NAME), + new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), + new StepKey(phase, NAME, CleanupSnapshotStep.NAME), + new StepKey(phase, NAME, CreateSnapshotStep.NAME), + new StepKey(phase, NAME, MountSnapshotStep.NAME), + new StepKey(phase, NAME, WaitForIndexColorStep.NAME), + new StepKey(phase, NAME, CopyExecutionStateStep.NAME), + new StepKey(phase, NAME, CopySettingsStep.NAME), + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), + new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), + new StepKey(phase, NAME, DeleteStep.NAME), + new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME)); + } + + private List expectedStepKeysNoForceMerge(String phase) { + return List.of( + new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), + new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), + new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), + new StepKey(phase, NAME, CleanupSnapshotStep.NAME), + new StepKey(phase, NAME, CreateSnapshotStep.NAME), + new StepKey(phase, NAME, MountSnapshotStep.NAME), + new StepKey(phase, NAME, WaitForIndexColorStep.NAME), + new StepKey(phase, NAME, CopyExecutionStateStep.NAME), + new StepKey(phase, NAME, CopySettingsStep.NAME), + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), + new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), + new StepKey(phase, NAME, DeleteStep.NAME), + new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME)); } @Override @@ -79,6 +110,6 @@ protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction insta } static SearchableSnapshotAction randomInstance() { - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10)); + return new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), randomBoolean()); } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index 617524fa48368..4a57b9943b61d 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,12 +37,15 @@ import java.io.InputStream; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.rest.ESRestTestCase.ensureGreen; /** * This class provides the operational REST functions needed to control an ILM time series lifecycle. @@ -204,4 +208,36 @@ public static Map getOnlyIndexSettings(RestClient client, String } } + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings) + throws IOException { + createIndexWithSettings(client, index, alias, settings, randomBoolean()); + } + + public static void createIndexWithSettings(RestClient client, String index, String alias, Settings.Builder settings, + boolean useWriteIndex) throws IOException { + Request request = new Request("PUT", "/" + index); + + String writeIndexSnippet = ""; + if (useWriteIndex) { + writeIndexSnippet = "\"is_write_index\": true"; + } + request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build()) + + ", \"aliases\" : { \"" + alias + "\": { " + writeIndexSnippet + " } } }"); + client.performRequest(request); + // wait for the shards to initialize + ensureGreen(index); + } + + @SuppressWarnings("unchecked") + public static Integer getNumberOfSegments(RestClient client, String index) throws IOException { + Response response = client.performRequest(new Request("GET", index + "/_segments")); + XContentType entityContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + Map responseEntity = XContentHelper.convertToMap(entityContentType.xContent(), + response.getEntity().getContent(), false); + responseEntity = (Map) responseEntity.get("indices"); + responseEntity = (Map) responseEntity.get(index); + responseEntity = (Map) responseEntity.get("shards"); + List> shards = (List>) responseEntity.get("0"); + return (Integer) shards.get(0).get("num_search_segments"); + } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index e5cd67ae528b8..b783dfd1f4598 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -223,10 +223,10 @@ public void testGetDataStreamReturnsILMPolicy() throws Exception { } private static Template getTemplate(String policyName) throws IOException { - return new Template(getLifcycleSettings(policyName), null, null); + return new Template(getLifecycleSettings(policyName), null, null); } - private static Settings getLifcycleSettings(String policyName) { + private static Settings getLifecycleSettings(String policyName) { return Settings.builder() .put(LifecycleSettings.LIFECYCLE_NAME, policyName) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 0adf7f87385e9..69afbc53e2e82 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -8,7 +8,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Request; @@ -62,15 +61,16 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createFullPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explain; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; @@ -113,7 +113,7 @@ public void testFullPolicy() throws Exception { String originalIndex = index + "-000001"; String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -141,7 +141,7 @@ public void testFullPolicy() throws Exception { public void testMoveToAllocateStep() throws Exception { String originalIndex = index + "-000001"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias")); @@ -175,7 +175,7 @@ public void testMoveToRolloverStep() throws Exception { String originalIndex = index + "-000001"; String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.include._name", "integTest-0") .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -219,7 +219,7 @@ public void testMoveToRolloverStep() throws Exception { public void testRetryFailedDeleteAction() throws Exception { createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) @@ -239,7 +239,7 @@ public void testRetryFailedDeleteAction() throws Exception { public void testRetryFreezeDeleteAction() throws Exception { createNewSingletonPolicy(client(), policy, "cold", new FreezeAction()); - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_READ_ONLY, true) @@ -261,7 +261,7 @@ public void testRetryFailedShrinkAction() throws Exception { int divisor = randomFrom(2, 4); int expectedFinalShards = numShards / divisor; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numShards + randomIntBetween(1, numShards))); updatePolicy(index, policy); @@ -294,7 +294,7 @@ public void testRetryFailedShrinkAction() throws Exception { public void testRolloverAction() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -313,7 +313,7 @@ public void testRolloverAction() throws Exception { public void testRolloverActionWithIndexingComplete() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -352,7 +352,7 @@ public void testRolloverActionWithIndexingComplete() throws Exception { } public void testAllocateOnlyAllocation() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String allocateNodeName = "integTest-" + randomFrom(0, 1); AllocateAction allocateAction = new AllocateAction(null, null, null, singletonMap("_name", allocateNodeName)); @@ -369,7 +369,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { int numShards = randomFrom(1, 5); int numReplicas = randomFrom(0, 1); int finalNumReplicas = (numReplicas + 1) % 2; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)); AllocateAction allocateAction = new AllocateAction(finalNumReplicas, null, null, null); String endPhase = randomFrom("warm", "cold"); @@ -383,7 +383,7 @@ public void testAllocateActionOnlyReplicas() throws Exception { } public void testWaitForSnapshot() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String slmPolicy = randomAlphaOfLengthBetween(4, 10); createNewSingletonPolicy(client(), policy, "delete", new WaitForSnapshotAction(slmPolicy)); @@ -413,7 +413,7 @@ public void testWaitForSnapshot() throws Exception { } public void testWaitForSnapshotSlmExecutedBefore() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); String slmPolicy = randomAlphaOfLengthBetween(4, 10); createNewSingletonPolicy(client(), policy, "delete", new WaitForSnapshotAction(slmPolicy)); @@ -459,7 +459,7 @@ public void testWaitForSnapshotSlmExecutedBefore() throws Exception { } public void testDelete() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); updatePolicy(index, policy); @@ -467,7 +467,7 @@ public void testDelete() throws Exception { } public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction(), TimeValue.timeValueHours(1)); updatePolicy(index, policy); @@ -496,7 +496,7 @@ public void testDeleteDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "delete", new DeleteAction(), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); // index document so snapshot actually does something indexDocument(client(), index); @@ -516,7 +516,7 @@ public void testDeleteDuringSnapshot() throws Exception { } public void testReadOnly() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ReadOnlyAction()); updatePolicy(index, policy); @@ -527,9 +527,8 @@ public void testReadOnly() throws Exception { }); } - @SuppressWarnings("unchecked") public void forceMergeActionWithCodec(String codec) throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); for (int i = 0; i < randomIntBetween(2, 10); i++) { Request request = new Request("PUT", index + "/_doc/" + i); @@ -538,26 +537,14 @@ public void forceMergeActionWithCodec(String codec) throws Exception { client().performRequest(request); } - Supplier numSegments = () -> { - try { - Map segmentResponse = getAsMap(index + "/_segments"); - segmentResponse = (Map) segmentResponse.get("indices"); - segmentResponse = (Map) segmentResponse.get(index); - segmentResponse = (Map) segmentResponse.get("shards"); - List> shards = (List>) segmentResponse.get("0"); - return (Integer) shards.get(0).get("num_search_segments"); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - assertThat(numSegments.get(), greaterThan(1)); + assertThat(getNumberOfSegments(client(), index), greaterThan(1)); createNewSingletonPolicy(client(), policy, "warm", new ForceMergeAction(1, codec)); updatePolicy(index, policy); assertBusy(() -> { assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("warm").getKey())); Map settings = getOnlyIndexSettings(client(), index); - assertThat(numSegments.get(), equalTo(1)); + assertThat(getNumberOfSegments(client(), index), equalTo(1)); assertThat(settings.get(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true")); }); expectThrows(ResponseException.class, () -> indexDocument(client(), index)); @@ -572,7 +559,7 @@ public void testShrinkAction() throws Exception { int divisor = randomFrom(2, 4); int expectedFinalShards = numShards / divisor; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(expectedFinalShards)); updatePolicy(index, policy); @@ -591,7 +578,7 @@ public void testShrinkAction() throws Exception { public void testShrinkSameShards() throws Exception { int numberOfShards = randomFrom(1, 2); String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(numberOfShards)); updatePolicy(index, policy); @@ -625,7 +612,7 @@ public void testShrinkDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(1), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder() + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // required so the shrink doesn't wait on SetSingleNodeAllocateStep @@ -659,7 +646,7 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception int numShards = 2; int expectedFinalShards = 1; String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); ensureGreen(index); @@ -711,7 +698,7 @@ public void testSetSingleNodeAllocationRetriesUntilItSucceeds() throws Exception } public void testFreezeAction() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); createNewSingletonPolicy(client(), policy, "cold", new FreezeAction()); updatePolicy(index, policy); @@ -741,7 +728,7 @@ public void testFreezeDuringSnapshot() throws Exception { // create delete policy createNewSingletonPolicy(client(), policy, "cold", new FreezeAction(), TimeValue.timeValueMillis(0)); // create index without policy - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)); // index document so snapshot actually does something indexDocument(client(), index); @@ -766,7 +753,7 @@ public void testFreezeDuringSnapshot() throws Exception { } public void testSetPriority() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100)); int priority = randomIntBetween(0, 99); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction(priority)); @@ -779,7 +766,7 @@ public void testSetPriority() throws Exception { } public void testSetNullPriority() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 100)); createNewSingletonPolicy(client(), policy, "warm", new SetPriorityAction((Integer) null)); updatePolicy(index, policy); @@ -904,7 +891,9 @@ public void testRemoveAndReaddPolicy() throws Exception { // Set up a policy with rollover createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); createIndexWithSettings( + client(), originalIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -940,7 +929,7 @@ public void testMoveToInjectedStep() throws Exception { String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; createNewSingletonPolicy(client(), policy, "warm", new ShrinkAction(1), TimeValue.timeValueHours(12)); - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)); @@ -976,7 +965,7 @@ public void testMoveToInjectedStep() throws Exception { public void testMoveToStepRereadsPolicy() throws Exception { createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, TimeValue.timeValueHours(1), null), TimeValue.ZERO); - createIndexWithSettings("test-1", Settings.builder() + createIndexWithSettings(client(), "test-1", alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1011,7 +1000,7 @@ public void testMoveToStepRereadsPolicy() throws Exception { } public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception { - createIndexWithSettings(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5,15))); @@ -1057,7 +1046,7 @@ public void testExplainFilters() throws Exception { assertOK(client().performRequest(request)); } - createIndexWithSettings(goodIndex, Settings.builder() + createIndexWithSettings(client(), goodIndex, alias, Settings.builder() .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1113,7 +1102,9 @@ public void testILMRolloverRetriesOnReadOnlyBlock() throws Exception { // create the index as readonly and associate the ILM policy to it createIndexWithSettings( + client(), firstIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1159,7 +1150,9 @@ public void testILMRolloverOnManuallyRolledIndex() throws Exception { client().performRequest(createIndexTemplate); createIndexWithSettings( + client(), originalIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), true @@ -1204,7 +1197,9 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except // create the rolled index so the rollover of the first index fails createIndexWithSettings( + client(), rolledIndex, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), @@ -1212,7 +1207,9 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except ); createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1276,7 +1273,9 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1328,7 +1327,7 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing( public void testWaitForActiveShardsStep() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; - createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + createIndexWithSettings(client(), originalIndex, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias), true); @@ -1375,7 +1374,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { "}"); client().performRequest(createIndexTemplate); - createIndexWithSettings(index + "-1", Settings.builder(), true); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), true); // Index a document index(client(), index + "-1", "1", "foo", "bar"); @@ -1401,7 +1400,7 @@ public void testHistoryIsWrittenWithSuccess() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353") public void testHistoryIsWrittenWithFailure() throws Exception { - createIndexWithSettings(index + "-1", Settings.builder(), false); + createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L)); updatePolicy(index + "-1", policy); @@ -1420,7 +1419,7 @@ public void testHistoryIsWrittenWithFailure() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53718") public void testHistoryIsWrittenWithDeletion() throws Exception { // Index should be created and then deleted by ILM - createIndexWithSettings(index, Settings.builder(), false); + createIndexWithSettings(client(), index, alias, Settings.builder(), false); createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()); updatePolicy(index, policy); @@ -1444,7 +1443,9 @@ public void testRetryableInitializationStep() throws Exception { // Create the index with the origination parsing turn *off* so it doesn't prevent creation createIndexWithSettings( + client(), index, + alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) @@ -1495,7 +1496,7 @@ public void testRefreshablePhaseJson() throws Exception { "}"); client().performRequest(createIndexTemplate); - createIndexWithSettings(index + "-1", + createIndexWithSettings(client(), index + "-1", alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0), true); @@ -1519,7 +1520,7 @@ public void testHaltAtEndOfPhase() throws Exception { createNewSingletonPolicy(client(), policy, "hot", new SetPriorityAction(100)); - createIndexWithSettings(index, + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -1551,88 +1552,6 @@ public void testHaltAtEndOfPhase() throws Exception { assertBusy(() -> assertFalse("expected " + index + " to be deleted by ILM", indexExists(index))); } - public void testSearchableSnapshotAction() throws Exception { - String snapshotRepo = randomAlphaOfLengthBetween(4, 10); - createSnapshotRepo(client(), snapshotRepo, randomBoolean()); - createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo)); - - createIndexWithSettings(index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); - - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; - assertTrue(waitUntil(() -> { - try { - return indexExists(restoredIndexName); - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - - assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, - TimeUnit.SECONDS); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") - public void testDeleteActionDeletesSearchableSnapshot() throws Exception { - String snapshotRepo = randomAlphaOfLengthBetween(4, 10); - createSnapshotRepo(client(), snapshotRepo, randomBoolean()); - - // create policy with cold and delete phases - Map coldActions = - Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); - Map phases = new HashMap<>(); - phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); - phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), singletonMap(DeleteAction.NAME, - new DeleteAction(true)))); - LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); - // PUT policy - XContentBuilder builder = jsonBuilder(); - lifecyclePolicy.toXContent(builder, null); - final StringEntity entity = new StringEntity( - "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); - Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); - createPolicyRequest.setEntity(entity); - assertOK(client().performRequest(createPolicyRequest)); - - createIndexWithSettings(index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy), - randomBoolean()); - - String[] snapshotName = new String[1]; - String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + this.index; - assertTrue(waitUntil(() -> { - try { - Map explainIndex = explainIndex(client(), index); - if(explainIndex == null) { - // in case we missed the original index and it was deleted - explainIndex = explainIndex(client(), restoredIndexName); - } - snapshotName[0] = (String) explainIndex.get("snapshot_name"); - return snapshotName[0] != null; - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - assertBusy(() -> assertFalse(indexExists(restoredIndexName))); - - assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> { - try { - Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]); - Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); - return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception"); - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - } - @SuppressWarnings("unchecked") public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); @@ -1655,7 +1574,7 @@ public void testDeleteActionDoesntDeleteSearchableSnapshot() throws Exception { createPolicyRequest.setEntity(entity); assertOK(client().performRequest(createPolicyRequest)); - createIndexWithSettings(index, + createIndexWithSettings(client(), index, alias, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -1812,24 +1731,6 @@ private void createIndexWithSettingsNoAlias(String index, Settings.Builder setti ensureGreen(index); } - private void createIndexWithSettings(String index, Settings.Builder settings) throws IOException { - createIndexWithSettings(index, settings, randomBoolean()); - } - - private void createIndexWithSettings(String index, Settings.Builder settings, boolean useWriteIndex) throws IOException { - Request request = new Request("PUT", "/" + index); - - String writeIndexSnippet = ""; - if (useWriteIndex) { - writeIndexSnippet = "\"is_write_index\": true"; - } - request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build()) - + ", \"aliases\" : { \"" + alias + "\": { " + writeIndexSnippet + " } } }"); - client().performRequest(request); - // wait for the shards to initialize - ensureGreen(index); - } - private static void index(RestClient client, String index, String id, Object... fields) throws IOException { XContentBuilder document = jsonBuilder().startObject(); for (int i = 0; i < fields.length; i += 2) { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java new file mode 100644 index 0000000000000..a8a9ebae2ab96 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ilm.actions; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +public class SearchableSnapshotActionIT extends ESRestTestCase { + + private String policy; + private String dataStream; + + @Before + public void refreshIndex() { + dataStream = "logs-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + policy = "policy-" + randomAlphaOfLength(5); + } + + public void testSearchableSnapshotAction() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true)); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)); + + indexDocument(client(), dataStream, true); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, + TimeUnit.SECONDS); + } + + public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true)); + + createComposableTemplate(client(), "template-name", dataStream, new Template(null, null, null)); + + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStream, true); + } + + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + assertThat(getNumberOfSegments(client(), backingIndexName), greaterThan(1)); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + updateIndexSettings(dataStream, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy)); + assertTrue(waitUntil(() -> { + try { + return getNumberOfSegments(client(), backingIndexName) == 1; + } catch (IOException e) { + return false; + } + }, 60, TimeUnit.SECONDS)); + + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, + TimeUnit.SECONDS); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") + public void testDeleteActionDeletesSearchableSnapshot() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + + // create policy with cold and delete phases + Map coldActions = + Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); + Map phases = new HashMap<>(); + phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); + phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), singletonMap(DeleteAction.NAME, + new DeleteAction(true)))); + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); + // PUT policy + XContentBuilder builder = jsonBuilder(); + lifecyclePolicy.toXContent(builder, null); + final StringEntity entity = new StringEntity( + "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); + Request createPolicyRequest = new Request("PUT", "_ilm/policy/" + policy); + createPolicyRequest.setEntity(entity); + assertOK(client().performRequest(createPolicyRequest)); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), null, null)); + + indexDocument(client(), dataStream, true); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + String[] snapshotName = new String[1]; + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName; + assertTrue(waitUntil(() -> { + try { + Map explainIndex = explainIndex(client(), backingIndexName); + if (explainIndex == null) { + // in case we missed the original index and it was deleted + explainIndex = explainIndex(client(), restoredIndexName); + } + snapshotName[0] = (String) explainIndex.get("snapshot_name"); + return snapshotName[0] != null; + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + assertBusy(() -> assertFalse(indexExists(restoredIndexName))); + + assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> { + try { + Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]); + Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); + return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception"); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + } + +}