From 7b68166d004f37390f415e4669e7297133f996ac Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 9 Nov 2020 18:10:30 +0000 Subject: [PATCH 01/16] Enable the searchable_snapshot action in the hot phase --- .../actions/ilm-searchable-snapshot.asciidoc | 6 +- .../xpack/core/ilm/ForceMergeAction.java | 5 ++ .../xpack/core/ilm/FreezeAction.java | 6 +- .../xpack/core/ilm/ShrinkAction.java | 10 +-- .../core/ilm/TimeseriesLifecycleType.java | 29 ++++++- .../xpack/core/ilm/ForceMergeActionTests.java | 40 ++++++---- .../xpack/core/ilm/FreezeActionTests.java | 17 ++-- .../ilm/TimeseriesLifecycleTypeTests.java | 20 ++++- .../xpack/TimeSeriesRestDriver.java | 29 +++++++ .../actions/SearchableSnapshotActionIT.java | 77 +++++++++++++++++++ 10 files changed, 207 insertions(+), 32 deletions(-) diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 667453e3a431f..19da3374ba8f4 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -4,13 +4,17 @@ beta::[] -Phases allowed: cold. +Phases allowed: hot, cold. Takes a snapshot of the managed index in the configured repository and mounts it as a searchable snapshot. If the managed index is part of a <>, the mounted index replaces the original index in the data stream. +IMPORTANT: If the `searchable_snapshot` action is used in the `hot` phase the +subsequent phases cannot define any of the `shrink`, `forcemerge` or `freeze` +actions. + [NOTE] This action cannot be performed on a data stream's write index. Attempts to do so will fail. To convert the index to a searchable snapshot, first diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index 28c72557b3f78..b76d230d54f9e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -34,6 +34,7 @@ public class ForceMergeAction implements LifecycleAction { public static final String NAME = "forcemerge"; public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); public static final ParseField CODEC = new ParseField("index_codec"); + public static final String CONDITIONAL_SKIP_FORCE_MERGE_STEP = BranchingStep.NAME + "-forcemerge-check-prerequisites"; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, false, a -> { @@ -120,6 +121,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) final boolean codecChange = codec != null && codec.equals(CodecService.BEST_COMPRESSION_CODEC); + StepKey preForceMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FORCE_MERGE_STEP); StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); @@ -131,6 +133,8 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME); StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME); + BranchingStep conditionalSkipShrinkStep = new BranchingStep(preForceMergeBranchingKey, checkNotWriteIndex, nextStepKey, + (index, clusterState) -> clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); UpdateSettingsStep readOnlyStep = @@ -147,6 +151,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) SegmentCountStep segmentCountStep = new SegmentCountStep(countKey, nextStepKey, client, maxNumSegments); List mergeSteps = new ArrayList<>(); + mergeSteps.add(conditionalSkipShrinkStep); mergeSteps.add(checkNotWriteIndexStep); mergeSteps.add(readOnlyStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 083a015bb91ac..9791c16531ffb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -23,6 +23,7 @@ */ public class FreezeAction implements LifecycleAction { public static final String NAME = "freeze"; + public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites"; private static final ObjectParser PARSER = new ObjectParser<>(NAME, FreezeAction::new); @@ -59,13 +60,16 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey preFreezeMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FREEZE_STEP); StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey freezeStepKey = new StepKey(phase, NAME, FreezeStep.NAME); + BranchingStep conditionalSkipFreezeStep = new BranchingStep(preFreezeMergeBranchingKey, checkNotWriteIndex, nextStepKey, + (index, clusterState) -> clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null); CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, freezeStepKey); FreezeStep freezeStep = new FreezeStep(freezeStepKey, nextStepKey, client); - return Arrays.asList(checkNoWriteIndexStep, freezeStep); + return Arrays.asList(conditionalSkipFreezeStep, checkNoWriteIndexStep, freezeStep); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index c278de30e3b4f..6524b23198517 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -35,8 +33,6 @@ public class ShrinkAction implements LifecycleAction { public static final String CONDITIONAL_SKIP_SHRINK_STEP = BranchingStep.NAME + "-check-prerequisites"; public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; - private static final Logger logger = LogManager.getLogger(ShrinkAction.class); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0])); @@ -108,7 +104,11 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); BranchingStep conditionalSkipShrinkStep = new BranchingStep(preShrinkBranchingKey, checkNotWriteIndex, nextStepKey, - (index, clusterState) -> clusterState.getMetadata().index(index).getNumberOfShards() == numberOfShards); + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + return indexMetadata.getNumberOfShards() == numberOfShards || + indexMetadata.getSettings().get("index.store.snapshot.index_name") != null; + }); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, waitForNoFollowerStepKey); WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index d3bf38892444e..c8ce87a6d674b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +41,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; static final List VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE); static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); + ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, @@ -58,6 +59,9 @@ public class TimeseriesLifecycleType implements LifecycleType { static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); + // a set of actions that cannot be defined (executed) after the managed index has been mounted as searchable snapshot + static final Set ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME, + FreezeAction.NAME); private TimeseriesLifecycleType() { } @@ -255,6 +259,29 @@ public void validate(Collection phases) { MigrateAction.NAME + " action and an " + AllocateAction.NAME + " action with allocation rules. specify only a single " + "data migration in each phase"); } + + validateActionsFollowingSearchableSnapshot(phases); + } + + static void validateActionsFollowingSearchableSnapshot(Collection phases) { + boolean hotPhaseContainsSearchableSnapshot = phases.stream() + .filter(phase -> HOT_PHASE.equals(phase.getName())) + .anyMatch(phase -> phase.getActions().containsKey(SearchableSnapshotAction.NAME)); + if (hotPhaseContainsSearchableSnapshot) { + String phasesDefiningIllegalActions = phases.stream() + // we're looking for prohibited actions in phases other than hot + .filter(phase -> HOT_PHASE.equals(phase.getName()) == false) + // filter the phases that define illegal actions + .filter(phase -> + Collections.disjoint(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, phase.getActions().keySet()) == false) + .map(Phase::getName) + .collect(Collectors.joining(",")); + if (Strings.hasText(phasesDefiningIllegalActions)) { + throw new IllegalArgumentException("phases [" + phasesDefiningIllegalActions + "] define one or more of [" + + String.join(",", ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT) + "] actions which are not allowed after the " + + "managed index was mounted as searchable snapshot"); + } + } } private static boolean definesAllocationRules(AllocateAction action) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java index cb5d6f9387149..0122cdfe4c04f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java @@ -24,6 +24,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class ForceMergeActionTests extends AbstractActionTestCase { @@ -65,21 +66,24 @@ private void assertNonBestCompression(ForceMergeAction instance) { StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); List steps = instance.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(4, steps.size()); - CheckNotDataStreamWriteIndexStep firstStep = (CheckNotDataStreamWriteIndexStep) steps.get(0); - UpdateSettingsStep secondStep = (UpdateSettingsStep) steps.get(1); - ForceMergeStep thirdStep = (ForceMergeStep) steps.get(2); - SegmentCountStep fourthStep = (SegmentCountStep) steps.get(3); - - assertThat(firstStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME))); - assertThat(firstStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); - assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); - assertThat(secondStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); - assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(secondStep.getSettings())); - assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); - assertThat(thirdStep.getNextStepKey(), equalTo(fourthStep.getKey())); - assertThat(fourthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME))); - assertThat(fourthStep.getNextStepKey(), equalTo(nextStepKey)); + assertEquals(5, steps.size()); + BranchingStep firstStep = (BranchingStep) steps.get(0); + CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1); + UpdateSettingsStep thirdStep = (UpdateSettingsStep) steps.get(2); + ForceMergeStep fourthStep = (ForceMergeStep) steps.get(3); + SegmentCountStep fifthStep = (SegmentCountStep) steps.get(4); + + assertThat(firstStep.getKey(), + equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP))); + assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME))); + assertThat(secondStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); + assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); + assertThat(thirdStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); + assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(thirdStep.getSettings())); + assertThat(fourthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); + assertThat(fourthStep.getNextStepKey(), equalTo(fifthStep.getKey())); + assertThat(fifthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME))); + assertThat(fifthStep.getNextStepKey(), equalTo(nextStepKey)); } private void assertBestCompression(ForceMergeAction instance) { @@ -87,8 +91,11 @@ private void assertBestCompression(ForceMergeAction instance) { StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); List steps = instance.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(8, steps.size()); + assertEquals(9, steps.size()); List> stepKeys = steps.stream() + // skip the first branching step as `performAction` needs to be executed to evaluate the condition before the next step is + // available + .skip(1) .map(s -> new Tuple<>(s.getKey(), s.getNextStepKey())) .collect(Collectors.toList()); StepKey checkNotWriteIndex = new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); @@ -99,6 +106,7 @@ private void assertBestCompression(ForceMergeAction instance) { StepKey waitForGreen = new StepKey(phase, ForceMergeAction.NAME, WaitForIndexColorStep.NAME); StepKey forceMerge = new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME); StepKey segmentCount = new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME); + assertThat(steps.get(0).getKey(), is(ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP)); assertThat(stepKeys, contains( new Tuple<>(checkNotWriteIndex, readOnly), new Tuple<>(readOnly, closeIndex), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java index 71f2d963b1c6b..7966bc8b83b1a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java @@ -38,17 +38,20 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(2, steps.size()); - StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); - StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME); + assertEquals(3, steps.size()); + StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP); + StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); + StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME); - CheckNotDataStreamWriteIndexStep firstStep = (CheckNotDataStreamWriteIndexStep) steps.get(0); - FreezeStep secondStep = (FreezeStep) steps.get(1); + BranchingStep firstStep = (BranchingStep) steps.get(0); + CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1); + FreezeStep thirdStep = (FreezeStep) steps.get(2); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); - assertThat(firstStep.getNextStepKey(), equalTo(expectedSecondStepKey)); assertEquals(expectedSecondStepKey, secondStep.getKey()); - assertEquals(nextStepKey, secondStep.getNextStepKey()); + assertEquals(expectedThirdStepKey, secondStep.getNextStepKey()); + assertEquals(expectedThirdStepKey, thirdStep.getKey()); + assertEquals(nextStepKey, thirdStep.getNextStepKey()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index ee51129c05964..c313bac7bd9c7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -20,6 +20,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.HOT_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_COLD_ACTIONS; @@ -32,6 +33,7 @@ import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_PHASES; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_WARM_ACTIONS; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.WARM_PHASE; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -191,13 +193,29 @@ public void testValidateConflictingDataMigrationConfigurations() { } } + public void testActionsThatCannotFollowSearchableSnapshot() { + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(3)); + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, containsInAnyOrder(ShrinkAction.NAME, FreezeAction.NAME, + ForceMergeAction.NAME)); + } + + public void testValidateActionsFollowingSearchableSnapshot() { + Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction("repo"))); + Phase warmPhase = new Phase("warm", TimeValue.ZERO, Map.of(ShrinkAction.NAME, new ShrinkAction(1))); + Phase coldPhase = new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TimeseriesLifecycleType.validateActionsFollowingSearchableSnapshot(List.of(hotPhase, warmPhase, coldPhase))); + assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [shrink,forcemerge,freeze] actions which are not allowed" + + " after the managed index was mounted as searchable snapshot")); + } + public void testGetOrderedPhases() { Map phaseMap = new HashMap<>(); for (String phaseName : randomSubsetOf(randomIntBetween(0, VALID_PHASES.size()), VALID_PHASES)) { phaseMap.put(phaseName, new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap())); } - assertTrue(isSorted(TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap), Phase::getName, VALID_PHASES)); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index cb55434c38944..1f675dc169bdd 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -191,6 +192,34 @@ public static void createFullPolicy(RestClient client, String policyName, TimeVa client.performRequest(request); } + public static void createPolicy(RestClient client, String policyName, @Nullable Phase hotPhase, @Nullable Phase warmPhase, + @Nullable Phase coldPhase, @Nullable Phase deletePhase) throws IOException { + if (hotPhase == null && warmPhase == null && coldPhase == null && deletePhase == null) { + throw new IllegalArgumentException("specify at least one phase"); + } + Map phases = new HashMap<>(); + if (hotPhase != null) { + phases.put("hot", hotPhase); + } + if (warmPhase != null) { + phases.put("warm", warmPhase); + } + if (coldPhase != null) { + phases.put("cold", coldPhase); + } + if (deletePhase != null) { + phases.put("delete", deletePhase); + } + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policyName, phases); + XContentBuilder builder = jsonBuilder(); + lifecyclePolicy.toXContent(builder, null); + final StringEntity entity = new StringEntity( + "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); + Request request = new Request("PUT", "_ilm/policy/" + policyName); + request.setEntity(entity); + client.performRequest(request); + } + public static void createSnapshotRepo(RestClient client, String repoName, boolean compress) throws IOException { Request request = new Request("PUT", "/_snapshot/" + repoName); request.setJsonEntity(Strings diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index e1be0bf7ccc3d..443cfac67f3ca 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -19,12 +20,17 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.ForceMergeAction; +import org.elasticsearch.xpack.core.ilm.FreezeAction; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; +import org.elasticsearch.xpack.core.ilm.SetPriorityAction; +import org.elasticsearch.xpack.core.ilm.ShrinkAction; +import org.elasticsearch.xpack.core.ilm.Step; import org.junit.Before; import java.io.IOException; @@ -37,9 +43,11 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.hamcrest.Matchers.greaterThan; @@ -188,4 +196,73 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { }, 30, TimeUnit.SECONDS)); } + public void testCreateInvalidPolicy() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("warm", TimeValue.ZERO, Map.of(ForceMergeAction.NAME, new ForceMergeAction(1, null))), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + null + ) + ); + + assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [shrink,forcemerge,freeze] actions which are not " + + "allowed after the managed index was mounted as searchable snapshot")); + } + + public void testRestoredIndexToInvalidPolicySkipsInvalidActions() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), + null, null + ); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .build(), null, null) + ); + + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStream, true); + } + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + DataStream.getDefaultBackingIndexName(dataStream, 1L); + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), restoredIndexName); + assertThat(stepKeyForIndex.getPhase(), is("hot")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(SetPriorityAction.NAME, new SetPriorityAction(10))), + new Phase("warm", TimeValue.ZERO, + Map.of(ShrinkAction.NAME, new ShrinkAction(1), ForceMergeAction.NAME, new ForceMergeAction(1, null)) + ), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + null + ); + + // even though the index is now mounted as a searchable snapshot, the actions that can't operate on it should + // skip and ILM should not be blocked (not should the managed index move into the ERROR step) + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), restoredIndexName); + assertThat(stepKeyForIndex.getPhase(), is("cold")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + } } From 0e92f001e7df3ddb1732059d436faaf054d21f67 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 10 Nov 2020 17:03:04 +0000 Subject: [PATCH 02/16] Add IT for restoring index mounted as searchable snapshot --- .../actions/SearchableSnapshotActionIT.java | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 443cfac67f3ca..46900b3cff770 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -211,7 +211,7 @@ public void testCreateInvalidPolicy() throws Exception { "allowed after the managed index was mounted as searchable snapshot")); } - public void testRestoredIndexToInvalidPolicySkipsInvalidActions() throws Exception { + public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); createSnapshotRepo(client(), snapshotRepo, randomBoolean()); createPolicy(client(), policy, @@ -265,4 +265,80 @@ public void testRestoredIndexToInvalidPolicySkipsInvalidActions() throws Excepti assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); }, 30, TimeUnit.SECONDS); } + + public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Exception{ + // let's create a data stream, rollover it and convert the first generation backing index into a searchable snapshot + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), + null, null + ); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .build(), null, null) + ); + + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStream, true); + } + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + rolloverMaxOneDocCondition(client(), dataStream); + + String searchableSnapMountedIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + + DataStream.getDefaultBackingIndexName(dataStream, 1L); + assertTrue(waitUntil(() -> { + try { + return indexExists(searchableSnapMountedIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + assertThat(stepKeyForIndex.getPhase(), is("hot")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + + // snapshot the data stream + String dsSnapshotName = "snapshot_ds_" + dataStream; + Request takeSnapshotRequest = new Request("PUT", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName); + takeSnapshotRequest.addParameter("wait_for_completion", "true"); + takeSnapshotRequest.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}"); + assertOK(client().performRequest(takeSnapshotRequest)); + + // now that we have a backup of the data stream, let's delete the local one and update the ILM policy to include some illegal + // actions for when we restore the data stream (given that the first generation backing index will be backed by a searchable + // snapshot) + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStream))); + + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of()), + new Phase("warm", TimeValue.ZERO, + Map.of(ShrinkAction.NAME, new ShrinkAction(1), ForceMergeAction.NAME, new ForceMergeAction(1, null)) + ), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + null + ); + + // restore the datastream + Request restoreSnapshot = new Request("POST", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName + "/_restore"); + restoreSnapshot.addParameter("wait_for_completion", "true"); + restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": true}"); + assertOK(client().performRequest(restoreSnapshot)); + + assertThat(indexExists(searchableSnapMountedIndexName), is(true)); + + // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + assertThat(stepKeyForIndex.getPhase(), is("cold")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + } } From b278e052e5364d90f679310656a0f92f6d65d70d Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 10 Nov 2020 17:48:33 +0000 Subject: [PATCH 03/16] Add logging --- .../xpack/core/ilm/ForceMergeAction.java | 13 ++++++++++++- .../xpack/core/ilm/FreezeAction.java | 13 ++++++++++++- .../xpack/core/ilm/ShrinkAction.java | 15 +++++++++++++-- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index b76d230d54f9e..d861a3720f37c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -31,6 +33,8 @@ * A {@link LifecycleAction} which force-merges the index. */ public class ForceMergeAction implements LifecycleAction { + private final Logger logger = LogManager.getLogger(ForceMergeAction.class); + public static final String NAME = "forcemerge"; public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); public static final ParseField CODEC = new ParseField("index_codec"); @@ -134,7 +138,14 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME); BranchingStep conditionalSkipShrinkStep = new BranchingStep(preForceMergeBranchingKey, checkNotWriteIndex, nextStepKey, - (index, clusterState) -> clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null); + (index, clusterState) -> { + if (clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null) { + logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", + ForceMergeAction.NAME, index.getName()); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); UpdateSettingsStep readOnlyStep = diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 9791c16531ffb..32ff1d49c4d02 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -22,6 +24,8 @@ * A {@link LifecycleAction} which freezes the index. */ public class FreezeAction implements LifecycleAction { + private final Logger logger = LogManager.getLogger(FreezeAction.class); + public static final String NAME = "freeze"; public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites"; @@ -65,7 +69,14 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey freezeStepKey = new StepKey(phase, NAME, FreezeStep.NAME); BranchingStep conditionalSkipFreezeStep = new BranchingStep(preFreezeMergeBranchingKey, checkNotWriteIndex, nextStepKey, - (index, clusterState) -> clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null); + (index, clusterState) -> { + if (clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null) { + logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", + FreezeAction.NAME, index.getName()); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, freezeStepKey); FreezeStep freezeStep = new FreezeStep(freezeStepKey, nextStepKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 6524b23198517..0ef8c88027c09 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -27,6 +29,8 @@ * A {@link LifecycleAction} which shrinks the index. */ public class ShrinkAction implements LifecycleAction { + private final Logger logger = LogManager.getLogger(ShrinkAction.class); + public static final String NAME = "shrink"; public static final String SHRUNKEN_INDEX_PREFIX = "shrink-"; public static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards"); @@ -106,8 +110,15 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) BranchingStep conditionalSkipShrinkStep = new BranchingStep(preShrinkBranchingKey, checkNotWriteIndex, nextStepKey, (index, clusterState) -> { IndexMetadata indexMetadata = clusterState.getMetadata().index(index); - return indexMetadata.getNumberOfShards() == numberOfShards || - indexMetadata.getSettings().get("index.store.snapshot.index_name") != null; + if (indexMetadata.getNumberOfShards() == numberOfShards) { + return true; + } + if (indexMetadata.getSettings().get("index.store.snapshot.index_name") != null) { + logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", + ShrinkAction.NAME, indexMetadata.getIndex().getName()); + return true; + } + return false; }); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, waitForNoFollowerStepKey); From 97ec99fe2733c8fbd36433c2e161a72f2716e883 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 10 Nov 2020 17:54:07 +0000 Subject: [PATCH 04/16] Fix test --- .../xpack/ilm/action/TransportPutLifecycleActionTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index bd1447625089c..8f9d28a8940b0 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -161,10 +161,12 @@ public void testReadStepKeys() { assertThat(TransportPutLifecycleAction.readStepKeys(REGISTRY, client, phaseDef, "phase"), contains( + new Step.StepKey("phase", "freeze", FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP), new Step.StepKey("phase", "freeze", CheckNotDataStreamWriteIndexStep.NAME), new Step.StepKey("phase", "freeze", FreezeAction.NAME), new Step.StepKey("phase", "allocate", AllocateAction.NAME), new Step.StepKey("phase", "allocate", AllocationRoutedStep.NAME), + new Step.StepKey("phase", "forcemerge", ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP), new Step.StepKey("phase", "forcemerge", CheckNotDataStreamWriteIndexStep.NAME), new Step.StepKey("phase", "forcemerge", ReadOnlyAction.NAME), new Step.StepKey("phase", "forcemerge", ForceMergeAction.NAME), From 2bd01c59388950bdbc524377a699526bf79ea540 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 11:58:57 +0000 Subject: [PATCH 05/16] Update error message Co-authored-by: Lee Hinman --- .../xpack/core/ilm/TimeseriesLifecycleType.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index c8ce87a6d674b..a54d067e30af6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -277,9 +277,9 @@ static void validateActionsFollowingSearchableSnapshot(Collection phases) .map(Phase::getName) .collect(Collectors.joining(",")); if (Strings.hasText(phasesDefiningIllegalActions)) { - throw new IllegalArgumentException("phases [" + phasesDefiningIllegalActions + "] define one or more of [" + - String.join(",", ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT) + "] actions which are not allowed after the " + - "managed index was mounted as searchable snapshot"); + throw new IllegalArgumentException("phases [" + phasesDefiningIllegalActions + "] define one or more of " + + ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT + " actions which are not allowed after a " + + "managed index is mounted as a searchable snapshot"); } } } From b663f72112cecef22d90b6eb3f54d7e1891af00a Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 16:27:52 +0000 Subject: [PATCH 06/16] Make ensure `searchable_snapshot` action cannot follow itself and cleanups --- .../xpack/core/ilm/ForceMergeAction.java | 9 ++++--- .../xpack/core/ilm/FreezeAction.java | 15 ++++++++--- .../xpack/core/ilm/LifecycleSettings.java | 2 ++ .../core/ilm/SearchableSnapshotAction.java | 20 ++++++++++++++ .../xpack/core/ilm/ShrinkAction.java | 7 ++--- .../core/ilm/TimeseriesLifecycleType.java | 4 +-- .../xpack/core/ilm/LifecyclePolicyTests.java | 18 ++++++++++--- .../ilm/SearchableSnapshotActionTests.java | 14 +++++----- .../ilm/TimeseriesLifecycleTypeTests.java | 4 +-- .../actions/SearchableSnapshotActionIT.java | 26 +++++++++---------- 10 files changed, 83 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index d861a3720f37c..7c4658a45349a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -139,9 +139,12 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) BranchingStep conditionalSkipShrinkStep = new BranchingStep(preForceMergeBranchingKey, checkNotWriteIndex, nextStepKey, (index, clusterState) -> { - if (clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null) { - logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", - ForceMergeAction.NAME, index.getName()); + IndexMetadata indexMetadata = clusterState.metadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings()); + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", ForceMergeAction.NAME, index.getName(), policyName); return true; } return false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 32ff1d49c4d02..61901b67f2864 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -70,9 +71,17 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { BranchingStep conditionalSkipFreezeStep = new BranchingStep(preFreezeMergeBranchingKey, checkNotWriteIndex, nextStepKey, (index, clusterState) -> { - if (clusterState.getMetadata().index(index).getSettings().get("index.store.snapshot.index_name") != null) { - logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", - FreezeAction.NAME, index.getName()); + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings()); + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", FreezeAction.NAME, index.getName(), policyName); + return true; + } + if (indexMetadata.getSettings().getAsBoolean("index.frozen", false)) { + logger.debug("skipping [{}] action for index [{}] in policy [{}] as the index is already frozen", FreezeAction.NAME, + index.getName(), policyName); return true; } return false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java index 3d36e51ce8d09..a0395e8cfe619 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java @@ -27,6 +27,8 @@ public class LifecycleSettings { public static final String SLM_RETENTION_DURATION = "slm.retention_duration"; public static final String SLM_MINIMUM_INTERVAL = "slm.minimum_interval"; + public static final String SNAPSHOT_INDEX_NAME = "index.store.snapshot.index_name"; + public static final Setting LIFECYCLE_POLL_INTERVAL_SETTING = Setting.timeSetting(LIFECYCLE_POLL_INTERVAL, TimeValue.timeValueMinutes(10), TimeValue.timeValueSeconds(1), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index f5a5e5ac00e2d..b7ca1aaa8ed91 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -5,10 +5,13 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,11 +32,14 @@ * newly created searchable snapshot backed index. */ public class SearchableSnapshotAction implements LifecycleAction { + private final Logger logger = LogManager.getLogger(SearchableSnapshotAction.class); + public static final String NAME = "searchable_snapshot"; public static final ParseField SNAPSHOT_REPOSITORY = new ParseField("snapshot_repository"); public static final ParseField FORCE_MERGE_INDEX = new ParseField("force_merge_index"); public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; + public static final String CONDITIONAL_SKIP_ACTION_STEP = BranchingStep.NAME + "-check-prerequisites"; public static final String RESTORED_INDEX_PREFIX = "restored-"; @@ -74,6 +80,7 @@ boolean isForceMergeIndex() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey preActionBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_ACTION_STEP); StepKey checkNoWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); StepKey forceMergeStepKey = new StepKey(phase, NAME, ForceMergeStep.NAME); @@ -90,6 +97,18 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); + BranchingStep conditionalSkipActionStep = new BranchingStep(preActionBranchingKey, checkNoWriteIndex, nextStepKey, + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is already mounted as searchable " + + "snapshot. Skipping this action", SearchableSnapshotAction.NAME, index.getName(), + LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex, waitForNoFollowerStepKey); final WaitForNoFollowersStep waitForNoFollowersStep; @@ -130,6 +149,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { null, client, RESTORED_INDEX_PREFIX); List steps = new ArrayList<>(); + steps.add(conditionalSkipActionStep); steps.add(checkNoWriteIndexStep); steps.add(waitForNoFollowersStep); if (forceMergeIndex) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 0ef8c88027c09..6b59f526b2530 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -113,9 +113,10 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) if (indexMetadata.getNumberOfShards() == numberOfShards) { return true; } - if (indexMetadata.getSettings().get("index.store.snapshot.index_name") != null) { - logger.warn("[{}] action is configured for index [{}] which is mounted as searchable snapshot. Skipping this action", - ShrinkAction.NAME, indexMetadata.getIndex().getName()); + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", ShrinkAction.NAME, indexMetadata.getIndex().getName(), + LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())); return true; } return false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index a54d067e30af6..343dd1a1c9336 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -58,10 +58,10 @@ public class TimeseriesLifecycleType implements LifecycleType { DELETE_PHASE, VALID_DELETE_ACTIONS); static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ReadOnlyAction.NAME, ShrinkAction.NAME, - ForceMergeAction.NAME); + ForceMergeAction.NAME, SearchableSnapshotAction.NAME); // a set of actions that cannot be defined (executed) after the managed index has been mounted as searchable snapshot static final Set ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME, - FreezeAction.NAME); + FreezeAction.NAME, SearchableSnapshotAction.NAME); private TimeseriesLifecycleType() { } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 916184f2eefc3..7595384710ab4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -199,15 +199,25 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; + boolean hotPhaseContainsSearchableSnap = false; for (String phase : phaseNames) { TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); Map actions = new HashMap<>(); List actionNames = randomSubsetOf(validActions.apply(phase)); - // If the hot phase has any actions that require a rollover, then ensure there is one so that the policy will validate - if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) - && actionNames.stream().anyMatch(TimeseriesLifecycleType.HOT_ACTIONS_THAT_REQUIRE_ROLLOVER::contains)) { - actionNames.add(RolloverAction.NAME); + if (phase.equals(TimeseriesLifecycleType.HOT_PHASE)) { + // If the hot phase has any actions that require a rollover, then ensure there is one so that the policy will validate + if (actionNames.stream().anyMatch(TimeseriesLifecycleType.HOT_ACTIONS_THAT_REQUIRE_ROLLOVER::contains)) { + actionNames.add(RolloverAction.NAME); + } + + if (actionNames.contains(SearchableSnapshotAction.NAME)) { + hotPhaseContainsSearchableSnap = true; + } + } else { + // let's make sure the other phases don't configure actions that conflict with a possible `searchable_snapshot` action + // configured in the hot phase + actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); } for (String action : actionNames) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index 48757fc483064..88cf918d05b8e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -24,7 +24,7 @@ public void testToSteps() { StepKey nextStepKey = new StepKey(phase, randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(1, 5)); List steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), is(action.isForceMergeIndex() ? 15 : 13)); + assertThat(steps.size(), is(action.isForceMergeIndex() ? 16 : 14)); List expectedSteps = action.isForceMergeIndex() ? expectedStepKeysWithForceMerge(phase) : expectedStepKeysNoForceMerge(phase); @@ -44,17 +44,18 @@ public void testToSteps() { assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); if (action.isForceMergeIndex()) { - assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(6); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(5))); + assertThat(steps.get(14).getKey(), is(expectedSteps.get(14))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(7); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(6))); } else { - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(3))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(5); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(4))); } } private List expectedStepKeysWithForceMerge(String phase) { return List.of( + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, ForceMergeStep.NAME), @@ -74,6 +75,7 @@ private List expectedStepKeysWithForceMerge(String phase) { private List expectedStepKeysNoForceMerge(String phase) { return List.of( + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index c313bac7bd9c7..4cea575043dc0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -206,8 +206,8 @@ public void testValidateActionsFollowingSearchableSnapshot() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.validateActionsFollowingSearchableSnapshot(List.of(hotPhase, warmPhase, coldPhase))); - assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [shrink,forcemerge,freeze] actions which are not allowed" + - " after the managed index was mounted as searchable snapshot")); + assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [shrink, forcemerge, freeze] actions which are not " + + "allowed after a managed index is mounted as a searchable snapshot")); } public void testGetOrderedPhases() { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 46900b3cff770..84781e1ac900c 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; @@ -196,26 +197,26 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { }, 30, TimeUnit.SECONDS)); } - public void testCreateInvalidPolicy() throws Exception { - String snapshotRepo = randomAlphaOfLengthBetween(4, 10); - createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + public void testCreateInvalidPolicy() { IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createPolicy(client(), policy, - new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), SearchableSnapshotAction.NAME, + new SearchableSnapshotAction(randomAlphaOfLengthBetween(4, 10)))), new Phase("warm", TimeValue.ZERO, Map.of(ForceMergeAction.NAME, new ForceMergeAction(1, null))), new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), null ) ); - assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [shrink,forcemerge,freeze] actions which are not " + - "allowed after the managed index was mounted as searchable snapshot")); + assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [shrink, forcemerge, freeze] actions which are " + + "not allowed after a managed index is mounted as a searchable snapshot")); } public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws Exception { String snapshotRepo = randomAlphaOfLengthBetween(4, 10); createSnapshotRepo(client(), snapshotRepo, randomBoolean()); createPolicy(client(), policy, - new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), SearchableSnapshotAction.NAME, + new SearchableSnapshotAction(snapshotRepo))), new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), null, null ); @@ -227,11 +228,10 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws .build(), null, null) ); + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index for (int i = 0; i < randomIntBetween(5, 10); i++) { indexDocument(client(), dataStream, true); } - // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index - rolloverMaxOneDocCondition(client(), dataStream); String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + DataStream.getDefaultBackingIndexName(dataStream, 1L); assertTrue(waitUntil(() -> { @@ -253,7 +253,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws new Phase("warm", TimeValue.ZERO, Map.of(ShrinkAction.NAME, new ShrinkAction(1), ForceMergeAction.NAME, new ForceMergeAction(1, null)) ), - new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + new Phase("cold", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), null ); @@ -271,7 +271,8 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex String snapshotRepo = randomAlphaOfLengthBetween(4, 10); createSnapshotRepo(client(), snapshotRepo, randomBoolean()); createPolicy(client(), policy, - new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), + SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), null, null ); @@ -283,11 +284,10 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex .build(), null, null) ); + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index for (int i = 0; i < randomIntBetween(5, 10); i++) { indexDocument(client(), dataStream, true); } - // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index - rolloverMaxOneDocCondition(client(), dataStream); String searchableSnapMountedIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + DataStream.getDefaultBackingIndexName(dataStream, 1L); From 82c9e661749e5c4989efc8b70d6bda4bebc125a6 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 16:30:51 +0000 Subject: [PATCH 07/16] Fix tests --- .../xpack/core/ilm/TimeseriesLifecycleTypeTests.java | 8 ++++---- .../xpack/ilm/actions/SearchableSnapshotActionIT.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index 4cea575043dc0..df0017eadd492 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -194,9 +194,9 @@ public void testValidateConflictingDataMigrationConfigurations() { } public void testActionsThatCannotFollowSearchableSnapshot() { - assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(3)); + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(4)); assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, containsInAnyOrder(ShrinkAction.NAME, FreezeAction.NAME, - ForceMergeAction.NAME)); + ForceMergeAction.NAME, SearchableSnapshotAction.NAME)); } public void testValidateActionsFollowingSearchableSnapshot() { @@ -206,8 +206,8 @@ public void testValidateActionsFollowingSearchableSnapshot() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.validateActionsFollowingSearchableSnapshot(List.of(hotPhase, warmPhase, coldPhase))); - assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [shrink, forcemerge, freeze] actions which are not " + - "allowed after a managed index is mounted as a searchable snapshot")); + assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink] actions" + + " which are not allowed after a managed index is mounted as a searchable snapshot")); } public void testGetOrderedPhases() { diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 84781e1ac900c..82c7e543a4d21 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -207,8 +207,8 @@ public void testCreateInvalidPolicy() { ) ); - assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [shrink, forcemerge, freeze] actions which are " + - "not allowed after a managed index is mounted as a searchable snapshot")); + assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink]" + + " actions which are not allowed after a managed index is mounted as a searchable snapshot")); } public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws Exception { From 3feb501be85e64b5aa3a97910dd8cc74bb0ee8dd Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 16:42:58 +0000 Subject: [PATCH 08/16] Fix ForceMergeActionTests --- .../elasticsearch/xpack/core/ilm/ForceMergeActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java index 0122cdfe4c04f..fb7b94526ae0c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java @@ -106,7 +106,8 @@ private void assertBestCompression(ForceMergeAction instance) { StepKey waitForGreen = new StepKey(phase, ForceMergeAction.NAME, WaitForIndexColorStep.NAME); StepKey forceMerge = new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME); StepKey segmentCount = new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME); - assertThat(steps.get(0).getKey(), is(ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP)); + assertThat(steps.get(0).getKey(), is(new StepKey(phase, ForceMergeAction.NAME, + ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP))); assertThat(stepKeys, contains( new Tuple<>(checkNotWriteIndex, readOnly), new Tuple<>(readOnly, closeIndex), From 5e2366de6cf47301b536b72dee3c39fd79bdc087 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 16:45:54 +0000 Subject: [PATCH 09/16] Fix ForceMergeActionTest --- .../xpack/core/ilm/ForceMergeActionTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java index fb7b94526ae0c..730639c210a51 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java @@ -118,11 +118,11 @@ private void assertBestCompression(ForceMergeAction instance) { new Tuple<>(forceMerge, segmentCount), new Tuple<>(segmentCount, nextStepKey))); - UpdateSettingsStep secondStep = (UpdateSettingsStep) steps.get(1); - UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3); + UpdateSettingsStep thirdStep = (UpdateSettingsStep) steps.get(2); + UpdateSettingsStep fifthStep = (UpdateSettingsStep) steps.get(4); - assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(secondStep.getSettings())); - assertThat(fourthStep.getSettings().get(EngineConfig.INDEX_CODEC_SETTING.getKey()), equalTo(CodecService.BEST_COMPRESSION_CODEC)); + assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(thirdStep.getSettings())); + assertThat(fifthStep.getSettings().get(EngineConfig.INDEX_CODEC_SETTING.getKey()), equalTo(CodecService.BEST_COMPRESSION_CODEC)); } public void testMissingMaxNumSegments() throws IOException { From 471d329a6a9a9c10eeb11ef8ca295391041e014e Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 17:14:49 +0000 Subject: [PATCH 10/16] Fix random lifecycle generation --- .../xpack/core/ilm/LifecyclePolicyTests.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 7595384710ab4..0d20abed260a1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -145,6 +145,12 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Null TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); Map actions = new HashMap<>(); Set actionNames = validActions.apply(phase); + if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) == false) { + // let's make sure the other phases don't configure actions that conflict with the `searchable_snapshot` action + // configured in the hot phase + actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); + + } for (String action : actionNames) { actions.put(action, randomAction.apply(action)); } @@ -215,9 +221,11 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l hotPhaseContainsSearchableSnap = true; } } else { - // let's make sure the other phases don't configure actions that conflict with a possible `searchable_snapshot` action - // configured in the hot phase - actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); + if (hotPhaseContainsSearchableSnap) { + // let's make sure the other phases don't configure actions that conflict with a possible `searchable_snapshot` action + // configured in the hot phase + actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); + } } for (String action : actionNames) { From 0c3b35e6ccc39f3168b346689374e9617edd6d3b Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 17:58:15 +0000 Subject: [PATCH 11/16] Fix phases validation when generating test policies --- .../elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 0d20abed260a1..1dba01e2a6eb4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -149,7 +149,6 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Null // let's make sure the other phases don't configure actions that conflict with the `searchable_snapshot` action // configured in the hot phase actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); - } for (String action : actionNames) { actions.put(action, randomAction.apply(action)); @@ -205,6 +204,12 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; + // as what actions end up in the hot phase influence what actions are allowed in the subsequent phases we'll move the hot phase + // at the front of the phases to process (if it exists) + if (phaseNames.contains(TimeseriesLifecycleType.HOT_PHASE)) { + phaseNames.remove(TimeseriesLifecycleType.HOT_PHASE); + phaseNames.add(0, TimeseriesLifecycleType.HOT_PHASE); + } boolean hotPhaseContainsSearchableSnap = false; for (String phase : phaseNames) { TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); From 1128f0766b3d140960c4642962476d4ad527084d Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 11 Nov 2020 18:25:18 +0000 Subject: [PATCH 12/16] Update doc --- docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 19da3374ba8f4..864198fae7f42 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -12,8 +12,8 @@ If the managed index is part of a <>, the mounted index replaces the original index in the data stream. IMPORTANT: If the `searchable_snapshot` action is used in the `hot` phase the -subsequent phases cannot define any of the `shrink`, `forcemerge` or `freeze` -actions. +subsequent phases cannot define any of the `shrink`, `forcemerge`, `freeze` or +`searchable_snapshot` (also available in the cold phase) actions. [NOTE] This action cannot be performed on a data stream's write index. Attempts to do From a1326c352bac916a90b32adfaa41d2111fee7a95 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 12 Nov 2020 09:29:57 +0000 Subject: [PATCH 13/16] Duplicate sets as we remove actions to generate valid policies --- .../xpack/core/ilm/LifecyclePolicyTests.java | 137 +++++++----------- 1 file changed, 53 insertions(+), 84 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 1dba01e2a6eb4..04461e556fa20 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,48 +100,8 @@ protected LifecyclePolicy createTestInstance() { public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Nullable String lifecycleName) { List phaseNames = TimeseriesLifecycleType.VALID_PHASES; Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { - switch (phase) { - case "hot": - return TimeseriesLifecycleType.VALID_HOT_ACTIONS; - case "warm": - return TimeseriesLifecycleType.VALID_WARM_ACTIONS; - case "cold": - return TimeseriesLifecycleType.VALID_COLD_ACTIONS; - case "delete": - return TimeseriesLifecycleType.VALID_DELETE_ACTIONS; - default: - throw new IllegalArgumentException("invalid phase [" + phase + "]"); - }}; - Function randomAction = (action) -> { - switch (action) { - case AllocateAction.NAME: - return AllocateActionTests.randomInstance(); - case DeleteAction.NAME: - return new DeleteAction(); - case WaitForSnapshotAction.NAME: - return WaitForSnapshotActionTests.randomInstance(); - case ForceMergeAction.NAME: - return ForceMergeActionTests.randomInstance(); - case ReadOnlyAction.NAME: - return new ReadOnlyAction(); - case RolloverAction.NAME: - return RolloverActionTests.randomInstance(); - case ShrinkAction.NAME: - return ShrinkActionTests.randomInstance(); - case FreezeAction.NAME: - return new FreezeAction(); - case SetPriorityAction.NAME: - return SetPriorityActionTests.randomInstance(); - case UnfollowAction.NAME: - return new UnfollowAction(); - case SearchableSnapshotAction.NAME: - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); - case MigrateAction.NAME: - return new MigrateAction(false); - default: - throw new IllegalArgumentException("invalid action [" + action + "]"); - }}; + Function> validActions = getPhaseToValidActions(); + Function randomAction = getNameToActionFunction(); for (String phase : phaseNames) { TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); Map actions = new HashMap<>(); @@ -162,48 +123,8 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l List phaseNames = randomSubsetOf( between(0, TimeseriesLifecycleType.VALID_PHASES.size() - 1), TimeseriesLifecycleType.VALID_PHASES); Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { - switch (phase) { - case "hot": - return TimeseriesLifecycleType.VALID_HOT_ACTIONS; - case "warm": - return TimeseriesLifecycleType.VALID_WARM_ACTIONS; - case "cold": - return TimeseriesLifecycleType.VALID_COLD_ACTIONS; - case "delete": - return TimeseriesLifecycleType.VALID_DELETE_ACTIONS; - default: - throw new IllegalArgumentException("invalid phase [" + phase + "]"); - }}; - Function randomAction = (action) -> { - switch (action) { - case AllocateAction.NAME: - return AllocateActionTests.randomInstance(); - case WaitForSnapshotAction.NAME: - return WaitForSnapshotActionTests.randomInstance(); - case DeleteAction.NAME: - return new DeleteAction(); - case ForceMergeAction.NAME: - return ForceMergeActionTests.randomInstance(); - case ReadOnlyAction.NAME: - return new ReadOnlyAction(); - case RolloverAction.NAME: - return RolloverActionTests.randomInstance(); - case ShrinkAction.NAME: - return ShrinkActionTests.randomInstance(); - case FreezeAction.NAME: - return new FreezeAction(); - case SetPriorityAction.NAME: - return SetPriorityActionTests.randomInstance(); - case UnfollowAction.NAME: - return new UnfollowAction(); - case SearchableSnapshotAction.NAME: - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); - case MigrateAction.NAME: - return new MigrateAction(false); - default: - throw new IllegalArgumentException("invalid action [" + action + "]"); - }}; + Function> validActions = getPhaseToValidActions(); + Function randomAction = getNameToActionFunction(); // as what actions end up in the hot phase influence what actions are allowed in the subsequent phases we'll move the hot phase // at the front of the phases to process (if it exists) if (phaseNames.contains(TimeseriesLifecycleType.HOT_PHASE)) { @@ -241,6 +162,54 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases); } + private static Function> getPhaseToValidActions() { + return (phase) -> { + switch (phase) { + case "hot": + return new HashSet<>(TimeseriesLifecycleType.VALID_HOT_ACTIONS); + case "warm": + return new HashSet<>(TimeseriesLifecycleType.VALID_WARM_ACTIONS); + case "cold": + return new HashSet<>(TimeseriesLifecycleType.VALID_COLD_ACTIONS); + case "delete": + return new HashSet<>(TimeseriesLifecycleType.VALID_DELETE_ACTIONS); + default: + throw new IllegalArgumentException("invalid phase [" + phase + "]"); + }}; + } + + private static Function getNameToActionFunction() { + return (action) -> { + switch (action) { + case AllocateAction.NAME: + return AllocateActionTests.randomInstance(); + case WaitForSnapshotAction.NAME: + return WaitForSnapshotActionTests.randomInstance(); + case DeleteAction.NAME: + return new DeleteAction(); + case ForceMergeAction.NAME: + return ForceMergeActionTests.randomInstance(); + case ReadOnlyAction.NAME: + return new ReadOnlyAction(); + case RolloverAction.NAME: + return RolloverActionTests.randomInstance(); + case ShrinkAction.NAME: + return ShrinkActionTests.randomInstance(); + case FreezeAction.NAME: + return new FreezeAction(); + case SetPriorityAction.NAME: + return SetPriorityActionTests.randomInstance(); + case UnfollowAction.NAME: + return new UnfollowAction(); + case SearchableSnapshotAction.NAME: + return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); + case MigrateAction.NAME: + return new MigrateAction(false); + default: + throw new IllegalArgumentException("invalid action [" + action + "]"); + }}; + } + public static LifecyclePolicy randomTestLifecyclePolicy(@Nullable String lifecycleName) { int numberPhases = randomInt(5); Map phases = new HashMap<>(numberPhases); From 54180d60d635fd2d4b98da9ca38ad9161e67c55c Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 12 Nov 2020 09:32:12 +0000 Subject: [PATCH 14/16] Static loggers --- .../xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java | 2 +- .../java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java | 2 +- .../java/org/elasticsearch/xpack/core/ilm/FreezeAction.java | 2 +- .../elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java | 2 +- .../java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java index f0c6e2e5af2f7..4d84bd1aa7537 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java @@ -26,7 +26,7 @@ * registers an observer and waits to try again when a snapshot is no longer running. */ public abstract class AsyncRetryDuringSnapshotActionStep extends AsyncActionStep { - private final Logger logger = LogManager.getLogger(AsyncRetryDuringSnapshotActionStep.class); + private static final Logger logger = LogManager.getLogger(AsyncRetryDuringSnapshotActionStep.class); public AsyncRetryDuringSnapshotActionStep(StepKey key, StepKey nextStepKey, Client client) { super(key, nextStepKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index 7c4658a45349a..ca610fb941548 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -33,7 +33,7 @@ * A {@link LifecycleAction} which force-merges the index. */ public class ForceMergeAction implements LifecycleAction { - private final Logger logger = LogManager.getLogger(ForceMergeAction.class); + private static final Logger logger = LogManager.getLogger(ForceMergeAction.class); public static final String NAME = "forcemerge"; public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 61901b67f2864..414185c9ba8e5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -25,7 +25,7 @@ * A {@link LifecycleAction} which freezes the index. */ public class FreezeAction implements LifecycleAction { - private final Logger logger = LogManager.getLogger(FreezeAction.class); + private static final Logger logger = LogManager.getLogger(FreezeAction.class); public static final String NAME = "freeze"; public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index b7ca1aaa8ed91..169a2dc09f0d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -32,7 +32,7 @@ * newly created searchable snapshot backed index. */ public class SearchableSnapshotAction implements LifecycleAction { - private final Logger logger = LogManager.getLogger(SearchableSnapshotAction.class); + private static final Logger logger = LogManager.getLogger(SearchableSnapshotAction.class); public static final String NAME = "searchable_snapshot"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 6b59f526b2530..94bdc1f747b3b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -29,7 +29,7 @@ * A {@link LifecycleAction} which shrinks the index. */ public class ShrinkAction implements LifecycleAction { - private final Logger logger = LogManager.getLogger(ShrinkAction.class); + private static final Logger logger = LogManager.getLogger(ShrinkAction.class); public static final String NAME = "shrink"; public static final String SHRUNKEN_INDEX_PREFIX = "shrink-"; From a5e43548f491c1ccd83f77846ac6ab9c655096e8 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 12 Nov 2020 09:33:57 +0000 Subject: [PATCH 15/16] Docs: Mention rollover is needed in order to use action in the hot phase --- docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 864198fae7f42..27a8c71bfa9c9 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -11,6 +11,10 @@ and mounts it as a searchable snapshot. If the managed index is part of a <>, the mounted index replaces the original index in the data stream. +To use the `searchable_snapshot` action in the `hot` phase, the `rollover` +action *must* be present. If no rollover action is configured, {ilm-init} +will reject the policy. + IMPORTANT: If the `searchable_snapshot` action is used in the `hot` phase the subsequent phases cannot define any of the `shrink`, `forcemerge`, `freeze` or `searchable_snapshot` (also available in the cold phase) actions. From 56af507e427a9dd3a9bb0bc72d85930c174fcd23 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 12 Nov 2020 09:36:48 +0000 Subject: [PATCH 16/16] Document setting name --- .../org/elasticsearch/xpack/core/ilm/LifecycleSettings.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java index a0395e8cfe619..60a059484ae7b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java @@ -27,6 +27,8 @@ public class LifecycleSettings { public static final String SLM_RETENTION_DURATION = "slm.retention_duration"; public static final String SLM_MINIMUM_INTERVAL = "slm.minimum_interval"; + // This is not a setting configuring ILM per se, but certain ILM actions need to validate the managed index is not + // already mounted as a searchable snapshot. Those ILM actions will check if the index has this setting name configured. public static final String SNAPSHOT_INDEX_NAME = "index.store.snapshot.index_name"; public static final Setting LIFECYCLE_POLL_INTERVAL_SETTING = Setting.timeSetting(LIFECYCLE_POLL_INTERVAL,