diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 667453e3a431f..27a8c71bfa9c9 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -4,13 +4,21 @@ beta::[] -Phases allowed: cold. +Phases allowed: hot, cold. Takes a snapshot of the managed index in the configured repository and mounts it as a searchable snapshot. If the managed index is part of a <>, the mounted index replaces the original index in the data stream. +To use the `searchable_snapshot` action in the `hot` phase, the `rollover` +action *must* be present. If no rollover action is configured, {ilm-init} +will reject the policy. + +IMPORTANT: If the `searchable_snapshot` action is used in the `hot` phase the +subsequent phases cannot define any of the `shrink`, `forcemerge`, `freeze` or +`searchable_snapshot` (also available in the cold phase) actions. + [NOTE] This action cannot be performed on a data stream's write index. Attempts to do so will fail. To convert the index to a searchable snapshot, first diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java index f0c6e2e5af2f7..4d84bd1aa7537 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java @@ -26,7 +26,7 @@ * registers an observer and waits to try again when a snapshot is no longer running. */ public abstract class AsyncRetryDuringSnapshotActionStep extends AsyncActionStep { - private final Logger logger = LogManager.getLogger(AsyncRetryDuringSnapshotActionStep.class); + private static final Logger logger = LogManager.getLogger(AsyncRetryDuringSnapshotActionStep.class); public AsyncRetryDuringSnapshotActionStep(StepKey key, StepKey nextStepKey, Client client) { super(key, nextStepKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index 28c72557b3f78..ca610fb941548 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -31,9 +33,12 @@ * A {@link LifecycleAction} which force-merges the index. */ public class ForceMergeAction implements LifecycleAction { + private static final Logger logger = LogManager.getLogger(ForceMergeAction.class); + public static final String NAME = "forcemerge"; public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); public static final ParseField CODEC = new ParseField("index_codec"); + public static final String CONDITIONAL_SKIP_FORCE_MERGE_STEP = BranchingStep.NAME + "-forcemerge-check-prerequisites"; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, false, a -> { @@ -120,6 +125,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) final boolean codecChange = codec != null && codec.equals(CodecService.BEST_COMPRESSION_CODEC); + StepKey preForceMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FORCE_MERGE_STEP); StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); @@ -131,6 +137,18 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME); StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME); + BranchingStep conditionalSkipShrinkStep = new BranchingStep(preForceMergeBranchingKey, checkNotWriteIndex, nextStepKey, + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.metadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings()); + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", ForceMergeAction.NAME, index.getName(), policyName); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, readOnlyKey); UpdateSettingsStep readOnlyStep = @@ -147,6 +165,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) SegmentCountStep segmentCountStep = new SegmentCountStep(countKey, nextStepKey, client, maxNumSegments); List mergeSteps = new ArrayList<>(); + mergeSteps.add(conditionalSkipShrinkStep); mergeSteps.add(checkNotWriteIndexStep); mergeSteps.add(readOnlyStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 083a015bb91ac..414185c9ba8e5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -5,7 +5,10 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -22,7 +25,10 @@ * A {@link LifecycleAction} which freezes the index. */ public class FreezeAction implements LifecycleAction { + private static final Logger logger = LogManager.getLogger(FreezeAction.class); + public static final String NAME = "freeze"; + public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites"; private static final ObjectParser PARSER = new ObjectParser<>(NAME, FreezeAction::new); @@ -59,13 +65,31 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey preFreezeMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FREEZE_STEP); StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey freezeStepKey = new StepKey(phase, NAME, FreezeStep.NAME); + BranchingStep conditionalSkipFreezeStep = new BranchingStep(preFreezeMergeBranchingKey, checkNotWriteIndex, nextStepKey, + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings()); + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", FreezeAction.NAME, index.getName(), policyName); + return true; + } + if (indexMetadata.getSettings().getAsBoolean("index.frozen", false)) { + logger.debug("skipping [{}] action for index [{}] in policy [{}] as the index is already frozen", FreezeAction.NAME, + index.getName(), policyName); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, freezeStepKey); FreezeStep freezeStep = new FreezeStep(freezeStepKey, nextStepKey, client); - return Arrays.asList(checkNoWriteIndexStep, freezeStep); + return Arrays.asList(conditionalSkipFreezeStep, checkNoWriteIndexStep, freezeStep); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java index 3d36e51ce8d09..60a059484ae7b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java @@ -27,6 +27,10 @@ public class LifecycleSettings { public static final String SLM_RETENTION_DURATION = "slm.retention_duration"; public static final String SLM_MINIMUM_INTERVAL = "slm.minimum_interval"; + // This is not a setting configuring ILM per se, but certain ILM actions need to validate the managed index is not + // already mounted as a searchable snapshot. Those ILM actions will check if the index has this setting name configured. + public static final String SNAPSHOT_INDEX_NAME = "index.store.snapshot.index_name"; + public static final Setting LIFECYCLE_POLL_INTERVAL_SETTING = Setting.timeSetting(LIFECYCLE_POLL_INTERVAL, TimeValue.timeValueMinutes(10), TimeValue.timeValueSeconds(1), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index f5a5e5ac00e2d..169a2dc09f0d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -5,10 +5,13 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,11 +32,14 @@ * newly created searchable snapshot backed index. */ public class SearchableSnapshotAction implements LifecycleAction { + private static final Logger logger = LogManager.getLogger(SearchableSnapshotAction.class); + public static final String NAME = "searchable_snapshot"; public static final ParseField SNAPSHOT_REPOSITORY = new ParseField("snapshot_repository"); public static final ParseField FORCE_MERGE_INDEX = new ParseField("force_merge_index"); public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; + public static final String CONDITIONAL_SKIP_ACTION_STEP = BranchingStep.NAME + "-check-prerequisites"; public static final String RESTORED_INDEX_PREFIX = "restored-"; @@ -74,6 +80,7 @@ boolean isForceMergeIndex() { @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey preActionBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_ACTION_STEP); StepKey checkNoWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME); StepKey forceMergeStepKey = new StepKey(phase, NAME, ForceMergeStep.NAME); @@ -90,6 +97,18 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME); StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); + BranchingStep conditionalSkipActionStep = new BranchingStep(preActionBranchingKey, checkNoWriteIndex, nextStepKey, + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is already mounted as searchable " + + "snapshot. Skipping this action", SearchableSnapshotAction.NAME, index.getName(), + LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex, waitForNoFollowerStepKey); final WaitForNoFollowersStep waitForNoFollowersStep; @@ -130,6 +149,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { null, client, RESTORED_INDEX_PREFIX); List steps = new ArrayList<>(); + steps.add(conditionalSkipActionStep); steps.add(checkNoWriteIndexStep); steps.add(waitForNoFollowersStep); if (forceMergeIndex) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index c278de30e3b4f..94bdc1f747b3b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -29,14 +29,14 @@ * A {@link LifecycleAction} which shrinks the index. */ public class ShrinkAction implements LifecycleAction { + private static final Logger logger = LogManager.getLogger(ShrinkAction.class); + public static final String NAME = "shrink"; public static final String SHRUNKEN_INDEX_PREFIX = "shrink-"; public static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards"); public static final String CONDITIONAL_SKIP_SHRINK_STEP = BranchingStep.NAME + "-check-prerequisites"; public static final String CONDITIONAL_DATASTREAM_CHECK_KEY = BranchingStep.NAME + "-on-datastream-check"; - private static final Logger logger = LogManager.getLogger(ShrinkAction.class); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0])); @@ -108,7 +108,19 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME); BranchingStep conditionalSkipShrinkStep = new BranchingStep(preShrinkBranchingKey, checkNotWriteIndex, nextStepKey, - (index, clusterState) -> clusterState.getMetadata().index(index).getNumberOfShards() == numberOfShards); + (index, clusterState) -> { + IndexMetadata indexMetadata = clusterState.getMetadata().index(index); + if (indexMetadata.getNumberOfShards() == numberOfShards) { + return true; + } + if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { + logger.warn("[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " + + "Skipping this action", ShrinkAction.NAME, indexMetadata.getIndex().getName(), + LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetadata.getSettings())); + return true; + } + return false; + }); CheckNotDataStreamWriteIndexStep checkNotWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, waitForNoFollowerStepKey); WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index d3bf38892444e..343dd1a1c9336 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +41,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; static final List VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE); static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, - ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); + ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, @@ -57,7 +58,10 @@ public class TimeseriesLifecycleType implements LifecycleType { DELETE_PHASE, VALID_DELETE_ACTIONS); static final Set HOT_ACTIONS_THAT_REQUIRE_ROLLOVER = Sets.newHashSet(ReadOnlyAction.NAME, ShrinkAction.NAME, - ForceMergeAction.NAME); + ForceMergeAction.NAME, SearchableSnapshotAction.NAME); + // a set of actions that cannot be defined (executed) after the managed index has been mounted as searchable snapshot + static final Set ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT = Sets.newHashSet(ShrinkAction.NAME, ForceMergeAction.NAME, + FreezeAction.NAME, SearchableSnapshotAction.NAME); private TimeseriesLifecycleType() { } @@ -255,6 +259,29 @@ public void validate(Collection phases) { MigrateAction.NAME + " action and an " + AllocateAction.NAME + " action with allocation rules. specify only a single " + "data migration in each phase"); } + + validateActionsFollowingSearchableSnapshot(phases); + } + + static void validateActionsFollowingSearchableSnapshot(Collection phases) { + boolean hotPhaseContainsSearchableSnapshot = phases.stream() + .filter(phase -> HOT_PHASE.equals(phase.getName())) + .anyMatch(phase -> phase.getActions().containsKey(SearchableSnapshotAction.NAME)); + if (hotPhaseContainsSearchableSnapshot) { + String phasesDefiningIllegalActions = phases.stream() + // we're looking for prohibited actions in phases other than hot + .filter(phase -> HOT_PHASE.equals(phase.getName()) == false) + // filter the phases that define illegal actions + .filter(phase -> + Collections.disjoint(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, phase.getActions().keySet()) == false) + .map(Phase::getName) + .collect(Collectors.joining(",")); + if (Strings.hasText(phasesDefiningIllegalActions)) { + throw new IllegalArgumentException("phases [" + phasesDefiningIllegalActions + "] define one or more of " + + ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT + " actions which are not allowed after a " + + "managed index is mounted as a searchable snapshot"); + } + } } private static boolean definesAllocationRules(AllocateAction action) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java index cb5d6f9387149..730639c210a51 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java @@ -24,6 +24,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class ForceMergeActionTests extends AbstractActionTestCase { @@ -65,21 +66,24 @@ private void assertNonBestCompression(ForceMergeAction instance) { StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); List steps = instance.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(4, steps.size()); - CheckNotDataStreamWriteIndexStep firstStep = (CheckNotDataStreamWriteIndexStep) steps.get(0); - UpdateSettingsStep secondStep = (UpdateSettingsStep) steps.get(1); - ForceMergeStep thirdStep = (ForceMergeStep) steps.get(2); - SegmentCountStep fourthStep = (SegmentCountStep) steps.get(3); - - assertThat(firstStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME))); - assertThat(firstStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); - assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); - assertThat(secondStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); - assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(secondStep.getSettings())); - assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); - assertThat(thirdStep.getNextStepKey(), equalTo(fourthStep.getKey())); - assertThat(fourthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME))); - assertThat(fourthStep.getNextStepKey(), equalTo(nextStepKey)); + assertEquals(5, steps.size()); + BranchingStep firstStep = (BranchingStep) steps.get(0); + CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1); + UpdateSettingsStep thirdStep = (UpdateSettingsStep) steps.get(2); + ForceMergeStep fourthStep = (ForceMergeStep) steps.get(3); + SegmentCountStep fifthStep = (SegmentCountStep) steps.get(4); + + assertThat(firstStep.getKey(), + equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP))); + assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME))); + assertThat(secondStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); + assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME))); + assertThat(thirdStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); + assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(thirdStep.getSettings())); + assertThat(fourthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME))); + assertThat(fourthStep.getNextStepKey(), equalTo(fifthStep.getKey())); + assertThat(fifthStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME))); + assertThat(fifthStep.getNextStepKey(), equalTo(nextStepKey)); } private void assertBestCompression(ForceMergeAction instance) { @@ -87,8 +91,11 @@ private void assertBestCompression(ForceMergeAction instance) { StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); List steps = instance.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(8, steps.size()); + assertEquals(9, steps.size()); List> stepKeys = steps.stream() + // skip the first branching step as `performAction` needs to be executed to evaluate the condition before the next step is + // available + .skip(1) .map(s -> new Tuple<>(s.getKey(), s.getNextStepKey())) .collect(Collectors.toList()); StepKey checkNotWriteIndex = new StepKey(phase, ForceMergeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); @@ -99,6 +106,8 @@ private void assertBestCompression(ForceMergeAction instance) { StepKey waitForGreen = new StepKey(phase, ForceMergeAction.NAME, WaitForIndexColorStep.NAME); StepKey forceMerge = new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME); StepKey segmentCount = new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME); + assertThat(steps.get(0).getKey(), is(new StepKey(phase, ForceMergeAction.NAME, + ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP))); assertThat(stepKeys, contains( new Tuple<>(checkNotWriteIndex, readOnly), new Tuple<>(readOnly, closeIndex), @@ -109,11 +118,11 @@ private void assertBestCompression(ForceMergeAction instance) { new Tuple<>(forceMerge, segmentCount), new Tuple<>(segmentCount, nextStepKey))); - UpdateSettingsStep secondStep = (UpdateSettingsStep) steps.get(1); - UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3); + UpdateSettingsStep thirdStep = (UpdateSettingsStep) steps.get(2); + UpdateSettingsStep fifthStep = (UpdateSettingsStep) steps.get(4); - assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(secondStep.getSettings())); - assertThat(fourthStep.getSettings().get(EngineConfig.INDEX_CODEC_SETTING.getKey()), equalTo(CodecService.BEST_COMPRESSION_CODEC)); + assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(thirdStep.getSettings())); + assertThat(fifthStep.getSettings().get(EngineConfig.INDEX_CODEC_SETTING.getKey()), equalTo(CodecService.BEST_COMPRESSION_CODEC)); } public void testMissingMaxNumSegments() throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java index 71f2d963b1c6b..7966bc8b83b1a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java @@ -38,17 +38,20 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(2, steps.size()); - StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); - StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME); + assertEquals(3, steps.size()); + StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP); + StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); + StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME); - CheckNotDataStreamWriteIndexStep firstStep = (CheckNotDataStreamWriteIndexStep) steps.get(0); - FreezeStep secondStep = (FreezeStep) steps.get(1); + BranchingStep firstStep = (BranchingStep) steps.get(0); + CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1); + FreezeStep thirdStep = (FreezeStep) steps.get(2); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); - assertThat(firstStep.getNextStepKey(), equalTo(expectedSecondStepKey)); assertEquals(expectedSecondStepKey, secondStep.getKey()); - assertEquals(nextStepKey, secondStep.getNextStepKey()); + assertEquals(expectedThirdStepKey, secondStep.getNextStepKey()); + assertEquals(expectedThirdStepKey, thirdStep.getKey()); + assertEquals(nextStepKey, thirdStep.getNextStepKey()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 916184f2eefc3..04461e556fa20 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,52 +100,17 @@ protected LifecyclePolicy createTestInstance() { public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Nullable String lifecycleName) { List phaseNames = TimeseriesLifecycleType.VALID_PHASES; Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { - switch (phase) { - case "hot": - return TimeseriesLifecycleType.VALID_HOT_ACTIONS; - case "warm": - return TimeseriesLifecycleType.VALID_WARM_ACTIONS; - case "cold": - return TimeseriesLifecycleType.VALID_COLD_ACTIONS; - case "delete": - return TimeseriesLifecycleType.VALID_DELETE_ACTIONS; - default: - throw new IllegalArgumentException("invalid phase [" + phase + "]"); - }}; - Function randomAction = (action) -> { - switch (action) { - case AllocateAction.NAME: - return AllocateActionTests.randomInstance(); - case DeleteAction.NAME: - return new DeleteAction(); - case WaitForSnapshotAction.NAME: - return WaitForSnapshotActionTests.randomInstance(); - case ForceMergeAction.NAME: - return ForceMergeActionTests.randomInstance(); - case ReadOnlyAction.NAME: - return new ReadOnlyAction(); - case RolloverAction.NAME: - return RolloverActionTests.randomInstance(); - case ShrinkAction.NAME: - return ShrinkActionTests.randomInstance(); - case FreezeAction.NAME: - return new FreezeAction(); - case SetPriorityAction.NAME: - return SetPriorityActionTests.randomInstance(); - case UnfollowAction.NAME: - return new UnfollowAction(); - case SearchableSnapshotAction.NAME: - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); - case MigrateAction.NAME: - return new MigrateAction(false); - default: - throw new IllegalArgumentException("invalid action [" + action + "]"); - }}; + Function> validActions = getPhaseToValidActions(); + Function randomAction = getNameToActionFunction(); for (String phase : phaseNames) { TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); Map actions = new HashMap<>(); Set actionNames = validActions.apply(phase); + if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) == false) { + // let's make sure the other phases don't configure actions that conflict with the `searchable_snapshot` action + // configured in the hot phase + actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); + } for (String action : actionNames) { actions.put(action, randomAction.apply(action)); } @@ -157,57 +123,35 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l List phaseNames = randomSubsetOf( between(0, TimeseriesLifecycleType.VALID_PHASES.size() - 1), TimeseriesLifecycleType.VALID_PHASES); Map phases = new HashMap<>(phaseNames.size()); - Function> validActions = (phase) -> { - switch (phase) { - case "hot": - return TimeseriesLifecycleType.VALID_HOT_ACTIONS; - case "warm": - return TimeseriesLifecycleType.VALID_WARM_ACTIONS; - case "cold": - return TimeseriesLifecycleType.VALID_COLD_ACTIONS; - case "delete": - return TimeseriesLifecycleType.VALID_DELETE_ACTIONS; - default: - throw new IllegalArgumentException("invalid phase [" + phase + "]"); - }}; - Function randomAction = (action) -> { - switch (action) { - case AllocateAction.NAME: - return AllocateActionTests.randomInstance(); - case WaitForSnapshotAction.NAME: - return WaitForSnapshotActionTests.randomInstance(); - case DeleteAction.NAME: - return new DeleteAction(); - case ForceMergeAction.NAME: - return ForceMergeActionTests.randomInstance(); - case ReadOnlyAction.NAME: - return new ReadOnlyAction(); - case RolloverAction.NAME: - return RolloverActionTests.randomInstance(); - case ShrinkAction.NAME: - return ShrinkActionTests.randomInstance(); - case FreezeAction.NAME: - return new FreezeAction(); - case SetPriorityAction.NAME: - return SetPriorityActionTests.randomInstance(); - case UnfollowAction.NAME: - return new UnfollowAction(); - case SearchableSnapshotAction.NAME: - return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); - case MigrateAction.NAME: - return new MigrateAction(false); - default: - throw new IllegalArgumentException("invalid action [" + action + "]"); - }}; + Function> validActions = getPhaseToValidActions(); + Function randomAction = getNameToActionFunction(); + // as what actions end up in the hot phase influence what actions are allowed in the subsequent phases we'll move the hot phase + // at the front of the phases to process (if it exists) + if (phaseNames.contains(TimeseriesLifecycleType.HOT_PHASE)) { + phaseNames.remove(TimeseriesLifecycleType.HOT_PHASE); + phaseNames.add(0, TimeseriesLifecycleType.HOT_PHASE); + } + boolean hotPhaseContainsSearchableSnap = false; for (String phase : phaseNames) { TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); Map actions = new HashMap<>(); List actionNames = randomSubsetOf(validActions.apply(phase)); - // If the hot phase has any actions that require a rollover, then ensure there is one so that the policy will validate - if (phase.equals(TimeseriesLifecycleType.HOT_PHASE) - && actionNames.stream().anyMatch(TimeseriesLifecycleType.HOT_ACTIONS_THAT_REQUIRE_ROLLOVER::contains)) { - actionNames.add(RolloverAction.NAME); + if (phase.equals(TimeseriesLifecycleType.HOT_PHASE)) { + // If the hot phase has any actions that require a rollover, then ensure there is one so that the policy will validate + if (actionNames.stream().anyMatch(TimeseriesLifecycleType.HOT_ACTIONS_THAT_REQUIRE_ROLLOVER::contains)) { + actionNames.add(RolloverAction.NAME); + } + + if (actionNames.contains(SearchableSnapshotAction.NAME)) { + hotPhaseContainsSearchableSnap = true; + } + } else { + if (hotPhaseContainsSearchableSnap) { + // let's make sure the other phases don't configure actions that conflict with a possible `searchable_snapshot` action + // configured in the hot phase + actionNames.removeAll(TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT); + } } for (String action : actionNames) { @@ -218,6 +162,54 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases); } + private static Function> getPhaseToValidActions() { + return (phase) -> { + switch (phase) { + case "hot": + return new HashSet<>(TimeseriesLifecycleType.VALID_HOT_ACTIONS); + case "warm": + return new HashSet<>(TimeseriesLifecycleType.VALID_WARM_ACTIONS); + case "cold": + return new HashSet<>(TimeseriesLifecycleType.VALID_COLD_ACTIONS); + case "delete": + return new HashSet<>(TimeseriesLifecycleType.VALID_DELETE_ACTIONS); + default: + throw new IllegalArgumentException("invalid phase [" + phase + "]"); + }}; + } + + private static Function getNameToActionFunction() { + return (action) -> { + switch (action) { + case AllocateAction.NAME: + return AllocateActionTests.randomInstance(); + case WaitForSnapshotAction.NAME: + return WaitForSnapshotActionTests.randomInstance(); + case DeleteAction.NAME: + return new DeleteAction(); + case ForceMergeAction.NAME: + return ForceMergeActionTests.randomInstance(); + case ReadOnlyAction.NAME: + return new ReadOnlyAction(); + case RolloverAction.NAME: + return RolloverActionTests.randomInstance(); + case ShrinkAction.NAME: + return ShrinkActionTests.randomInstance(); + case FreezeAction.NAME: + return new FreezeAction(); + case SetPriorityAction.NAME: + return SetPriorityActionTests.randomInstance(); + case UnfollowAction.NAME: + return new UnfollowAction(); + case SearchableSnapshotAction.NAME: + return new SearchableSnapshotAction(randomAlphaOfLengthBetween(1, 10)); + case MigrateAction.NAME: + return new MigrateAction(false); + default: + throw new IllegalArgumentException("invalid action [" + action + "]"); + }}; + } + public static LifecyclePolicy randomTestLifecyclePolicy(@Nullable String lifecycleName) { int numberPhases = randomInt(5); Map phases = new HashMap<>(numberPhases); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index 48757fc483064..88cf918d05b8e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -24,7 +24,7 @@ public void testToSteps() { StepKey nextStepKey = new StepKey(phase, randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(1, 5)); List steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), is(action.isForceMergeIndex() ? 15 : 13)); + assertThat(steps.size(), is(action.isForceMergeIndex() ? 16 : 14)); List expectedSteps = action.isForceMergeIndex() ? expectedStepKeysWithForceMerge(phase) : expectedStepKeysNoForceMerge(phase); @@ -44,17 +44,18 @@ public void testToSteps() { assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); if (action.isForceMergeIndex()) { - assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(6); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(5))); + assertThat(steps.get(14).getKey(), is(expectedSteps.get(14))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(7); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(6))); } else { - AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4); - assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(3))); + AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(5); + assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedSteps.get(4))); } } private List expectedStepKeysWithForceMerge(String phase) { return List.of( + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, ForceMergeStep.NAME), @@ -74,6 +75,7 @@ private List expectedStepKeysWithForceMerge(String phase) { private List expectedStepKeysNoForceMerge(String phase) { return List.of( + new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java index ee51129c05964..df0017eadd492 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java @@ -20,6 +20,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.HOT_PHASE; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_COLD_ACTIONS; @@ -32,6 +33,7 @@ import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_PHASES; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_WARM_ACTIONS; import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.WARM_PHASE; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -191,13 +193,29 @@ public void testValidateConflictingDataMigrationConfigurations() { } } + public void testActionsThatCannotFollowSearchableSnapshot() { + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT.size(), is(4)); + assertThat(ACTIONS_CANNOT_FOLLOW_SEARCHABLE_SNAPSHOT, containsInAnyOrder(ShrinkAction.NAME, FreezeAction.NAME, + ForceMergeAction.NAME, SearchableSnapshotAction.NAME)); + } + + public void testValidateActionsFollowingSearchableSnapshot() { + Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction("repo"))); + Phase warmPhase = new Phase("warm", TimeValue.ZERO, Map.of(ShrinkAction.NAME, new ShrinkAction(1))); + Phase coldPhase = new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TimeseriesLifecycleType.validateActionsFollowingSearchableSnapshot(List.of(hotPhase, warmPhase, coldPhase))); + assertThat(e.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink] actions" + + " which are not allowed after a managed index is mounted as a searchable snapshot")); + } + public void testGetOrderedPhases() { Map phaseMap = new HashMap<>(); for (String phaseName : randomSubsetOf(randomIntBetween(0, VALID_PHASES.size()), VALID_PHASES)) { phaseMap.put(phaseName, new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap())); } - assertTrue(isSorted(TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap), Phase::getName, VALID_PHASES)); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java index cb55434c38944..1f675dc169bdd 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/TimeSeriesRestDriver.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -191,6 +192,34 @@ public static void createFullPolicy(RestClient client, String policyName, TimeVa client.performRequest(request); } + public static void createPolicy(RestClient client, String policyName, @Nullable Phase hotPhase, @Nullable Phase warmPhase, + @Nullable Phase coldPhase, @Nullable Phase deletePhase) throws IOException { + if (hotPhase == null && warmPhase == null && coldPhase == null && deletePhase == null) { + throw new IllegalArgumentException("specify at least one phase"); + } + Map phases = new HashMap<>(); + if (hotPhase != null) { + phases.put("hot", hotPhase); + } + if (warmPhase != null) { + phases.put("warm", warmPhase); + } + if (coldPhase != null) { + phases.put("cold", coldPhase); + } + if (deletePhase != null) { + phases.put("delete", deletePhase); + } + LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policyName, phases); + XContentBuilder builder = jsonBuilder(); + lifecyclePolicy.toXContent(builder, null); + final StringEntity entity = new StringEntity( + "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON); + Request request = new Request("PUT", "_ilm/policy/" + policyName); + request.setEntity(entity); + client.performRequest(request); + } + public static void createSnapshotRepo(RestClient client, String repoName, boolean compress) throws IOException { Request request = new Request("PUT", "/_snapshot/" + repoName); request.setJsonEntity(Strings diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index e1be0bf7ccc3d..82c7e543a4d21 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -19,12 +20,18 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.ForceMergeAction; +import org.elasticsearch.xpack.core.ilm.FreezeAction; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; +import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction; +import org.elasticsearch.xpack.core.ilm.SetPriorityAction; +import org.elasticsearch.xpack.core.ilm.ShrinkAction; +import org.elasticsearch.xpack.core.ilm.Step; import org.junit.Before; import java.io.IOException; @@ -37,9 +44,11 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createComposableTemplate; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createPolicy; import static org.elasticsearch.xpack.TimeSeriesRestDriver.createSnapshotRepo; import static org.elasticsearch.xpack.TimeSeriesRestDriver.explainIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.getNumberOfSegments; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.indexDocument; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; import static org.hamcrest.Matchers.greaterThan; @@ -188,4 +197,148 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { }, 30, TimeUnit.SECONDS)); } + public void testCreateInvalidPolicy() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), SearchableSnapshotAction.NAME, + new SearchableSnapshotAction(randomAlphaOfLengthBetween(4, 10)))), + new Phase("warm", TimeValue.ZERO, Map.of(ForceMergeAction.NAME, new ForceMergeAction(1, null))), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + null + ) + ); + + assertThat(exception.getMessage(), is("phases [warm,cold] define one or more of [searchable_snapshot, forcemerge, freeze, shrink]" + + " actions which are not allowed after a managed index is mounted as a searchable snapshot")); + } + + public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws Exception { + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), SearchableSnapshotAction.NAME, + new SearchableSnapshotAction(snapshotRepo))), + new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), + null, null + ); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .build(), null, null) + ); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStream, true); + } + + String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + DataStream.getDefaultBackingIndexName(dataStream, 1L); + assertTrue(waitUntil(() -> { + try { + return indexExists(restoredIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), restoredIndexName); + assertThat(stepKeyForIndex.getPhase(), is("hot")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(SetPriorityAction.NAME, new SetPriorityAction(10))), + new Phase("warm", TimeValue.ZERO, + Map.of(ShrinkAction.NAME, new ShrinkAction(1), ForceMergeAction.NAME, new ForceMergeAction(1, null)) + ), + new Phase("cold", TimeValue.ZERO, Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + null + ); + + // even though the index is now mounted as a searchable snapshot, the actions that can't operate on it should + // skip and ILM should not be blocked (not should the managed index move into the ERROR step) + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), restoredIndexName); + assertThat(stepKeyForIndex.getPhase(), is("cold")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + } + + public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Exception{ + // let's create a data stream, rollover it and convert the first generation backing index into a searchable snapshot + String snapshotRepo = randomAlphaOfLengthBetween(4, 10); + createSnapshotRepo(client(), snapshotRepo, randomBoolean()); + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of(RolloverAction.NAME, new RolloverAction(null, null, 1L), + SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo))), + new Phase("warm", TimeValue.timeValueDays(30), Map.of(SetPriorityAction.NAME, new SetPriorityAction(999))), + null, null + ); + + createComposableTemplate(client(), "template-name", dataStream, + new Template(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .build(), null, null) + ); + + // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexDocument(client(), dataStream, true); + } + + String searchableSnapMountedIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + + DataStream.getDefaultBackingIndexName(dataStream, 1L); + assertTrue(waitUntil(() -> { + try { + return indexExists(searchableSnapMountedIndexName); + } catch (IOException e) { + return false; + } + }, 30, TimeUnit.SECONDS)); + + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + assertThat(stepKeyForIndex.getPhase(), is("hot")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + + // snapshot the data stream + String dsSnapshotName = "snapshot_ds_" + dataStream; + Request takeSnapshotRequest = new Request("PUT", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName); + takeSnapshotRequest.addParameter("wait_for_completion", "true"); + takeSnapshotRequest.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}"); + assertOK(client().performRequest(takeSnapshotRequest)); + + // now that we have a backup of the data stream, let's delete the local one and update the ILM policy to include some illegal + // actions for when we restore the data stream (given that the first generation backing index will be backed by a searchable + // snapshot) + assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStream))); + + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, Map.of()), + new Phase("warm", TimeValue.ZERO, + Map.of(ShrinkAction.NAME, new ShrinkAction(1), ForceMergeAction.NAME, new ForceMergeAction(1, null)) + ), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, new FreezeAction())), + null + ); + + // restore the datastream + Request restoreSnapshot = new Request("POST", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName + "/_restore"); + restoreSnapshot.addParameter("wait_for_completion", "true"); + restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": true}"); + assertOK(client().performRequest(restoreSnapshot)); + + assertThat(indexExists(searchableSnapMountedIndexName), is(true)); + + // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase + assertBusy(() -> { + Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + assertThat(stepKeyForIndex.getPhase(), is("cold")); + assertThat(stepKeyForIndex.getName(), is(PhaseCompleteStep.NAME)); + }, 30, TimeUnit.SECONDS); + } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index bd1447625089c..8f9d28a8940b0 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -161,10 +161,12 @@ public void testReadStepKeys() { assertThat(TransportPutLifecycleAction.readStepKeys(REGISTRY, client, phaseDef, "phase"), contains( + new Step.StepKey("phase", "freeze", FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP), new Step.StepKey("phase", "freeze", CheckNotDataStreamWriteIndexStep.NAME), new Step.StepKey("phase", "freeze", FreezeAction.NAME), new Step.StepKey("phase", "allocate", AllocateAction.NAME), new Step.StepKey("phase", "allocate", AllocationRoutedStep.NAME), + new Step.StepKey("phase", "forcemerge", ForceMergeAction.CONDITIONAL_SKIP_FORCE_MERGE_STEP), new Step.StepKey("phase", "forcemerge", CheckNotDataStreamWriteIndexStep.NAME), new Step.StepKey("phase", "forcemerge", ReadOnlyAction.NAME), new Step.StepKey("phase", "forcemerge", ForceMergeAction.NAME),