Skip to content

Commit 49bd871

Browse files
gwbrowngwbrown
authored
Inject Unfollow before Rollover and Shrink (#37625)
We inject an Unfollow action before Shrink because the Shrink action cannot be safely used on a following index, as it may not be fully caught up with the leader index before the "original" following index is deleted and replaced with a non-following Shrunken index. The Unfollow action will verify that 1) the index is marked as "complete", and 2) all operations up to this point have been replicated from the leader to the follower before explicitly disconnecting the follower from the leader. Injecting an Unfollow action before the Rollover action is done mainly as a convenience: This allow users to use the same lifecycle policy on both the leader and follower cluster without having to explictly modify the policy to unfollow the index, while doing what we expect users to want in most cases.
1 parent 19529da commit 49bd871

File tree

10 files changed

+297
-79
lines changed

10 files changed

+297
-79
lines changed

‎client/rest-high-level/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ integTestCluster {
107107
// Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
108108
setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
109109
setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
110+
setting 'indices.lifecycle.poll_interval', '1000ms'
110111
keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
111112
setupCommand 'setupDummyUser',
112113
'bin/elasticsearch-users',

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -184,31 +184,37 @@ public void testExplainLifecycle() throws Exception {
184184

185185
createIndex("squash", Settings.EMPTY);
186186

187-
ExplainLifecycleRequest req = new ExplainLifecycleRequest("foo-01", "baz-01", "squash");
188-
ExplainLifecycleResponse response = execute(req, highLevelClient().indexLifecycle()::explainLifecycle,
187+
// The injected Unfollow step will run pretty rapidly here, so we need
188+
// to wait for it to settle into the "stable" step of waiting to be
189+
// ready to roll over
190+
assertBusy(() -> {
191+
ExplainLifecycleRequest req = new ExplainLifecycleRequest("foo-01", "baz-01", "squash");
192+
ExplainLifecycleResponse response = execute(req, highLevelClient().indexLifecycle()::explainLifecycle,
189193
highLevelClient().indexLifecycle()::explainLifecycleAsync);
190-
Map<String, IndexLifecycleExplainResponse> indexResponses = response.getIndexResponses();
191-
assertEquals(3, indexResponses.size());
192-
IndexLifecycleExplainResponse fooResponse = indexResponses.get("foo-01");
193-
assertNotNull(fooResponse);
194-
assertTrue(fooResponse.managedByILM());
195-
assertEquals("foo-01", fooResponse.getIndex());
196-
assertEquals("hot", fooResponse.getPhase());
197-
assertEquals("rollover", fooResponse.getAction());
198-
assertEquals("check-rollover-ready", fooResponse.getStep());
199-
assertEquals(new PhaseExecutionInfo(policy.getName(), new Phase("", hotPhase.getMinimumAge(), hotPhase.getActions()),
194+
Map<String, IndexLifecycleExplainResponse> indexResponses = response.getIndexResponses();
195+
assertEquals(3, indexResponses.size());
196+
IndexLifecycleExplainResponse fooResponse = indexResponses.get("foo-01");
197+
assertNotNull(fooResponse);
198+
assertTrue(fooResponse.managedByILM());
199+
assertEquals("foo-01", fooResponse.getIndex());
200+
assertEquals("hot", fooResponse.getPhase());
201+
assertEquals("rollover", fooResponse.getAction());
202+
assertEquals("check-rollover-ready", fooResponse.getStep());
203+
assertEquals(new PhaseExecutionInfo(policy.getName(), new Phase("", hotPhase.getMinimumAge(), hotPhase.getActions()),
200204
1L, expectedPolicyModifiedDate), fooResponse.getPhaseExecutionInfo());
201-
IndexLifecycleExplainResponse bazResponse = indexResponses.get("baz-01");
202-
assertNotNull(bazResponse);
203-
assertTrue(bazResponse.managedByILM());
204-
assertEquals("baz-01", bazResponse.getIndex());
205-
assertEquals("hot", bazResponse.getPhase());
206-
assertEquals("rollover", bazResponse.getAction());
207-
assertEquals("check-rollover-ready", bazResponse.getStep());
208-
IndexLifecycleExplainResponse squashResponse = indexResponses.get("squash");
209-
assertNotNull(squashResponse);
210-
assertFalse(squashResponse.managedByILM());
211-
assertEquals("squash", squashResponse.getIndex());
205+
IndexLifecycleExplainResponse bazResponse = indexResponses.get("baz-01");
206+
assertNotNull(bazResponse);
207+
assertTrue(bazResponse.managedByILM());
208+
assertEquals("baz-01", bazResponse.getIndex());
209+
assertEquals("hot", bazResponse.getPhase());
210+
assertEquals("rollover", bazResponse.getAction());
211+
assertEquals("check-rollover-ready", bazResponse.getStep());
212+
IndexLifecycleExplainResponse squashResponse = indexResponses.get("squash");
213+
assertNotNull(squashResponse);
214+
assertFalse(squashResponse.managedByILM());
215+
assertEquals("squash", squashResponse.getIndex());
216+
217+
});
212218
}
213219

214220
public void testDeleteLifecycle() throws IOException {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java‎

Lines changed: 56 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.http.util.EntityUtils;
2323
import org.elasticsearch.action.ActionListener;
2424
import org.elasticsearch.action.LatchedActionListener;
25+
import org.elasticsearch.action.admin.indices.alias.Alias;
2526
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
2627
import org.elasticsearch.client.RequestOptions;
2728
import org.elasticsearch.client.Response;
@@ -38,17 +39,17 @@
3839
import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
3940
import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
4041
import org.elasticsearch.client.indexlifecycle.LifecyclePolicy;
41-
import org.elasticsearch.client.indexlifecycle.OperationMode;
4242
import org.elasticsearch.client.indexlifecycle.LifecyclePolicyMetadata;
43+
import org.elasticsearch.client.indexlifecycle.OperationMode;
4344
import org.elasticsearch.client.indexlifecycle.Phase;
4445
import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
4546
import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
4647
import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyResponse;
4748
import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
4849
import org.elasticsearch.client.indexlifecycle.RolloverAction;
50+
import org.elasticsearch.client.indexlifecycle.ShrinkAction;
4951
import org.elasticsearch.client.indexlifecycle.StartILMRequest;
5052
import org.elasticsearch.client.indexlifecycle.StopILMRequest;
51-
import org.elasticsearch.client.indexlifecycle.ShrinkAction;
5253
import org.elasticsearch.client.indices.CreateIndexRequest;
5354
import org.elasticsearch.cluster.metadata.IndexMetaData;
5455
import org.elasticsearch.common.Strings;
@@ -337,11 +338,13 @@ public void testExplainLifecycle() throws Exception {
337338
new PutLifecyclePolicyRequest(policy);
338339
client.indexLifecycle().putLifecyclePolicy(putRequest, RequestOptions.DEFAULT);
339340

340-
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index")
341+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index-1")
341342
.settings(Settings.builder()
342343
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
343344
.put("index.lifecycle.name", "my_policy")
345+
.put("index.lifecycle.rollover_alias", "my_alias")
344346
.build());
347+
createIndexRequest.alias(new Alias("my_alias").writeIndex(true));
345348
client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
346349
CreateIndexRequest createOtherIndexRequest = new CreateIndexRequest("other_index")
347350
.settings(Settings.builder()
@@ -352,58 +355,62 @@ public void testExplainLifecycle() throws Exception {
352355

353356
// wait for the policy to become active
354357
assertBusy(() -> assertNotNull(client.indexLifecycle()
355-
.explainLifecycle(new ExplainLifecycleRequest("my_index"), RequestOptions.DEFAULT)
356-
.getIndexResponses().get("my_index").getAction()));
358+
.explainLifecycle(new ExplainLifecycleRequest("my_index-1"), RequestOptions.DEFAULT)
359+
.getIndexResponses().get("my_index-1").getAction()));
357360
}
358361

359362
// tag::ilm-explain-lifecycle-request
360363
ExplainLifecycleRequest request =
361-
new ExplainLifecycleRequest("my_index", "other_index"); // <1>
364+
new ExplainLifecycleRequest("my_index-1", "other_index"); // <1>
362365
// end::ilm-explain-lifecycle-request
363366

364-
// tag::ilm-explain-lifecycle-execute
365-
ExplainLifecycleResponse response = client.indexLifecycle()
366-
.explainLifecycle(request, RequestOptions.DEFAULT);
367-
// end::ilm-explain-lifecycle-execute
368-
assertNotNull(response);
369-
370-
// tag::ilm-explain-lifecycle-response
371-
Map<String, IndexLifecycleExplainResponse> indices =
372-
response.getIndexResponses();
373-
IndexLifecycleExplainResponse myIndex = indices.get("my_index");
374-
String policyName = myIndex.getPolicyName(); // <1>
375-
boolean isManaged = myIndex.managedByILM(); // <2>
376-
377-
String phase = myIndex.getPhase(); // <3>
378-
long phaseTime = myIndex.getPhaseTime(); // <4>
379-
String action = myIndex.getAction(); // <5>
380-
long actionTime = myIndex.getActionTime();
381-
String step = myIndex.getStep(); // <6>
382-
long stepTime = myIndex.getStepTime();
383-
384-
String failedStep = myIndex.getFailedStep(); // <7>
385-
// end::ilm-explain-lifecycle-response
386-
assertEquals("my_policy", policyName);
387-
assertTrue(isManaged);
388-
389-
assertEquals("hot", phase);
390-
assertNotEquals(0, phaseTime);
391-
assertEquals("rollover", action);
392-
assertNotEquals(0, actionTime);
393-
assertEquals("check-rollover-ready", step);
394-
assertNotEquals(0, stepTime);
395-
396-
assertNull(failedStep);
397-
398-
IndexLifecycleExplainResponse otherIndex = indices.get("other_index");
399-
assertFalse(otherIndex.managedByILM());
400-
assertNull(otherIndex.getPolicyName());
401-
assertNull(otherIndex.getPhase());
402-
assertNull(otherIndex.getAction());
403-
assertNull(otherIndex.getStep());
404-
assertNull(otherIndex.getFailedStep());
405-
assertNull(otherIndex.getPhaseExecutionInfo());
406-
assertNull(otherIndex.getStepInfo());
367+
368+
assertBusy(() -> {
369+
// tag::ilm-explain-lifecycle-execute
370+
ExplainLifecycleResponse response = client.indexLifecycle()
371+
.explainLifecycle(request, RequestOptions.DEFAULT);
372+
// end::ilm-explain-lifecycle-execute
373+
assertNotNull(response);
374+
375+
// tag::ilm-explain-lifecycle-response
376+
Map<String, IndexLifecycleExplainResponse> indices =
377+
response.getIndexResponses();
378+
IndexLifecycleExplainResponse myIndex = indices.get("my_index-1");
379+
String policyName = myIndex.getPolicyName(); // <1>
380+
boolean isManaged = myIndex.managedByILM(); // <2>
381+
382+
String phase = myIndex.getPhase(); // <3>
383+
long phaseTime = myIndex.getPhaseTime(); // <4>
384+
String action = myIndex.getAction(); // <5>
385+
long actionTime = myIndex.getActionTime();
386+
String step = myIndex.getStep(); // <6>
387+
long stepTime = myIndex.getStepTime();
388+
389+
String failedStep = myIndex.getFailedStep(); // <7>
390+
// end::ilm-explain-lifecycle-response
391+
392+
assertEquals("my_policy", policyName);
393+
assertTrue(isManaged);
394+
395+
assertEquals("hot", phase);
396+
assertNotEquals(0, phaseTime);
397+
assertEquals("rollover", action);
398+
assertNotEquals(0, actionTime);
399+
assertEquals("check-rollover-ready", step);
400+
assertNotEquals(0, stepTime);
401+
402+
assertNull(failedStep);
403+
404+
IndexLifecycleExplainResponse otherIndex = indices.get("other_index");
405+
assertFalse(otherIndex.managedByILM());
406+
assertNull(otherIndex.getPolicyName());
407+
assertNull(otherIndex.getPhase());
408+
assertNull(otherIndex.getAction());
409+
assertNull(otherIndex.getStep());
410+
assertNull(otherIndex.getFailedStep());
411+
assertNull(otherIndex.getPhaseExecutionInfo());
412+
assertNull(otherIndex.getStepInfo());
413+
});
407414

408415
// tag::ilm-explain-lifecycle-execute-listener
409416
ActionListener<ExplainLifecycleResponse> listener =

‎docs/reference/ilm/policy-definitions.asciidoc‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,13 @@ index format must match pattern '^.*-\\d+$', for example (`logs-000001`).
353353
The managed index must set `index.lifecycle.rollover_alias` as the
354354
alias to rollover. The index must also be the write index for the alias.
355355

356+
[IMPORTANT]
357+
If a policy using the Rollover action is used on a <<ccr-put-follow,follower
358+
index>>, policy execution will wait until the leader index rolls over (or has
359+
<<skipping-rollover, otherwise been marked as complete>>), then convert the
360+
follower index into a regular index as if <<ilm-unfollow-action,the Unfollow
361+
action>> had been used instead of rolling over.
362+
356363
For example, if an index to be managed has an alias `my_data`. The managed
357364
index "my_index" must be the write index for the alias. For more information, read
358365
<<indices-rollover-is-write-index,Write Index Alias Behavior>>.
@@ -578,6 +585,13 @@ PUT _ilm/policy/my_policy
578585

579586
NOTE: Index will be be made read-only when this action is run
580587
(see: <<dynamic-index-settings,index.blocks.write>>)
588+
[IMPORTANT]
589+
If a policy using the Shrink action is used on a <<ccr-put-follow,follower
590+
index>>, policy execution will wait until the leader index rolls over (or has
591+
<<skipping-rollover, otherwise been marked as complete>>), then convert the
592+
follower index into a regular index as if <<ilm-unfollow-action,the Unfollow
593+
action>> had been used before shrink is applied, as shrink cannot be safely
594+
applied to follower indices.
581595

582596
This action shrinks an existing index into a new index with fewer primary
583597
shards. It calls the <<indices-shrink-index,Shrink API>> to shrink the index.
@@ -622,11 +636,27 @@ PUT _ilm/policy/my_policy
622636
[[ilm-unfollow-action]]
623637
==== Unfollow
624638

639+
[IMPORTANT]
640+
This action may be used explicitly, as shown below, but this action is also run
641+
before <<ilm-rollover-action,the Rollover action>> and <<ilm-shrink-action,the
642+
Shrink action>> as described in the documentation for those actions.
643+
625644
This action turns a {ref}/ccr-apis.html[ccr] follower index
626645
into a regular index. This can be desired when moving follower
627646
indices into the next phase. Also certain actions like shrink
628647
and rollover can then be performed safely on follower indices.
629648

649+
This action will wait until is it safe to convert a follower index into a
650+
regular index. In particular, the following conditions must be met:
651+
652+
* The leader index must have `index.lifecycle.indexing_complete` set to `true`.
653+
This happens automatically if the leader index is rolled over using
654+
<<ilm-rollover-action,the Rollover action>>, or may be set manually using
655+
the <<indices-update-settings,Index Settings API>>.
656+
* All operations performed on the leader index must have been replicated to the
657+
follower index. This ensures that no operations will be lost when the index is
658+
converted into a regular index.
659+
630660
If the unfollow action encounters a follower index then
631661
the following operations will be performed on it:
632662

‎docs/reference/ilm/using-policies-rollover.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ When the rollover is performed, the newly-created index is set as the write
123123
index for the rolled over alias. Documents sent to the alias are indexed into
124124
the new index, enabling indexing to continue uninterrupted.
125125

126+
[[skipping-rollover]]
126127
=== Skipping Rollover
127128

128129
The `index.lifecycle.indexing_complete` setting indicates to {ilm} whether this

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyAction.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
*/
2626
public class ReadOnlyAction implements LifecycleAction {
2727
public static final String NAME = "readonly";
28-
public static final ReadOnlyAction INSTANCE = new ReadOnlyAction();
2928

3029
private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
3130

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,12 @@
66
package org.elasticsearch.xpack.core.indexlifecycle;
77

88
import org.elasticsearch.common.io.stream.StreamOutput;
9-
import org.elasticsearch.common.unit.TimeValue;
109
import org.elasticsearch.common.util.set.Sets;
1110

1211
import java.io.IOException;
1312
import java.util.ArrayList;
1413
import java.util.Arrays;
1514
import java.util.Collection;
16-
import java.util.Collections;
1715
import java.util.HashMap;
1816
import java.util.List;
1917
import java.util.Map;
@@ -44,8 +42,6 @@ public class TimeseriesLifecycleType implements LifecycleType {
4442
static final Set<String> VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS);
4543
static final Set<String> VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS);
4644
static final Set<String> VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS);
47-
private static final Phase EMPTY_WARM_PHASE = new Phase("warm", TimeValue.ZERO,
48-
Collections.singletonMap("readonly", ReadOnlyAction.INSTANCE));
4945
private static Map<String, Set<String>> ALLOWED_ACTIONS = new HashMap<>();
5046

5147
static {
@@ -72,6 +68,13 @@ public List<Phase> getOrderedPhases(Map<String, Phase> phases) {
7268
for (String phaseName : VALID_PHASES) {
7369
Phase phase = phases.get(phaseName);
7470
if (phase != null) {
71+
Map<String, LifecycleAction> actions = phase.getActions();
72+
if (actions.containsKey(UnfollowAction.NAME) == false
73+
&& (actions.containsKey(RolloverAction.NAME) || actions.containsKey(ShrinkAction.NAME))) {
74+
Map<String, LifecycleAction> actionMap = new HashMap<>(phase.getActions());
75+
actionMap.put(UnfollowAction.NAME, new UnfollowAction());
76+
phase = new Phase(phase.getName(), phase.getMinimumAge(), actionMap);
77+
}
7578
orderedPhases.add(phase);
7679
}
7780
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,32 @@ public void testGetOrderedPhases() {
152152
assertTrue(isSorted(TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap), Phase::getName, VALID_PHASES));
153153
}
154154

155+
public void testUnfollowInjections() {
156+
assertTrue(isUnfollowInjected("hot", RolloverAction.NAME));
157+
assertTrue(isUnfollowInjected("warm", ShrinkAction.NAME));
158+
159+
assertFalse(isUnfollowInjected("hot", SetPriorityAction.NAME));
160+
assertFalse(isUnfollowInjected("warm", SetPriorityAction.NAME));
161+
assertFalse(isUnfollowInjected("warm", AllocateAction.NAME));
162+
assertFalse(isUnfollowInjected("warm", ReadOnlyAction.NAME));
163+
assertFalse(isUnfollowInjected("warm", ForceMergeAction.NAME));
164+
assertFalse(isUnfollowInjected("cold", SetPriorityAction.NAME));
165+
assertFalse(isUnfollowInjected("cold", AllocateAction.NAME));
166+
assertFalse(isUnfollowInjected("cold", FreezeAction.NAME));
167+
assertFalse(isUnfollowInjected("delete", DeleteAction.NAME));
168+
169+
}
170+
171+
private boolean isUnfollowInjected(String phaseName, String actionName) {
172+
Map<String, Phase> phaseMap = new HashMap<>();
173+
Map<String, LifecycleAction> actionsMap = new HashMap<>();
174+
actionsMap.put(actionName, getTestAction(actionName));
175+
Phase warmPhase = new Phase(phaseName, TimeValue.ZERO, actionsMap);
176+
phaseMap.put(phaseName, warmPhase);
177+
List<Phase> phases = TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap);
178+
Phase processedWarmPhase = phases.stream().filter(phase -> phase.getName().equals(phaseName)).findFirst().get();
179+
return processedWarmPhase.getActions().containsKey("unfollow");
180+
}
155181

156182
public void testGetOrderedActionsInvalidPhase() {
157183
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.INSTANCE

0 commit comments

Comments
 (0)