ILM fix the init step to actually be retryable

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
+import java.util.Objects;
+
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
 import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
@@ -25,8 +27,11 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
     public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
     private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
 
-    InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
+    private final String policy;
+
+    InitializePolicyContextStep(String policy, Step.StepKey key, StepKey nextStepKey) {
         super(key, nextStepKey);
+        this.policy = policy;
     }
 
     @Override
@@ -38,22 +43,25 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             return clusterState;
         }
 
-        LifecycleExecutionState lifecycleState = LifecycleExecutionState
-            .fromIndexMetadata(indexMetaData);
-
-        if (lifecycleState.getLifecycleDate() != null) {
-            return clusterState;
-        }
-
         IndexMetaData.Builder indexMetadataBuilder = IndexMetaData.builder(indexMetaData);
-        if (shouldParseIndexName(indexMetaData.getSettings())) {
-            long parsedOriginationDate = parseIndexNameAndExtractDate(index.getName());
-            indexMetadataBuilder.settingsVersion(indexMetaData.getSettingsVersion() + 1)
-                .settings(Settings.builder()
-                    .put(indexMetaData.getSettings())
-                    .put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, parsedOriginationDate)
-                    .build()
-                );
+        LifecycleExecutionState lifecycleState;
+        try {
+            lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+            if (lifecycleState.getLifecycleDate() != null) {
+                return clusterState;
+            }
+
+            if (shouldParseIndexName(indexMetaData.getSettings())) {
+                long parsedOriginationDate = parseIndexNameAndExtractDate(index.getName());
+                indexMetadataBuilder.settingsVersion(indexMetaData.getSettingsVersion() + 1)
+                    .settings(Settings.builder()
+                        .put(indexMetaData.getSettings())
+                        .put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, parsedOriginationDate)
+                        .build()
+                    );
+            }
+        } catch (Exception e) {
+            throw new InitializePolicyException(policy, index.getName(), e);
         }
 
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
@@ -72,4 +80,22 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
     public boolean isRetryable() {
         return true;
     }
+
+    String policy() {
+        return policy;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        InitializePolicyContextStep that = (InitializePolicyContextStep) o;
+        return policy.equals(that.policy);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), policy);
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyException.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.ElasticsearchException;
+
+import java.util.Locale;
+
+/**
+ * Exception thrown when a problem is encountered while initialising an ILM policy for an index.
+ */
+public class InitializePolicyException extends ElasticsearchException {
+
+    public InitializePolicyException(String policy, String index, Throwable cause) {
+        super(String.format(Locale.ROOT, "unable to initialize policy [%s] for index [%s]", policy, index), cause);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java

@@ -215,7 +215,7 @@ public List<Step> toSteps(Client client) {
         lastStepKey = phaseAfterStep.getKey();
 
         // init step so that policy is guaranteed to have
-        steps.add(new InitializePolicyContextStep(InitializePolicyContextStep.KEY, lastStepKey));
+        steps.add(new InitializePolicyContextStep(name, InitializePolicyContextStep.KEY, lastStepKey));
 
         Collections.reverse(steps);
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrunkenIndexCheckStep.java

@@ -25,7 +25,7 @@
  */
 public class ShrunkenIndexCheckStep extends ClusterStateWaitStep {
     public static final String NAME = "is-shrunken-index";
-    private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
+    private static final Logger logger = LogManager.getLogger(ShrunkenIndexCheckStep.class);
     private String shrunkIndexPrefix;
 
     public ShrunkenIndexCheckStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStepTests.java

@@ -24,31 +24,35 @@ public InitializePolicyContextStep createRandomInstance() {
         StepKey stepKey = randomStepKey();
         StepKey nextStepKey = randomStepKey();
 
-        return new InitializePolicyContextStep(stepKey, nextStepKey);
+        return new InitializePolicyContextStep(randomAlphaOfLengthBetween(1, 10), stepKey, nextStepKey);
     }
 
     @Override
     public InitializePolicyContextStep mutateInstance(InitializePolicyContextStep instance) {
+        String policy = instance.policy();
         StepKey key = instance.getKey();
         StepKey nextKey = instance.getNextStepKey();
 
-        switch (between(0, 1)) {
-        case 0:
-            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
-            break;
-        case 1:
-            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
-            break;
-        default:
-            throw new AssertionError("Illegal randomisation branch");
+        switch (between(0, 2)) {
+            case 0:
+                policy = randomAlphaOfLengthBetween(1, 10);
+                break;
+            case 1:
+                key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 2:
+                nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
         }
 
-        return new InitializePolicyContextStep(key, nextKey);
+        return new InitializePolicyContextStep(policy, key, nextKey);
     }
 
     @Override
     public InitializePolicyContextStep copyInstance(InitializePolicyContextStep instance) {
-        return new InitializePolicyContextStep(instance.getKey(), instance.getNextStepKey());
+        return new InitializePolicyContextStep(instance.policy(), instance.getKey(), instance.getNextStepKey());
     }
 
     public void testAddCreationDate() {
@@ -63,7 +67,7 @@ public void testAddCreationDate() {
             .build();
         Index index = indexMetadata.getIndex();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
-        InitializePolicyContextStep step = new InitializePolicyContextStep(null, null);
+        InitializePolicyContextStep step = new InitializePolicyContextStep(randomAlphaOfLengthBetween(1, 10), null, null);
         ClusterState newState = step.performAction(index, clusterState);
         assertThat(getIndexLifecycleDate(index, newState), equalTo(creationDate));
     }
@@ -83,7 +87,7 @@ public void testDoNothing() {
             .build();
         Index index = indexMetadata.getIndex();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
-        InitializePolicyContextStep step = new InitializePolicyContextStep(null, null);
+        InitializePolicyContextStep step = new InitializePolicyContextStep(randomAlphaOfLengthBetween(1, 10), null, null);
         ClusterState newState = step.performAction(index, clusterState);
         assertTrue(newState == clusterState);
     }

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java

@@ -1176,26 +1176,26 @@ public void testRolloverStepRetriesUntilRolledOverIndexIsDeleted() throws Except
         // {@link org.elasticsearch.xpack.core.ilm.ErrorStep} in order to retry the failing step. As {@link #assertBusy}
         // increases the wait time between calls exponentially, we might miss the window where the policy is on
         // {@link WaitForRolloverReadyStep} and the move to `attempt-rollover` request will not be successful.
-        waitUntil(() -> {
+        assertTrue(waitUntil(() -> {
             try {
                 return client().performRequest(moveToStepRequest).getStatusLine().getStatusCode() == 200;
             } catch (IOException e) {
                 return false;
             }
-        }, 30, TimeUnit.SECONDS);
+        }, 30, TimeUnit.SECONDS));
 
         // Similar to above, using {@link #waitUntil} as we want to make sure the `attempt-rollover` step started failing and is being
         // retried (which means ILM moves back and forth between the `attempt-rollover` step and the `error` step)
-        waitUntil(() -> {
+        assertTrue("ILM did not start retrying the attempt-rollover step", waitUntil(() -> {
             try {
                 Map<String, Object> explainIndexResponse = explainIndex(index);
-                String step = (String) explainIndexResponse.get("step");
+                String failedStep = (String) explainIndexResponse.get("failed_step");
                 Integer retryCount = (Integer) explainIndexResponse.get(FAILED_STEP_RETRY_COUNT_FIELD);
-                return step != null && step.equals("attempt-rollover") && retryCount != null && retryCount >= 1;
+                return failedStep != null && failedStep.equals("attempt-rollover") && retryCount != null && retryCount >= 1;
             } catch (IOException e) {
                 return false;
             }
-        }, 30, TimeUnit.SECONDS);
+        }, 30, TimeUnit.SECONDS));
 
         deleteIndex(rolledIndex);
 
@@ -1237,16 +1237,17 @@ public void testUpdateRolloverLifecycleDateStepRetriesWhenRolloverInfoIsMissing(
             "}");
         client().performRequest(moveToStepRequest);
 
-        waitUntil(() -> {
+        assertTrue("ILM did not start retrying the update-rollover-lifecycle-date step", waitUntil(() -> {
             try {
                 Map<String, Object> explainIndexResponse = explainIndex(index);
-                String step = (String) explainIndexResponse.get("step");
+                String failedStep = (String) explainIndexResponse.get("failed_step");
                 Integer retryCount = (Integer) explainIndexResponse.get(FAILED_STEP_RETRY_COUNT_FIELD);
-                return step != null && step.equals(UpdateRolloverLifecycleDateStep.NAME) && retryCount != null && retryCount >= 1;
+                return failedStep != null && failedStep.equals(UpdateRolloverLifecycleDateStep.NAME) && retryCount != null
+                    && retryCount >= 1;
             } catch (IOException e) {
                 return false;
             }
-        });
+        }, 30, TimeUnit.SECONDS));
 
         index(client(), index, "1", "foo", "bar");
         Request refreshIndex = new Request("POST", "/" + index + "/_refresh");
@@ -1432,16 +1433,17 @@ public void testRetryableInitializationStep() throws Exception {
         assertOK(client().performRequest(startReq));
 
         // Wait until an error has occurred.
-        waitUntil(() -> {
+        assertTrue("ILM did not start retrying the init step", waitUntil(() -> {
             try {
                 Map<String, Object> explainIndexResponse = explainIndex(index);
-                String step = (String) explainIndexResponse.get("step");
+                String failedStep = (String) explainIndexResponse.get("failed_step");
                 Integer retryCount = (Integer) explainIndexResponse.get(FAILED_STEP_RETRY_COUNT_FIELD);
-                return step != null && step.equals(InitializePolicyContextStep.KEY.getAction()) && retryCount != null && retryCount >= 1;
+                return failedStep != null && failedStep.equals(InitializePolicyContextStep.KEY.getAction()) && retryCount != null
+                    && retryCount >= 1;
             } catch (IOException e) {
                 return false;
             }
-        }, 30, TimeUnit.SECONDS);
+        }, 30, TimeUnit.SECONDS));
 
         // Turn origination date parsing back off
         updateIndexSettings(index, Settings.builder()

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.ilm.ErrorStep;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.ilm.InitializePolicyException;
 import org.elasticsearch.xpack.core.ilm.LifecycleExecutionState;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
@@ -133,8 +134,16 @@ static ClusterState moveClusterStateToErrorStep(Index index, ClusterState cluste
         ElasticsearchException.generateThrowableXContent(causeXContentBuilder, STACKTRACE_PARAMS, cause);
         causeXContentBuilder.endObject();
         LifecycleExecutionState currentState = LifecycleExecutionState.fromIndexMetadata(idxMeta);
-        Step.StepKey currentStep = Objects.requireNonNull(LifecycleExecutionState.getCurrentStepKey(currentState),
-            "unable to move to an error step where there is no current step, state: " + currentState);
+        Step.StepKey currentStep;
+        // if an error is encountered while initialising the policy the lifecycle execution state will not yet contain any step information
+        // as we haven't yet initialised the policy, so we'll manually set the current step to be the "initialize policy" step so we can
+        // record the error (and later retry the init policy step)
+        if (cause instanceof InitializePolicyException) {
+            currentStep = InitializePolicyContextStep.KEY;
+        } else {
+            currentStep = Objects.requireNonNull(LifecycleExecutionState.getCurrentStepKey(currentState),
+                "unable to move to an error step where there is no current step, state: " + currentState);
+        }
         LifecycleExecutionState nextStepState = updateExecutionStateToStep(policyMetadata, currentState,
             new Step.StepKey(currentStep.getPhase(), currentStep.getAction(), ErrorStep.NAME), nowSupplier, false);