diff --git a/docs/reference/ilm/apis/ilm-api.asciidoc b/docs/reference/ilm/apis/ilm-api.asciidoc index 97a00e2c3fa73..149ba2a6b4491 100644 --- a/docs/reference/ilm/apis/ilm-api.asciidoc +++ b/docs/reference/ilm/apis/ilm-api.asciidoc @@ -1,7 +1,7 @@ [[index-lifecycle-management-api]] == {ilm-cap} APIs -You use the following APIs to set up policies to automatically manage the index lifecycle. +You use the following APIs to set up policies to automatically manage the index lifecycle. For more information about {ilm} ({ilm-init}), see <>. [discrete] @@ -28,6 +28,7 @@ For more information about {ilm} ({ilm-init}), see < * <> * <> * <> +* <> include::put-lifecycle.asciidoc[] @@ -42,3 +43,4 @@ include::get-status.asciidoc[] include::explain.asciidoc[] include::start.asciidoc[] include::stop.asciidoc[] +include::migrate-to-data-tiers.asciidoc[] diff --git a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc new file mode 100644 index 0000000000000..c0a612129b74c --- /dev/null +++ b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc @@ -0,0 +1,130 @@ +[role="xpack"] +[testenv="basic"] +[[ilm-migrate-to-data-tiers]] +=== Migrate to data tiers routing API +++++ +Migrate indices and ILM policies to data tiers routing +++++ + +Switches the indices and ILM policies from using custom node attributes and +<> to using <>, and +optionally deletes one legacy index template. +Using node roles enables {ilm-init} to <> between +data tiers. + +Migrating away from custom node attributes routing can be manually performed +as indicated in the <> page. + +This API provides an automated way of executing three out of the four manual steps listed +in the <>: + +. <> +. <> +. <> with the corresponding <> + +[[ilm-migrate-to-data-tiers-request]] +==== {api-request-title} + +`POST /_ilm/migrate_to_data_tiers` + +The API accepts an optional body that allows you to specify: + +- The legacy index template name to delete. Defaults to none. +- The name of the custom node attribute used for the indices and ILM policies allocation filtering. +Defaults to `data`. + +[[ilm-migrate-to-data-tiers-prereqs]] +==== {api-prereq-title} + +* {ilm-init} must be stopped before performing the migration. Use the <> +to stop {ilm-init} and <> to wait until the +reported operation mode is `STOPPED`. + +[[ilm-migrate-to-data-tiers-example]] +==== {api-examples-title} + +The following example migrates the indices and ILM policies away from defining +custom allocation filtering using the `custom_attribute_name` node attribute, and +deletes legacy template with name `global-template` if it exists in the system. + +//// +[source,console] +---- +POST _ilm/stop + +PUT _template/global-template +{ + "index_patterns": ["migrate-to-tiers-*"], + "settings": { + "index.routing.allocation.require.custom_attribute_name": "hot" + } +} + +PUT warm-index-to-migrate-000001 +{ + "settings": { + "index.routing.allocation.require.custom_attribute_name": "warm" + } +} + +PUT _ilm/policy/policy_with_allocate_action +{ + "policy": { + "phases": { + "warm": { + "actions": { + "allocate": { + "require": { + "custom_attribute_name": "warm" + } + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } +} +---- +// TESTSETUP + +[source,console] +---- +DELETE warm-index-to-migrate-000001 + +DELETE _ilm/policy/policy_with_allocate_action + +POST _ilm/start +---- +// TEARDOWN +//// + +[source,console] +---------------------------------------------------------------- +POST /_ilm/migrate_to_data_tiers +{ + "legacy_template_to_delete": "global-template", + "node_attribute": "custom_attribute_name" +} +---------------------------------------------------------------- + +If the request succeeds, a response like the following will be received: + +[source,console-result] +------------------------------------------------------------------------------ +{ + "removed_legacy_template":"global-template", <1> + "migrated_ilm_policies":["policy_with_allocate_action"], <2> + "migrated_indices":["warm-index-to-migrate-000001"] <3> +} +------------------------------------------------------------------------------ + +<1> Shows the name of the legacy index template that was deleted. This will be missing +if no legacy index template was deleted. +<2> The ILM policies that were updated. +<3> The indices that were migrated to <> routing. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json new file mode 100644 index 0000000000000..57b9cb2b717f7 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json @@ -0,0 +1,29 @@ +{ + "ilm.migrate_to_data_tiers":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate-to-data-tiers.html", + "description": "Migrates the indices and ILM policies away from custom node attribute allocation routing to data tiers routing" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ilm/migrate_to_data_tiers", + "methods":[ + "POST" + ] + } + ] + }, + "params":{}, + "body":{ + "description":"Optionally specify a legacy index template name to delete and optionally specify a node attribute name used for index shard routing (defaults to \"data\")", + "required":false + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersAction.java new file mode 100644 index 0000000000000..1769958168517 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersAction.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.xpack.cluster.action; + +import org.elasticsearch.action.ActionType; + +public class MigrateToDataTiersAction extends ActionType { + + public static final MigrateToDataTiersAction INSTANCE = new MigrateToDataTiersAction(); + public static final String NAME = "cluster:admin/migrate_to_data_tiers"; + + private MigrateToDataTiersAction() { + super(NAME, MigrateToDataTiersResponse::new); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java new file mode 100644 index 0000000000000..7b8c110e87bad --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.util.Objects; + +public class MigrateToDataTiersRequest extends AcknowledgedRequest { + + private static final ParseField LEGACY_TEMPLATE_TO_DELETE = new ParseField("legacy_template_to_delete"); + private static final ParseField NODE_ATTRIBUTE_NAME = new ParseField("node_attribute"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("index_template", + false, + a -> new MigrateToDataTiersRequest((String) a[0], (String) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LEGACY_TEMPLATE_TO_DELETE); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), NODE_ATTRIBUTE_NAME); + } + + /** + * Represents the name of node attribute used for index shard allocation filtering (usually `data`) + */ + @Nullable + private final String nodeAttributeName; + + /** + * Represents the name of the legacy (v1) index template to delete. + */ + @Nullable + private final String legacyTemplateToDelete; + + public static MigrateToDataTiersRequest parse(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + public MigrateToDataTiersRequest(@Nullable String legacyTemplateToDelete, @Nullable String nodeAttributeName) { + this.legacyTemplateToDelete = legacyTemplateToDelete; + this.nodeAttributeName = nodeAttributeName; + } + + public MigrateToDataTiersRequest() { + this(null, null); + } + + public MigrateToDataTiersRequest(StreamInput in) throws IOException { + super(in); + legacyTemplateToDelete = in.readOptionalString(); + nodeAttributeName = in.readOptionalString(); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(legacyTemplateToDelete); + out.writeOptionalString(nodeAttributeName); + } + + public String getNodeAttributeName() { + return nodeAttributeName; + } + + public String getLegacyTemplateToDelete() { + return legacyTemplateToDelete; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MigrateToDataTiersRequest that = (MigrateToDataTiersRequest) o; + return Objects.equals(nodeAttributeName, that.nodeAttributeName) && Objects.equals(legacyTemplateToDelete, + that.legacyTemplateToDelete); + } + + @Override + public int hashCode() { + return Objects.hash(nodeAttributeName, legacyTemplateToDelete); + } + + @Override + public String toString() { + return "MigrateToDataTiersRequest{" + + "nodeAttributeName='" + nodeAttributeName + '\'' + + ", legacyTemplateToDelete='" + legacyTemplateToDelete + '\'' + + '}'; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java new file mode 100644 index 0000000000000..a281d51458826 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.util.List; + +public class MigrateToDataTiersResponse extends ActionResponse implements ToXContentObject { + + public static final ParseField REMOVED_LEGACY_TEMPLATE = new ParseField("removed_legacy_template"); + public static final ParseField MIGRATED_INDICES = new ParseField("migrated_indices"); + public static final ParseField MIGRATED_ILM_POLICIES = new ParseField("migrated_ilm_policies"); + + @Nullable + private final String removedIndexTemplateName; + private final List migratedPolicies; + private final List migratedIndices; + + public MigrateToDataTiersResponse(@Nullable String removedIndexTemplateName, List migratedPolicies, + List migratedIndices) { + this.removedIndexTemplateName = removedIndexTemplateName; + this.migratedPolicies = migratedPolicies; + this.migratedIndices = migratedIndices; + } + + public MigrateToDataTiersResponse(StreamInput in) throws IOException { + super(in); + removedIndexTemplateName = in.readOptionalString(); + migratedPolicies = in.readStringList(); + migratedIndices = in.readStringList(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (this.removedIndexTemplateName != null) { + builder.field(REMOVED_LEGACY_TEMPLATE.getPreferredName(), this.removedIndexTemplateName); + } + if (migratedPolicies.size() > 0) { + builder.startArray(MIGRATED_ILM_POLICIES.getPreferredName()); + for (String policy : migratedPolicies) { + builder.value(policy); + } + builder.endArray(); + } + if (migratedIndices.size() > 0) { + builder.startArray(MIGRATED_INDICES.getPreferredName()); + for (String index : migratedIndices) { + builder.value(index); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(removedIndexTemplateName); + out.writeStringCollection(migratedPolicies); + out.writeStringCollection(migratedIndices); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a580d72c5d27e..f4d8de6d36665 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; import org.elasticsearch.xpack.core.enrich.EnrichFeatureSet; @@ -482,6 +483,7 @@ public List> getClientActions() { DeleteSnapshotLifecycleAction.INSTANCE, ExecuteSnapshotLifecycleAction.INSTANCE, GetSnapshotLifecycleStatsAction.INSTANCE, + MigrateToDataTiersAction.INSTANCE, // Freeze FreezeIndexAction.INSTANCE, // Data Frame diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java new file mode 100644 index 0000000000000..16aa64513de42 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class MigrateToDataTiersRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return MigrateToDataTiersRequest::new; + } + + @Override + protected MigrateToDataTiersRequest createTestInstance() { + return new MigrateToDataTiersRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } + + @Override + protected MigrateToDataTiersRequest mutateInstance(MigrateToDataTiersRequest instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java new file mode 100644 index 0000000000000..f4bcf095b3877 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersResponse; +import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; +import org.elasticsearch.xpack.core.ilm.AllocateAction; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.ForceMergeAction; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.SetPriorityAction; +import org.elasticsearch.xpack.core.ilm.ShrinkAction; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.createPolicy; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings; +import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class MigrateToDataTiersIT extends ESRestTestCase { + private static final Logger logger = LogManager.getLogger(MigrateToDataTiersIT.class); + + private String index; + private String policy; + private String alias; + + @Before + public void refreshIndexAndStartILM() throws IOException { + index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + policy = "policy-" + randomAlphaOfLength(5); + alias = "alias-" + randomAlphaOfLength(5); + assertOK(client().performRequest(new Request("POST", "_ilm/start"))); + } + + @AfterClass + public static void restartIlm() throws IOException { + // some tests might stop ILM in order to perform the migration to data tiers, let's restart it + assertOK(client().performRequest(new Request("POST", "_ilm/start"))); + } + + public void testAPIFailsIfILMIsNotStopped() throws IOException { + Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers"); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(migrateRequest)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(RestStatus.INTERNAL_SERVER_ERROR.getStatus())); + assertThat(e.getMessage(), containsString("stop ILM before migrating to data tiers, current state is [RUNNING]")); + } + + @SuppressWarnings("unchecked") + public void testMigrateToDataTiersAction() throws Exception { + // creating a legacy template to use in the migrate API + String templateName = randomAlphaOfLengthBetween(10, 15).toLowerCase(Locale.ROOT); + createLegacyTemplate(templateName); + + // let's create a policy that'll need migrating, with a long `min_age` for the cold phase such that managed indices stop in + // Warm/Complete/Complete - this will ensure the migration will have to update the cached phase for these indices + Map hotActions = new HashMap<>(); + hotActions.put(SetPriorityAction.NAME, new SetPriorityAction(100)); + Map warmActions = new HashMap<>(); + warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50)); + warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1, null)); + warmActions.put(AllocateAction.NAME, new AllocateAction(null, singletonMap("data", "warm"), null, null)); + warmActions.put(ShrinkAction.NAME, new ShrinkAction(1, null)); + Map coldActions = new HashMap<>(); + coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0)); + coldActions.put(AllocateAction.NAME, new AllocateAction(0, null, null, singletonMap("data", "cold"))); + + createPolicy(client(), policy, + new Phase("hot", TimeValue.ZERO, hotActions), + new Phase("warm", TimeValue.ZERO, warmActions), + new Phase("cold", TimeValue.timeValueDays(100), coldActions), + null, + new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction())) + ); + + createIndexWithSettings(client(), index, alias, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(LifecycleSettings.LIFECYCLE_NAME, policy) + .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + ); + + // wait for the index to advance to the warm phase + assertBusy(() -> + assertThat(getStepKeyForIndex(client(), index).getPhase(), equalTo("warm")), 30, TimeUnit.SECONDS); + + // let's also have a policy that doesn't need migrating + String rolloverOnlyPolicyName = "rollover-policy"; + createNewSingletonPolicy(client(), rolloverOnlyPolicyName, "hot", new RolloverAction(null, null, null, 1L)); + + String rolloverIndexPrefix = "rolloverpolicytest_index"; + for (int i = 1; i < randomIntBetween(2, 5); i++) { + // assign the rollover-only policy to a few other indices - these indices and the rollover-only policy should not be migrated + // in any way + createIndexWithSettings(client(), rolloverIndexPrefix + "-00000" + i, alias + i, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i) + ); + } + + // let's stop ILM so we can perform the migration + client().performRequest(new Request("POST", "_ilm/stop")); + assertBusy(() -> { + Response response = client().performRequest(new Request("GET", "_ilm/status")); + assertThat(EntityUtils.toString(response.getEntity()), containsString(OperationMode.STOPPED.toString())); + }); + + String indexWithDataWarmRouting = "indexwithdatawarmrouting"; + Settings.Builder settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm"); + createIndex(indexWithDataWarmRouting, settings.build()); + + Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers"); + migrateRequest.setJsonEntity( + "{\"legacy_template_to_delete\": \"" + templateName + "\", \"node_attribute\": \"data\"}" + ); + Response migrateDeploymentResponse = client().performRequest(migrateRequest); + assertOK(migrateDeploymentResponse); + + Map migrateResponseAsMap = responseAsMap(migrateDeploymentResponse); + assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()), + containsInAnyOrder(policy)); + assertThat((ArrayList) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()), + containsInAnyOrder(index, indexWithDataWarmRouting)); + assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()), + is(templateName)); + + // let's verify the legacy template doesn't exist anymore + Request getTemplateRequest = new Request("HEAD", "_template/" + templateName); + assertThat(client().performRequest(getTemplateRequest).getStatusLine().getStatusCode(), is(RestStatus.NOT_FOUND.getStatus())); + + // let's assert the require.data:warm configuration the "indexWithDataWarmRouting" had was migrated to + // _tier_preference:data_warm,data_hot + Map indexSettings = getOnlyIndexSettings(client(), indexWithDataWarmRouting); + assertThat(indexSettings.get(DataTierAllocationDecider.INDEX_ROUTING_PREFER), is("data_warm,data_hot")); + + // let's retrieve the migrated policy and check it was migrated correctly - namely the warm phase should not contain any allocate + // action anymore and the cold phase should contain an allocate action that only configures the number of replicas + Request getPolicy = new Request("GET", "/_ilm/policy/" + policy); + Map policyAsMap = (Map) responseAsMap(client().performRequest(getPolicy)).get(policy); + Map warmActionsMap = getActionsForPhase(policyAsMap, "warm"); + assertThat(warmActionsMap.size(), is(3)); + assertThat(warmActionsMap.get(AllocateAction.NAME), nullValue()); + Map coldActionsMap = getActionsForPhase(policyAsMap, "cold"); + assertThat(coldActionsMap.size(), is(2)); + assertThat(coldActionsMap.get(AllocateAction.NAME), notNullValue()); + Map coldAllocateActionMap = (Map) coldActionsMap.get(AllocateAction.NAME); + assertThat((Map) coldAllocateActionMap.get("include"), is(anEmptyMap())); + assertThat((Map) coldAllocateActionMap.get("require"), is(anEmptyMap())); + assertThat((Map) coldAllocateActionMap.get("exclude"), is(anEmptyMap())); + + Request getClusterMetadataRequest = new Request("GET", "/_cluster/state/metadata/" + index); + Response clusterMetadataResponse = client().performRequest(getClusterMetadataRequest); + + String cachedPhaseDefinition = getCachedPhaseDefAsMap(clusterMetadataResponse, index); + // let's also verify the cached phase definition was updated - as the managed index was in the warm phase, which after migration + // does not contain the allocate action anymore, the cached warm phase should not contain the allocate action either + assertThat("the cached phase definition should reflect the migrated warm phase which must NOT contain an allocate action anymore", + cachedPhaseDefinition, not(containsString(AllocateAction.NAME))); + assertThat(cachedPhaseDefinition, containsString(ShrinkAction.NAME)); + assertThat(cachedPhaseDefinition, containsString(SetPriorityAction.NAME)); + assertThat(cachedPhaseDefinition, containsString(ForceMergeAction.NAME)); + } + + @SuppressWarnings("unchecked") + private Map getActionsForPhase(Map policyAsMap, String phase) { + Map phases = (Map) ((Map) policyAsMap.get("policy")).get("phases"); + return (Map) ((Map) phases.get(phase)).get("actions"); + } + + @SuppressWarnings("unchecked") + private String getCachedPhaseDefAsMap(Response clusterMetadataResponse, String indexName) throws IOException { + Map clusterMetadataMap = responseAsMap(clusterMetadataResponse); + Map metadata = (Map) clusterMetadataMap.get("metadata"); + Map indices = (Map) metadata.get("indices"); + Map indexMetadata = (Map) indices.get(indexName); + Map ilmMetadata = (Map) indexMetadata.get("ilm"); + return (String) ilmMetadata.get("phase_definition"); + } + + private void createLegacyTemplate(String templateName) throws IOException { + String indexPrefix = randomAlphaOfLengthBetween(5, 15).toLowerCase(Locale.ROOT); + final StringEntity template = new StringEntity("{\n" + + " \"index_patterns\": \"" + indexPrefix + "*\",\n" + + " \"settings\": {\n" + + " \"index\": {\n" + + " \"lifecycle\": {\n" + + " \"name\": \"does_not_exist\",\n" + + " \"rollover_alias\": \"test_alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}", ContentType.APPLICATION_JSON); + Request templateRequest = new Request("PUT", "_template/" + templateName); + templateRequest.setEntity(template); + templateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); + client().performRequest(templateRequest); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 98881d027629b..5aeefb8d5d186 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; @@ -26,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -40,6 +40,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ilm.AllocateAction; import org.elasticsearch.xpack.core.ilm.DeleteAction; @@ -85,6 +86,7 @@ import org.elasticsearch.xpack.ilm.action.RestExplainLifecycleAction; import org.elasticsearch.xpack.ilm.action.RestGetLifecycleAction; import org.elasticsearch.xpack.ilm.action.RestGetStatusAction; +import org.elasticsearch.xpack.ilm.action.RestMigrateToDataTiersAction; import org.elasticsearch.xpack.ilm.action.RestMoveToStepAction; import org.elasticsearch.xpack.ilm.action.RestPutLifecycleAction; import org.elasticsearch.xpack.ilm.action.RestRemoveIndexLifecyclePolicyAction; @@ -95,6 +97,7 @@ import org.elasticsearch.xpack.ilm.action.TransportExplainLifecycleAction; import org.elasticsearch.xpack.ilm.action.TransportGetLifecycleAction; import org.elasticsearch.xpack.ilm.action.TransportGetStatusAction; +import org.elasticsearch.xpack.ilm.action.TransportMigrateToDataTiersAction; import org.elasticsearch.xpack.ilm.action.TransportMoveToStepAction; import org.elasticsearch.xpack.ilm.action.TransportPutLifecycleAction; import org.elasticsearch.xpack.ilm.action.TransportRemoveIndexLifecyclePolicyAction; @@ -275,8 +278,8 @@ public List getRestHandlers(Settings settings, RestController restC IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { List handlers = new ArrayList<>(); - handlers.addAll(Arrays.asList( + handlers.addAll(Arrays.asList( // add ILM rest handlers new RestPutLifecycleAction(), new RestGetLifecycleAction(), @@ -288,6 +291,7 @@ public List getRestHandlers(Settings settings, RestController restC new RestStopAction(), new RestStartILMAction(), new RestGetStatusAction(), + new RestMigrateToDataTiersAction(), // add SLM rest headers new RestPutSnapshotLifecycleAction(), @@ -319,6 +323,7 @@ public List getRestHandlers(Settings settings, RestController restC new ActionHandler<>(StartILMAction.INSTANCE, TransportStartILMAction.class), new ActionHandler<>(StopILMAction.INSTANCE, TransportStopILMAction.class), new ActionHandler<>(GetStatusAction.INSTANCE, TransportGetStatusAction.class), + new ActionHandler<>(MigrateToDataTiersAction.INSTANCE, TransportMigrateToDataTiersAction.class), // add SLM actions new ActionHandler<>(PutSnapshotLifecycleAction.INSTANCE, TransportPutSnapshotLifecycleAction.class), diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestMigrateToDataTiersAction.java new file mode 100644 index 0000000000000..c1f202fd91d7b --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestMigrateToDataTiersAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ilm.action; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersRequest; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestMigrateToDataTiersAction extends BaseRestHandler { + + @Override + public String getName() { + return "migrate_to_data_tiers_action"; + } + + @Override + public List routes() { + return org.elasticsearch.core.List.of(new Route(POST, "/_ilm/migrate_to_data_tiers")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + MigrateToDataTiersRequest migrateRequest = request.hasContent() ? + MigrateToDataTiersRequest.parse(request.contentParser()) : new MigrateToDataTiersRequest(); + return channel -> client.execute(MigrateToDataTiersAction.INSTANCE, migrateRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java new file mode 100644 index 0000000000000..c02c8f7edeb30 --- /dev/null +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ilm.action; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersRequest; +import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersResponse; +import org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.MigratedEntities; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; + +import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateToDataTiersRouting; +import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED; + +public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction { + + private final NamedXContentRegistry xContentRegistry; + private final Client client; + private final XPackLicenseState licenseState; + + @Inject + public TransportMigrateToDataTiersAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + NamedXContentRegistry xContentRegistry, Client client, XPackLicenseState licenseState) { + super(MigrateToDataTiersAction.NAME, transportService, clusterService, threadPool, actionFilters, MigrateToDataTiersRequest::new, + indexNameExpressionResolver, MigrateToDataTiersResponse::new, ThreadPool.Names.SAME); + this.xContentRegistry = xContentRegistry; + this.client = client; + this.licenseState = licenseState; + } + + @Override + protected void masterOperation(MigrateToDataTiersRequest request, ClusterState state, + ActionListener listener) throws Exception { + IndexLifecycleMetadata currentMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); + if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) { + listener.onFailure(new IllegalStateException("stop ILM before migrating to data tiers, current state is [" + + currentMetadata.getOperationMode() + "]")); + return; + } + final SetOnce migratedEntities = new SetOnce<>(); + clusterService.submitStateUpdateTask("migrate-to-data-tiers []", new ClusterStateUpdateTask(Priority.HIGH) { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + Tuple migratedEntitiesTuple = + migrateToDataTiersRouting(state, request.getNodeAttributeName(), request.getLegacyTemplateToDelete(), + xContentRegistry, client, licenseState); + + migratedEntities.set(migratedEntitiesTuple.v2()); + return migratedEntitiesTuple.v1(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + super.clusterStateProcessed(source, oldState, newState); + MigratedEntities entities = migratedEntities.get(); + listener.onResponse( + new MigrateToDataTiersResponse(entities.removedIndexTemplateName, entities.migratedPolicies, entities.migratedIndices) + ); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(MigrateToDataTiersRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 39a8c39ba2df0..d1908444072ce 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -44,6 +44,7 @@ public class Constants { "cluster:admin/logstash/pipeline/delete", "cluster:admin/logstash/pipeline/get", "cluster:admin/logstash/pipeline/put", + "cluster:admin/migrate_to_data_tiers", "cluster:admin/nodes/reload_secure_settings", "cluster:admin/persistent/completion", "cluster:admin/persistent/remove",