Add History UUID Index Setting (#56930) (#57104)

original-brownbear · web-flow · commit 9fa60f7367b2 · 2020-05-25T11:26:03.000+02:00
Pre-requesite for #50278 to be able to uniquely identify index metadata by its version fields and UUIDs when restoring into closed indices.
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -47,6 +48,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
@@ -293,6 +295,8 @@ public void test3MasterNodes2Failed() throws Exception {
 
         logger.info("--> ensure index test is green");
         ensureGreen("test");
+        IndexMetadata indexMetadata = clusterService().state().metadata().index("test");
+        assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
 
         logger.info("--> detach-cluster on 2nd and 3rd master-eligible nodes");
         Environment environmentMaster2 = TestEnvironment.newEnvironment(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices.recovery;
 
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -31,6 +32,7 @@
 import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 @ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST)
 public class DanglingIndicesIT extends ESIntegTestCase {
@@ -89,6 +91,8 @@ public Settings onNodeStopped(String nodeName) throws Exception {
                 equalTo("42s"));
         }
         ensureGreen(INDEX_NAME);
+        final IndexMetadata indexMetadata = clusterService().state().metadata().index(INDEX_NAME);
+        assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
     }
 
     /**
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -3597,6 +3597,7 @@ public void testRestoreIncreasesPrimaryTerms() {
 
         final IndexMetadata indexMetadata = client().admin().cluster().prepareState().clear().setIndices(indexName)
             .setMetadata(true).get().getState().metadata().index(indexName);
+        assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), nullValue());
         final int numPrimaries = getNumShards(indexName).numPrimaries;
         final Map<Integer, Long> primaryTerms = IntStream.range(0, numPrimaries)
             .boxed().collect(Collectors.toMap(shardId -> shardId, indexMetadata::primaryTerm));
@@ -3619,6 +3620,7 @@ public void testRestoreIncreasesPrimaryTerms() {
         for (int shardId = 0; shardId < numPrimaries; shardId++) {
             assertThat(restoredIndexMetadata.primaryTerm(shardId), greaterThan(primaryTerms.get(shardId)));
         }
+        assertThat(restoredIndexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());
     }
 
     public void testSnapshotDifferentIndicesBySameName() {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -121,8 +121,10 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi
         if (metadata == null) {
             throw new IndexNotFoundException(sourceIndexName);
         }
-        final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings())
-            .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
+        final Settings.Builder targetIndexSettingsBuilder = Settings.builder().put(targetIndex.settings())
+            .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
+        targetIndexSettingsBuilder.remove(IndexMetadata.SETTING_HISTORY_UUID);
+        final Settings targetIndexSettings = targetIndexSettingsBuilder.build();
         final int numShards;
         if (IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
             numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings);
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
@@ -18,12 +18,15 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import joptsimple.OptionSet;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -107,13 +110,18 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths, int nodeLoc
             .put(metadata.persistentSettings())
             .put(UNSAFE_BOOTSTRAP.getKey(), true)
             .build();
-        Metadata newMetadata = Metadata.builder(metadata)
+        Metadata.Builder newMetadata = Metadata.builder(metadata)
             .clusterUUID(Metadata.UNKNOWN_CLUSTER_UUID)
             .generateClusterUuidIfNeeded()
             .clusterUUIDCommitted(true)
             .persistentSettings(persistentSettings)
-            .coordinationMetadata(newCoordinationMetadata)
-            .build();
+            .coordinationMetadata(newCoordinationMetadata);
+        for (ObjectCursor<IndexMetadata> idx : metadata.indices().values()) {
+            IndexMetadata indexMetadata = idx.value;
+            newMetadata.put(IndexMetadata.builder(indexMetadata).settings(
+                Settings.builder().put(indexMetadata.getSettings())
+                    .put(IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID())));
+        }
 
         final ClusterState newClusterState = ClusterState.builder(oldClusterState)
             .metadata(newMetadata).build();
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -227,6 +227,7 @@ public Iterator<Setting<?>> settings() {
         Setting.intSetting("index.priority", 1, 0, Property.Dynamic, Property.IndexScope);
     public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
     public static final String SETTING_INDEX_UUID = "index.uuid";
+    public static final String SETTING_HISTORY_UUID = "index.history.uuid";
     public static final String SETTING_DATA_PATH = "index.data_path";
     public static final Setting<String> INDEX_DATA_PATH_SETTING =
         new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -214,6 +214,7 @@ public boolean isPrivateSetting(String key) {
         switch (key) {
             case IndexMetadata.SETTING_CREATION_DATE:
             case IndexMetadata.SETTING_INDEX_UUID:
+            case IndexMetadata.SETTING_HISTORY_UUID:
             case IndexMetadata.SETTING_VERSION_UPGRADED:
             case IndexMetadata.SETTING_INDEX_PROVIDED_NAME:
             case MergePolicyConfig.INDEX_MERGE_ENABLED:
diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -35,9 +35,11 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -142,6 +144,9 @@ public ClusterState execute(ClusterState currentState) {
                             // with the current version and upgrade it if needed.
                             upgradedIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(indexMetadata,
                                 minIndexCompatibilityVersion);
+                            upgradedIndexMetadata = IndexMetadata.builder(upgradedIndexMetadata).settings(
+                                Settings.builder().put(upgradedIndexMetadata.getSettings()).put(
+                                    IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID())).build();
                         } catch (Exception ex) {
                             // upgrade failed - adding index as closed
                             logger.warn(() -> new ParameterizedMessage("found dangled index [{}] on node [{}]. This index cannot be " +
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -708,6 +708,11 @@ public synchronized boolean updateIndexMetadata(IndexMetadata indexMetadata) {
         if (newUUID.equals(getUUID()) == false) {
             throw new IllegalArgumentException("uuid mismatch on settings update expected: " + getUUID() + " but was: " + newUUID);
         }
+        final String newRestoreUUID = newSettings.get(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE);
+        final String restoreUUID = this.settings.get(IndexMetadata.SETTING_HISTORY_UUID, IndexMetadata.INDEX_UUID_NA_VALUE);
+        if (newRestoreUUID.equals(restoreUUID) == false) {
+            throw new IllegalArgumentException("uuid mismatch on settings update expected: " + restoreUUID + " but was: " + newRestoreUUID);
+        }
         this.indexMetadata = indexMetadata;
         final Settings newIndexSettings = Settings.builder().put(nodeSettings).put(newSettings).build();
         if (same(this.settings, newIndexSettings)) {
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -91,6 +91,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
@@ -125,7 +126,8 @@ public class RestoreService implements ClusterStateApplier {
             SETTING_VERSION_CREATED,
             SETTING_INDEX_UUID,
             SETTING_CREATION_DATE,
-            IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey()));
+            IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
+            SETTING_HISTORY_UUID));
 
     // It's OK to change some settings, but we shouldn't allow simply removing them
     private static final Set<String> UNREMOVABLE_SETTINGS;
@@ -342,8 +344,8 @@ public ClusterState execute(ClusterState currentState) {
                                     }
                                     indexMdBuilder.settings(Settings.builder()
                                         .put(snapshotIndexMetadata.getSettings())
-                                        .put(IndexMetadata.SETTING_INDEX_UUID,
-                                            currentIndexMetadata.getIndexUUID()));
+                                        .put(IndexMetadata.SETTING_INDEX_UUID, currentIndexMetadata.getIndexUUID())
+                                        .put(IndexMetadata.SETTING_HISTORY_UUID, UUIDs.randomBase64UUID()));
                                     IndexMetadata updatedIndexMetadata = indexMdBuilder.index(renamedIndexName).build();
                                     rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource);
                                     blocks.updateBlocks(updatedIndexMetadata);
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
@@ -456,6 +456,7 @@ static Settings filter(Settings originalSettings) {
         settings.remove(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey());
         settings.remove(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey());
         settings.remove(IndexMetadata.SETTING_INDEX_UUID);
+        settings.remove(IndexMetadata.SETTING_HISTORY_UUID);
         settings.remove(IndexMetadata.SETTING_INDEX_PROVIDED_NAME);
         settings.remove(IndexMetadata.SETTING_CREATION_DATE);
 

Original file line number	Diff line number	Diff line change
`@@ -3597,6 +3597,7 @@ public void testRestoreIncreasesPrimaryTerms() {`
`3597`	`3597`
`3598`	`3598`	`final IndexMetadata indexMetadata = client().admin().cluster().prepareState().clear().setIndices(indexName)`
`3599`	`3599`	`.setMetadata(true).get().getState().metadata().index(indexName);`
	`3600`	`+ assertThat(indexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), nullValue());`
`3600`	`3601`	`final int numPrimaries = getNumShards(indexName).numPrimaries;`
`3601`	`3602`	`final Map<Integer, Long> primaryTerms = IntStream.range(0, numPrimaries)`
`3602`	`3603`	`.boxed().collect(Collectors.toMap(shardId -> shardId, indexMetadata::primaryTerm));`
`@@ -3619,6 +3620,7 @@ public void testRestoreIncreasesPrimaryTerms() {`
`3619`	`3620`	`for (int shardId = 0; shardId < numPrimaries; shardId++) {`
`3620`	`3621`	`assertThat(restoredIndexMetadata.primaryTerm(shardId), greaterThan(primaryTerms.get(shardId)));`
`3621`	`3622`	`}`
	`3623`	`+ assertThat(restoredIndexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue());`
`3622`	`3624`	`}`
`3623`	`3625`
`3624`	`3626`	`public void testSnapshotDifferentIndicesBySameName() {`