Fail node containing ancient closed index

Today we fail the node at startup if it contains an index that is too old to be
compatible with the current version, unless that index is closed. If the index
is closed then the node will start up and this puts us into a bad state: the
index cannot be opened and must be reindexed using an earlier version, but we
offer no way to get that index into a node running an earlier version so that
it can be reindexed. Downgrading the node in-place is decidedly unsupported and
cannot be expected to work since the node already started up and upgraded the
rest of its metadata. Since elastic#41731 we actively reject downgrades to versions ≥
v7.2.0 too.

This commit prevents the node from starting in the presence of any too-old
indices (closed or not). In particular, it does not write any upgraded metadata
in this situation, increasing the chances an in-place downgrade might be
successful. We still actively reject the downgrade using elastic#41731, because we
wrote the node metadata file before checking the index metadata, but at least
there is a way to override this check.

Relates elastic#21830, elastic#44230

Author: DaveCTurner

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -115,18 +115,15 @@ boolean isUpgraded(IndexMetaData indexMetaData) {
     }
 
     /**
-     * Elasticsearch v6.0 no longer supports indices created pre v5.0. All indices
-     * that were created before Elasticsearch v5.0 should be re-indexed in Elasticsearch 5.x
-     * before they can be opened by this version of elasticsearch.
+     * Elasticsearch does not support indices created before the previous major version. They must be reindexed using an earlier version
+     * before they can be opened here.
      */
     private void checkSupportedVersion(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
-        if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData,
-            minimumIndexCompatibilityVersion) == false) {
-            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created with version ["
+        if (isSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion) == false) {
+            throw new IllegalStateException("The index " + indexMetaData.getIndex() + " was created with version ["
                 + indexMetaData.getCreationVersion() + "] but the minimum compatible version is ["
-
-                + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch " + minimumIndexCompatibilityVersion.major
-                + ".x before upgrading to " + Version.CURRENT + ".");
+                + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch "
+                + minimumIndexCompatibilityVersion.major + ".x before upgrading to " + Version.CURRENT + ".");
         }
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -106,9 +107,9 @@ public void testFailUpgrade() {
             .build());
         String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(metaData,
             Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
-        assertEquals(message, "The index [[foo/BOOM]] was created with version [" + indexCreated + "] " +
+        assertThat(message, equalTo("The index [foo/" + metaData.getIndexUUID() + "] was created with version [" + indexCreated + "] " +
              "but the minimum compatible version is [" + minCompat + "]." +
-            " It should be re-indexed in Elasticsearch " + minCompat.major + ".x before upgrading to " + Version.CURRENT.toString() + ".");
+            " It should be re-indexed in Elasticsearch " + minCompat.major + ".x before upgrading to " + Version.CURRENT.toString() + "."));
 
         indexCreated = VersionUtils.randomVersionBetween(random(), minCompat, Version.CURRENT);
         indexUpgraded = VersionUtils.randomVersionBetween(random(), indexCreated, Version.CURRENT);
@@ -159,16 +160,25 @@ private MetaDataIndexUpgradeService getMetaDataIndexUpgradeService() {
             Collections.emptyList());
     }
 
-    public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
-        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_CREATION_DATE, 1)
-            .put(IndexMetaData.SETTING_INDEX_UUID, "BOOM")
-            .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT.minimumIndexCompatibilityVersion())
+    public IndexMetaData newIndexMeta(String name, Settings indexSettings) {
+        final Version createdVersion = VersionUtils.randomVersionBetween(random(),
+            Version.CURRENT.minimumIndexCompatibilityVersion(), VersionUtils.getPreviousVersion());
+        final Version upgradedVersion = VersionUtils.randomVersionBetween(random(), createdVersion, VersionUtils.getPreviousVersion());
+
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, createdVersion)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, between(0, 5))
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 5))
+            .put(IndexMetaData.SETTING_CREATION_DATE, randomNonNegativeLong())
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, upgradedVersion)
             .put(indexSettings)
             .build();
-        return IndexMetaData.builder(name).settings(build).build();
+        final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(name).settings(settings);
+        if (randomBoolean()) {
+            indexMetaDataBuilder.state(IndexMetaData.State.CLOSE);
+        }
+        return indexMetaDataBuilder.build();
     }
 
 }