[8.0] Prevent 6.8 indices upgrade to 8.0 (#82689)

build.gradle

@@ -125,9 +125,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/82689"
+String bwc_tests_disabled_issue = ""
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

docs/changelog/82689.yaml

@@ -0,0 +1,6 @@
+pr: 82689
+summary: Enhancement/prevent 6.8 indices upgrade
+area: Infra/Core
+type: enhancement
+issues:
+ - 81326

server/src/internalClusterTest/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java

@@ -32,8 +32,7 @@ public class RecoveryWithUnsupportedIndicesIT extends ESIntegTestCase {
     /**
      * Return settings that could be used to start a node that has the given zipped home directory.
      */
-    private Settings prepareBackwardsDataDir(Path backwardsIndex) throws IOException {
-        Path indexDir = createTempDir();
+    private Settings prepareBackwardsDataDir(Path indexDir, Path backwardsIndex) throws IOException {
         Path dataDir = indexDir.resolve("data");
         try (InputStream stream = Files.newInputStream(backwardsIndex)) {
             TestUtil.unzip(stream, indexDir);
@@ -73,8 +72,9 @@ private Settings prepareBackwardsDataDir(Path backwardsIndex) throws IOException
     public void testUpgradeStartClusterOn_2_4_5() throws Exception {
         String indexName = "unsupported-2.4.5";
 
+        Path indexDir = createTempDir();
         logger.info("Checking static index {}", indexName);
-        Settings nodeSettings = prepareBackwardsDataDir(getDataPath("/indices/bwc").resolve(indexName + ".zip"));
+        Settings nodeSettings = prepareBackwardsDataDir(indexDir, getDataPath("/indices/bwc").resolve(indexName + ".zip"));
         assertThat(
             ExceptionsHelper.unwrap(
                 expectThrows(Exception.class, () -> internalCluster().startNode(nodeSettings)),

server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java

@@ -608,7 +608,7 @@ public void testHalfDeletedIndexImport() throws Exception {
                     .putCustom(IndexGraveyard.TYPE, IndexGraveyard.builder().addTombstone(metadata.index("test").getIndex()).build())
                     .build()
             );
-            NodeMetadata.FORMAT.writeAndCleanup(new NodeMetadata(nodeId, Version.CURRENT), paths);
+            NodeMetadata.FORMAT.writeAndCleanup(new NodeMetadata(nodeId, Version.CURRENT, metadata.oldestIndexVersion()), paths);
         });
 
         ensureGreen();

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -215,6 +215,8 @@ default boolean isRestorable() {
     private SortedMap<String, IndexAbstraction> indicesLookup;
     private final Map<String, MappingMetadata> mappingsByHash;
 
+    private final Version oldestIndexVersion;
+
     private Metadata(
         String clusterUUID,
         boolean clusterUUIDCommitted,
@@ -236,7 +238,8 @@ private Metadata(
         String[] allClosedIndices,
         String[] visibleClosedIndices,
         SortedMap<String, IndexAbstraction> indicesLookup,
-        Map<String, MappingMetadata> mappingsByHash
+        Map<String, MappingMetadata> mappingsByHash,
+        Version oldestIndexVersion
     ) {
         this.clusterUUID = clusterUUID;
         this.clusterUUIDCommitted = clusterUUIDCommitted;
@@ -259,6 +262,7 @@ private Metadata(
         this.visibleClosedIndices = visibleClosedIndices;
         this.indicesLookup = indicesLookup;
         this.mappingsByHash = mappingsByHash;
+        this.oldestIndexVersion = oldestIndexVersion;
     }
 
     public Metadata withIncrementedVersion() {
@@ -283,7 +287,8 @@ public Metadata withIncrementedVersion() {
             allClosedIndices,
             visibleClosedIndices,
             indicesLookup,
-            mappingsByHash
+            mappingsByHash,
+            oldestIndexVersion
         );
     }
 
@@ -326,6 +331,10 @@ public CoordinationMetadata coordinationMetadata() {
         return this.coordinationMetadata;
     }
 
+    public Version oldestIndexVersion() {
+        return this.oldestIndexVersion;
+    }
+
     public boolean hasAlias(String alias) {
         IndexAbstraction indexAbstraction = getIndicesLookup().get(alias);
         if (indexAbstraction != null) {
@@ -1062,6 +1071,7 @@ public static Metadata readFrom(StreamInput in) throws IOException {
             Custom customIndexMetadata = in.readNamedWriteable(Custom.class);
             builder.putCustom(customIndexMetadata.getWriteableName(), customIndexMetadata);
         }
+
         return builder.build();
     }
 
@@ -1596,6 +1606,9 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
             final List<String> visibleClosedIndices = new ArrayList<>();
             final Set<String> allAliases = new HashSet<>();
             final ImmutableOpenMap<String, IndexMetadata> indicesMap = indices.build();
+
+            int oldestIndexVersionId = Version.CURRENT.id;
+
             for (IndexMetadata indexMetadata : indicesMap.values()) {
                 final String name = indexMetadata.getIndex().getName();
                 boolean added = allIndices.add(name);
@@ -1616,6 +1629,7 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
                     }
                 }
                 indexMetadata.getAliases().keysIt().forEachRemaining(allAliases::add);
+                oldestIndexVersionId = Math.min(oldestIndexVersionId, indexMetadata.getCreationVersion().id);
             }
 
             final ArrayList<String> duplicates = new ArrayList<>();
@@ -1738,7 +1752,8 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
                 allClosedIndicesArray,
                 visibleClosedIndicesArray,
                 indicesLookup,
-                Collections.unmodifiableMap(mappingsByHash)
+                Collections.unmodifiableMap(mappingsByHash),
+                Version.fromId(oldestIndexVersionId)
             );
         }
 
@@ -1888,6 +1903,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
                     builder.endObject();
                 }
             }
+
             builder.endObject();
         }
 

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.gateway.CorruptStateException;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.index.Index;
@@ -248,6 +249,7 @@ public void close() {
     /**
      * Setup the environment.
      * @param settings settings from elasticsearch.yml
+     * @param environment global environment
      */
     public NodeEnvironment(Settings settings, Environment environment) throws IOException {
         boolean success = false;
@@ -391,6 +393,9 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
         // move contents from legacy path to new path
         assert nodeLock.getNodePaths().length == legacyNodeLock.getNodePaths().length;
         try {
+            // first check if we are upgrading from an index compatible version
+            checkForIndexCompatibility(logger, legacyNodeLock.getNodePaths());
+
             final List<CheckedRunnable<IOException>> upgradeActions = new ArrayList<>();
             for (int i = 0; i < legacyNodeLock.getNodePaths().length; i++) {
                 final NodePath legacyNodePath = legacyNodeLock.getNodePaths()[i];
@@ -470,6 +475,46 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
         return true;
     }
 
+    /**
+     * Checks to see if we can upgrade to this version based on the existing index state. Upgrading
+     * from older versions can cause irreversible changes if allowed.
+     * @param logger
+     * @param nodePaths
+     * @throws IOException
+     */
+    static void checkForIndexCompatibility(Logger logger, NodePath... nodePaths) throws IOException {
+        final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
+        NodeMetadata metadata = PersistedClusterStateService.nodeMetadata(paths);
+
+        // We are upgrading the cluster, but we didn't find any previous metadata. Corrupted state or incompatible version.
+        if (metadata == null) {
+            throw new CorruptStateException(
+                "Format version is not supported. Upgrading to ["
+                    + Version.CURRENT
+                    + "] is only supported from version ["
+                    + Version.CURRENT.minimumCompatibilityVersion()
+                    + "]."
+            );
+        }
+
+        metadata.verifyUpgradeToCurrentVersion();
+
+        logger.info("oldest index version recorded in NodeMetadata {}", metadata.oldestIndexVersion());
+
+        if (metadata.oldestIndexVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion())) {
+            throw new IllegalStateException(
+                "cannot upgrade node because incompatible indices created with version ["
+                    + metadata.oldestIndexVersion()
+                    + "] exist, while the minimum compatible index version is ["
+                    + Version.CURRENT.minimumIndexCompatibilityVersion()
+                    + "]. "
+                    + "Upgrade your older indices by reindexing them in version ["
+                    + Version.CURRENT.minimumCompatibilityVersion()
+                    + "] first."
+            );
+        }
+    }
+
     private void maybeLogPathDetails() throws IOException {
 
         // We do some I/O in here, so skip this if DEBUG/INFO are not enabled:
@@ -551,12 +596,15 @@ private static NodeMetadata loadNodeMetadata(Settings settings, Logger logger, N
             final NodeMetadata legacyMetadata = NodeMetadata.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths);
             if (legacyMetadata == null) {
                 assert nodeIds.isEmpty() : nodeIds;
-                metadata = new NodeMetadata(generateNodeId(settings), Version.CURRENT);
+                // If we couldn't find legacy metadata, we set the latest index version to this version. This happens
+                // when we are starting a new node and there are no indices to worry about.
+                metadata = new NodeMetadata(generateNodeId(settings), Version.CURRENT, Version.CURRENT);
             } else {
                 assert nodeIds.equals(Collections.singleton(legacyMetadata.nodeId())) : nodeIds + " doesn't match " + legacyMetadata;
                 metadata = legacyMetadata;
             }
         }
+
         metadata = metadata.upgradeToCurrentVersion();
         assert metadata.nodeVersion().equals(Version.CURRENT) : metadata.nodeVersion() + " != " + Version.CURRENT;
 

server/src/main/java/org/elasticsearch/env/NodeMetadata.java

@@ -28,21 +28,30 @@ public final class NodeMetadata {
 
     static final String NODE_ID_KEY = "node_id";
     static final String NODE_VERSION_KEY = "node_version";
+    static final String OLDEST_INDEX_VERSION_KEY = "oldest_index_version";
 
     private final String nodeId;
 
     private final Version nodeVersion;
 
     private final Version previousNodeVersion;
 
-    private NodeMetadata(final String nodeId, final Version nodeVersion, final Version previousNodeVersion) {
+    private final Version oldestIndexVersion;
+
+    private NodeMetadata(
+        final String nodeId,
+        final Version nodeVersion,
+        final Version previousNodeVersion,
+        final Version oldestIndexVersion
+    ) {
         this.nodeId = Objects.requireNonNull(nodeId);
         this.nodeVersion = Objects.requireNonNull(nodeVersion);
         this.previousNodeVersion = Objects.requireNonNull(previousNodeVersion);
+        this.oldestIndexVersion = Objects.requireNonNull(oldestIndexVersion);
     }
 
-    public NodeMetadata(final String nodeId, final Version nodeVersion) {
-        this(nodeId, nodeVersion, nodeVersion);
+    public NodeMetadata(final String nodeId, final Version nodeVersion, final Version oldestIndexVersion) {
+        this(nodeId, nodeVersion, nodeVersion, oldestIndexVersion);
     }
 
     @Override
@@ -52,12 +61,13 @@ public boolean equals(Object o) {
         NodeMetadata that = (NodeMetadata) o;
         return nodeId.equals(that.nodeId)
             && nodeVersion.equals(that.nodeVersion)
+            && oldestIndexVersion.equals(that.oldestIndexVersion)
             && Objects.equals(previousNodeVersion, that.previousNodeVersion);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(nodeId, nodeVersion, previousNodeVersion);
+        return Objects.hash(nodeId, nodeVersion, previousNodeVersion, oldestIndexVersion);
     }
 
     @Override
@@ -70,6 +80,8 @@ public String toString() {
             + nodeVersion
             + ", previousNodeVersion="
             + previousNodeVersion
+            + ", oldestIndexVersion="
+            + oldestIndexVersion
             + '}';
     }
 
@@ -91,7 +103,11 @@ public Version previousNodeVersion() {
         return previousNodeVersion;
     }
 
-    public NodeMetadata upgradeToCurrentVersion() {
+    public Version oldestIndexVersion() {
+        return oldestIndexVersion;
+    }
+
+    public void verifyUpgradeToCurrentVersion() {
         assert (nodeVersion.equals(Version.V_EMPTY) == false) || (Version.CURRENT.major <= Version.V_7_0_0.major + 1)
             : "version is required in the node metadata from v9 onwards";
 
@@ -113,14 +129,19 @@ public NodeMetadata upgradeToCurrentVersion() {
                 "cannot downgrade a node from version [" + nodeVersion + "] to version [" + Version.CURRENT + "]"
             );
         }
+    }
 
-        return nodeVersion.equals(Version.CURRENT) ? this : new NodeMetadata(nodeId, Version.CURRENT, nodeVersion);
+    public NodeMetadata upgradeToCurrentVersion() {
+        verifyUpgradeToCurrentVersion();
+
+        return nodeVersion.equals(Version.CURRENT) ? this : new NodeMetadata(nodeId, Version.CURRENT, nodeVersion, oldestIndexVersion);
     }
 
     private static class Builder {
         String nodeId;
         Version nodeVersion;
         Version previousNodeVersion;
+        Version oldestIndexVersion;
 
         public void setNodeId(String nodeId) {
             this.nodeId = nodeId;
@@ -134,8 +155,13 @@ public void setPreviousNodeVersionId(int previousNodeVersionId) {
             this.previousNodeVersion = Version.fromId(previousNodeVersionId);
         }
 
+        public void setOldestIndexVersion(int oldestIndexVersion) {
+            this.oldestIndexVersion = Version.fromId(oldestIndexVersion);
+        }
+
         public NodeMetadata build() {
             final Version nodeVersion;
+            final Version oldestIndexVersion;
             if (this.nodeVersion == null) {
                 assert Version.CURRENT.major <= Version.V_7_0_0.major + 1 : "version is required in the node metadata from v9 onwards";
                 nodeVersion = Version.V_EMPTY;
@@ -145,8 +171,13 @@ public NodeMetadata build() {
             if (this.previousNodeVersion == null) {
                 previousNodeVersion = nodeVersion;
             }
+            if (this.oldestIndexVersion == null) {
+                oldestIndexVersion = Version.V_EMPTY;
+            } else {
+                oldestIndexVersion = this.oldestIndexVersion;
+            }
 
-            return new NodeMetadata(nodeId, nodeVersion, previousNodeVersion);
+            return new NodeMetadata(nodeId, nodeVersion, previousNodeVersion, oldestIndexVersion);
         }
     }
 
@@ -163,6 +194,7 @@ static class NodeMetadataStateFormat extends MetadataStateFormat<NodeMetadata> {
             objectParser = new ObjectParser<>("node_meta_data", ignoreUnknownFields, Builder::new);
             objectParser.declareString(Builder::setNodeId, new ParseField(NODE_ID_KEY));
             objectParser.declareInt(Builder::setNodeVersionId, new ParseField(NODE_VERSION_KEY));
+            objectParser.declareInt(Builder::setOldestIndexVersion, new ParseField(OLDEST_INDEX_VERSION_KEY));
         }
 
         @Override
@@ -176,6 +208,7 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str
         public void toXContent(XContentBuilder builder, NodeMetadata nodeMetadata) throws IOException {
             builder.field(NODE_ID_KEY, nodeMetadata.nodeId);
             builder.field(NODE_VERSION_KEY, nodeMetadata.nodeVersion.id);
+            builder.field(OLDEST_INDEX_VERSION_KEY, nodeMetadata.oldestIndexVersion.id);
         }
 
         @Override