Prevent downgrades from 8.x to 7.x (#78586)

Users sometimes attempt to downgrade a node in place, but downgrades are
totally untested and unsupported and generally don't work. We protect
against this by recording the node version in the metadata and refusing
to start if we encounter metadata written by a future version.

However in 8.0 (#42489) we changed the directory layout so that a 7.x
node won't find the upgraded metadata, or indeed any other data, and
will proceed as if it's a fresh node. That's almost certainly not what
the user wants, so with this commit we create a file at
`${path.data}/nodes` at each startup, preventing an older node from
starting.

Closes #52414

Author: DaveCTurner

server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java

@@ -10,10 +10,10 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -138,8 +138,11 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         // simulate older data path layout by moving data under "nodes/0" folder
         final List<Path> dataPaths = List.of(PathUtils.get(Environment.PATH_DATA_SETTING.get(dataPathSettings)));
         dataPaths.forEach(path -> {
-                final Path targetPath = path.resolve("nodes").resolve("0");
+                final Path nodesPath = path.resolve("nodes");
+                final Path targetPath = nodesPath.resolve("0");
                 try {
+                    assertTrue(Files.isRegularFile(nodesPath));
+                    Files.delete(nodesPath);
                     Files.createDirectories(targetPath);
 
                     try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@@ -192,9 +195,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         }
 
         // check that upgrade works
-        dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes"))));
         internalCluster().startNode(dataPathSettings);
-        dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes"))));
         assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId());
         assertTrue(indexExists("test"));
         ensureYellow("test");

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -25,21 +25,21 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.core.Releasable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.gateway.PersistedClusterStateService;
@@ -55,6 +55,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
@@ -269,6 +270,16 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                 assertCanWrite();
             }
 
+            // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades
+            final Path legacyNodesPath = environment.dataFile().resolve("nodes");
+            if (Files.isRegularFile(legacyNodesPath) == false) {
+                final String content = "written by Elasticsearch v" + Version.CURRENT +
+                    " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss";
+                Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8));
+                IOUtils.fsync(legacyNodesPath, false);
+                IOUtils.fsync(environment.dataFile(), true);
+            }
+
             if (DiscoveryNode.canContainData(settings) == false) {
                 if (DiscoveryNode.isMasterNode(settings) == false) {
                     ensureNoIndexMetadata(nodePath);
@@ -408,12 +419,11 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
                 IOUtils.fsync(nodePath.path, true);
             });
 
-            // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an
-            // intermediate state would be pretty disastrous
-            loadNodeMetadata(settings, logger, legacyNodeLock.getNodePath());
+            // now do the actual upgrade
             for (CheckedRunnable<IOException> upgradeAction : upgradeActions) {
                 upgradeAction.run();
             }
+
         } finally {
             legacyNodeLock.close();
         }

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.test.NodeRoles;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -41,6 +42,7 @@
 import static org.elasticsearch.test.NodeRoles.nonDataNode;
 import static org.elasticsearch.test.NodeRoles.nonMasterNode;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.startsWith;
@@ -439,6 +441,23 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException {
         verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName);
     }
 
+    public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException {
+        final Settings settings = buildEnvSettings(Settings.EMPTY);
+        for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing
+            try (NodeEnvironment env = newNodeEnvironment(settings)) {
+                final Path nodesPath = env.nodeDataPath().resolve("nodes");
+                assertTrue(Files.isRegularFile(nodesPath));
+                assertThat(
+                    Files.readString(nodesPath, StandardCharsets.UTF_8),
+                    allOf(
+                        containsString("written by Elasticsearch"),
+                        containsString("prevent a downgrade"),
+                        containsString("data loss")));
+                Files.delete(nodesPath);
+            }
+        }
+    }
+
     private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) {
         IllegalStateException ex = expectThrows(IllegalStateException.class,
             "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role",
@@ -459,17 +478,6 @@ private void verifyFailsOnMetadata(Settings settings, Path indexPath) {
         assertThat(ex.getMessage(), startsWith("node does not have the data and master roles but has index metadata"));
     }
 
-    /**
-     * Converts an array of Strings to an array of Paths, adding an additional child if specified
-     */
-    private Path[] stringsToPaths(String[] strings, String additional) {
-        Path[] locations = new Path[strings.length];
-        for (int i = 0; i < strings.length; i++) {
-            locations[i] = PathUtils.get(strings[i], additional);
-        }
-        return locations;
-    }
-
     @Override
     public NodeEnvironment newNodeEnvironment() throws IOException {
         return newNodeEnvironment(Settings.EMPTY);

test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java

@@ -26,6 +26,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
@@ -148,6 +149,10 @@ public long getUnallocatedSpace() throws IOException {
 
         private static long getTotalFileSize(Path path) throws IOException {
             if (Files.isRegularFile(path)) {
+                if (path.getFileName().toString().equals("nodes")
+                    && Files.readString(path, StandardCharsets.UTF_8).contains("prevent a downgrade")) {
+                    return 0;
+                }
                 try {
                     return Files.size(path);
                 } catch (NoSuchFileException | FileNotFoundException e) {