Skip to content

Commit 8033c57

Browse files
jasontedorjasontedor
authored
Detect remnants of path.data/default.path.data bug
In Elasticsearch 5.3.0 a bug was introduced in the merging of default settings when the target setting existed as an array. When this bug concerns path.data and default.path.data, we ended up in a situation where the paths specified in both settings would be used to write index data. Since our packaging sets default.path.data, users that configure multiple data paths via an array and use the packaging are subject to having shards land in paths in default.path.data when that is very likely not what they intended. This commit is an attempt to rectify this situation. If path.data and default.path.data are configured, we check for the presence of indices there. If we find any, we log messages explaining the situation and fail the node. Relates #24099
1 parent a8be0a5 commit 8033c57

File tree

4 files changed

+213
-17
lines changed

4 files changed

+213
-17
lines changed

‎core/src/main/java/org/elasticsearch/bootstrap/Security.java‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.elasticsearch.bootstrap;
2121

2222
import org.elasticsearch.SecureSM;
23-
import org.elasticsearch.common.Strings;
2423
import org.elasticsearch.common.SuppressForbidden;
2524
import org.elasticsearch.common.io.PathUtils;
2625
import org.elasticsearch.common.network.NetworkModule;
@@ -45,11 +44,9 @@
4544
import java.security.Permissions;
4645
import java.security.Policy;
4746
import java.security.URIParameter;
48-
import java.util.ArrayList;
4947
import java.util.Collections;
5048
import java.util.HashMap;
5149
import java.util.LinkedHashSet;
52-
import java.util.List;
5350
import java.util.Map;
5451
import java.util.Set;
5552

@@ -269,6 +266,26 @@ static void addFilePermissions(Permissions policy, Environment environment) {
269266
for (Path path : environment.dataFiles()) {
270267
addPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
271268
}
269+
/*
270+
* If path.data and default.path.data are set, we need read access to the paths in default.path.data to check for the existence of
271+
* index directories there that could have arisen from a bug in the handling of simultaneous configuration of path.data and
272+
* default.path.data that was introduced in Elasticsearch 5.3.0.
273+
*
274+
* If path.data is not set then default.path.data would take precedence in setting the data paths for the environment and
275+
* permissions would have been granted above.
276+
*
277+
* If path.data is not set and default.path.data is not set, then we would fallback to the default data directory under
278+
* Elasticsearch home and again permissions would have been granted above.
279+
*
280+
* If path.data is set and default.path.data is not set, there is nothing to do here.
281+
*/
282+
if (Environment.PATH_DATA_SETTING.exists(environment.settings())
283+
&& Environment.DEFAULT_PATH_DATA_SETTING.exists(environment.settings())) {
284+
for (final String path : Environment.DEFAULT_PATH_DATA_SETTING.get(environment.settings())) {
285+
// write permissions are not needed here, we are not going to be writing to any paths here
286+
addPath(policy, Environment.DEFAULT_PATH_DATA_SETTING.getKey(), getPath(path), "read,readlink");
287+
}
288+
}
272289
for (Path path : environment.repoFiles()) {
273290
addPath(policy, Environment.PATH_REPO_SETTING.getKey(), path, "read,readlink,write,delete");
274291
}
@@ -278,6 +295,11 @@ static void addFilePermissions(Permissions policy, Environment environment) {
278295
}
279296
}
280297

298+
@SuppressForbidden(reason = "read path that is not configured in environment")
299+
private static Path getPath(final String path) {
300+
return PathUtils.get(path);
301+
}
302+
281303
/**
282304
* Add dynamic {@link SocketPermission}s based on HTTP and transport settings.
283305
*

‎core/src/main/java/org/elasticsearch/env/NodeEnvironment.java‎

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
202202
for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
203203
Path dataDirWithClusterName = environment.dataWithClusterFiles()[dirIndex];
204204
Path dataDir = environment.dataFiles()[dirIndex];
205-
Path dir = dataDir.resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
205+
Path dir = resolveNodePath(dataDir, possibleLockId);
206206
Files.createDirectories(dir);
207207

208208
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
@@ -268,6 +268,17 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
268268
}
269269
}
270270

271+
/**
272+
* Resolve a specific nodes/{node.id} path for the specified path and node lock id.
273+
*
274+
* @param path the path
275+
* @param nodeLockId the node lock id
276+
* @return the resolved path
277+
*/
278+
public static Path resolveNodePath(final Path path, final int nodeLockId) {
279+
return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
280+
}
281+
271282
/** Returns true if the directory is empty */
272283
private static boolean dirEmpty(final Path path) throws IOException {
273284
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@@ -724,6 +735,14 @@ public NodePath[] nodePaths() {
724735
return nodePaths;
725736
}
726737

738+
public int getNodeLockId() {
739+
assertEnvIsLocked();
740+
if (nodePaths == null || locks == null) {
741+
throw new IllegalStateException("node is not configured to store local location");
742+
}
743+
return nodeLockId;
744+
}
745+
727746
/**
728747
* Returns all index paths.
729748
*/
@@ -736,6 +755,8 @@ public Path[] indexPaths(Index index) {
736755
return indexPaths;
737756
}
738757

758+
759+
739760
/**
740761
* Returns all shard paths excluding custom shard path. Note: Shards are only allocated on one of the
741762
* returned paths. The returned array may contain paths to non-existing directories.
@@ -764,19 +785,36 @@ public Set<String> availableIndexFolders() throws IOException {
764785
assertEnvIsLocked();
765786
Set<String> indexFolders = new HashSet<>();
766787
for (NodePath nodePath : nodePaths) {
767-
Path indicesLocation = nodePath.indicesPath;
768-
if (Files.isDirectory(indicesLocation)) {
769-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
770-
for (Path index : stream) {
771-
if (Files.isDirectory(index)) {
772-
indexFolders.add(index.getFileName().toString());
773-
}
788+
indexFolders.addAll(availableIndexFoldersForPath(nodePath));
789+
}
790+
return indexFolders;
791+
792+
}
793+
794+
/**
795+
* Return all directory names in the nodes/{node.id}/indices directory for the given node path.
796+
*
797+
* @param nodePath the path
798+
* @return all directories that could be indices for the given node path.
799+
* @throws IOException if an I/O exception occurs traversing the filesystem
800+
*/
801+
public Set<String> availableIndexFoldersForPath(final NodePath nodePath) throws IOException {
802+
if (nodePaths == null || locks == null) {
803+
throw new IllegalStateException("node is not configured to store local location");
804+
}
805+
assertEnvIsLocked();
806+
final Set<String> indexFolders = new HashSet<>();
807+
Path indicesLocation = nodePath.indicesPath;
808+
if (Files.isDirectory(indicesLocation)) {
809+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
810+
for (Path index : stream) {
811+
if (Files.isDirectory(index)) {
812+
indexFolders.add(index.getFileName().toString());
774813
}
775814
}
776815
}
777816
}
778817
return indexFolders;
779-
780818
}
781819

782820
/**

‎core/src/main/java/org/elasticsearch/node/Node.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.elasticsearch.cluster.routing.allocation.AllocationService;
5151
import org.elasticsearch.cluster.service.ClusterService;
5252
import org.elasticsearch.common.StopWatch;
53+
import org.elasticsearch.common.SuppressForbidden;
5354
import org.elasticsearch.common.component.Lifecycle;
5455
import org.elasticsearch.common.component.LifecycleComponent;
5556
import org.elasticsearch.common.inject.Binder;
@@ -58,6 +59,7 @@
5859
import org.elasticsearch.common.inject.Module;
5960
import org.elasticsearch.common.inject.ModulesBuilder;
6061
import org.elasticsearch.common.inject.util.Providers;
62+
import org.elasticsearch.common.io.PathUtils;
6163
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
6264
import org.elasticsearch.common.lease.Releasables;
6365
import org.elasticsearch.common.logging.DeprecationLogger;
@@ -146,7 +148,9 @@
146148
import java.util.Collection;
147149
import java.util.Collections;
148150
import java.util.List;
151+
import java.util.Locale;
149152
import java.util.Map;
153+
import java.util.Set;
150154
import java.util.concurrent.CountDownLatch;
151155
import java.util.concurrent.TimeUnit;
152156
import java.util.function.Consumer;
@@ -262,6 +266,9 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
262266
Logger logger = Loggers.getLogger(Node.class, tmpSettings);
263267
final String nodeId = nodeEnvironment.nodeId();
264268
tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeId);
269+
if (DiscoveryNode.nodeRequiresLocalStorage(tmpSettings)) {
270+
checkForIndexDataInDefaultPathData(tmpSettings, nodeEnvironment, logger);
271+
}
265272
// this must be captured after the node name is possibly added to the settings
266273
final String nodeName = NODE_NAME_SETTING.get(tmpSettings);
267274
if (hadPredefinedNodeName == false) {
@@ -500,6 +507,58 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
500507
}
501508
}
502509

510+
/**
511+
* Checks for path.data and default.path.data being configured, and there being index data in any of the paths in default.path.data.
512+
*
513+
* @param settings the settings to check for path.data and default.path.data
514+
* @param nodeEnv the current node environment
515+
* @param logger a logger where messages regarding the detection will be logged
516+
* @throws IOException if an I/O exception occurs reading the directory structure
517+
*/
518+
static void checkForIndexDataInDefaultPathData(
519+
final Settings settings, final NodeEnvironment nodeEnv, final Logger logger) throws IOException {
520+
if (!Environment.PATH_DATA_SETTING.exists(settings) || !Environment.DEFAULT_PATH_DATA_SETTING.exists(settings)) {
521+
return;
522+
}
523+
524+
boolean clean = true;
525+
for (final String defaultPathData : Environment.DEFAULT_PATH_DATA_SETTING.get(settings)) {
526+
final Path nodeDirectory = NodeEnvironment.resolveNodePath(getPath(defaultPathData), nodeEnv.getNodeLockId());
527+
if (Files.exists(nodeDirectory) == false) {
528+
continue;
529+
}
530+
final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(nodeDirectory);
531+
final Set<String> availableIndexFolders = nodeEnv.availableIndexFoldersForPath(nodePath);
532+
if (availableIndexFolders.isEmpty()) {
533+
continue;
534+
}
535+
clean = false;
536+
logger.error("detected index data in default.path.data [{}] where there should not be any", nodePath.indicesPath);
537+
for (final String availableIndexFolder : availableIndexFolders) {
538+
logger.info(
539+
"index folder [{}] in default.path.data [{}] must be moved to any of {}",
540+
availableIndexFolder,
541+
nodePath.indicesPath,
542+
Arrays.stream(nodeEnv.nodePaths()).map(np -> np.indicesPath).collect(Collectors.toList()));
543+
}
544+
}
545+
546+
if (clean) {
547+
return;
548+
}
549+
550+
final String message = String.format(
551+
Locale.ROOT,
552+
"detected index data in default.path.data %s where there should not be any; check the logs for details",
553+
Environment.DEFAULT_PATH_DATA_SETTING.get(settings));
554+
throw new IllegalStateException(message);
555+
}
556+
557+
@SuppressForbidden(reason = "read path that is not configured in environment")
558+
private static Path getPath(final String path) {
559+
return PathUtils.get(path);
560+
}
561+
503562
// visible for testing
504563
static void warnIfPreRelease(final Version version, final boolean isSnapshot, final Logger logger) {
505564
if (!version.isRelease() || isSnapshot) {

‎test/framework/src/main/java/org/elasticsearch/node/NodeTests.java‎

Lines changed: 82 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,31 +19,41 @@
1919
package org.elasticsearch.node;
2020

2121
import org.apache.logging.log4j.Logger;
22+
import org.apache.lucene.util.LuceneTestCase;
2223
import org.elasticsearch.Version;
2324
import org.elasticsearch.bootstrap.BootstrapCheck;
2425
import org.elasticsearch.cluster.ClusterName;
26+
import org.elasticsearch.common.UUIDs;
2527
import org.elasticsearch.common.network.NetworkModule;
2628
import org.elasticsearch.common.settings.Settings;
2729
import org.elasticsearch.common.transport.BoundTransportAddress;
2830
import org.elasticsearch.env.Environment;
31+
import org.elasticsearch.env.NodeEnvironment;
2932
import org.elasticsearch.plugins.Plugin;
3033
import org.elasticsearch.test.ESTestCase;
3134
import org.elasticsearch.test.InternalTestCluster;
3235
import org.elasticsearch.transport.MockTcpTransportPlugin;
3336

3437
import java.io.IOException;
38+
import java.nio.file.Files;
3539
import java.nio.file.Path;
3640
import java.util.Arrays;
3741
import java.util.Collections;
3842
import java.util.List;
43+
import java.util.Locale;
3944
import java.util.concurrent.atomic.AtomicBoolean;
45+
import java.util.stream.Collectors;
46+
import java.util.stream.IntStream;
4047

48+
import static org.hamcrest.Matchers.containsString;
4149
import static org.hamcrest.Matchers.equalTo;
50+
import static org.hamcrest.Matchers.hasToString;
4251
import static org.mockito.Mockito.mock;
4352
import static org.mockito.Mockito.reset;
4453
import static org.mockito.Mockito.verify;
4554
import static org.mockito.Mockito.verifyNoMoreInteractions;
4655

56+
@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS")
4757
public class NodeTests extends ESTestCase {
4858

4959
public void testNodeName() throws IOException {
@@ -165,14 +175,81 @@ public void testNodeAttributes() throws IOException {
165175
}
166176
}
167177

178+
public void testDefaultPathDataSet() throws IOException {
179+
final Path zero = createTempDir().toAbsolutePath();
180+
final Path one = createTempDir().toAbsolutePath();
181+
final Path defaultPathData = createTempDir().toAbsolutePath();
182+
final Settings settings = Settings.builder()
183+
.put("path.home", "/home")
184+
.put("path.data.0", zero)
185+
.put("path.data.1", one)
186+
.put("default.path.data", defaultPathData)
187+
.build();
188+
try (NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings))) {
189+
final Path defaultPathDataWithNodesAndId = defaultPathData.resolve("nodes/0");
190+
Files.createDirectories(defaultPathDataWithNodesAndId);
191+
final NodeEnvironment.NodePath defaultNodePath = new NodeEnvironment.NodePath(defaultPathDataWithNodesAndId);
192+
final boolean indexExists = randomBoolean();
193+
final List<String> indices;
194+
if (indexExists) {
195+
indices = IntStream.range(0, randomIntBetween(1, 3)).mapToObj(i -> UUIDs.randomBase64UUID()).collect(Collectors.toList());
196+
for (final String index : indices) {
197+
Files.createDirectories(defaultNodePath.indicesPath.resolve(index));
198+
}
199+
} else {
200+
indices = Collections.emptyList();
201+
}
202+
final Logger mock = mock(Logger.class);
203+
if (indexExists) {
204+
final IllegalStateException e = expectThrows(
205+
IllegalStateException.class,
206+
() -> Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock));
207+
final String message = String.format(
208+
Locale.ROOT,
209+
"detected index data in default.path.data [%s] where there should not be any; check the logs for details",
210+
defaultPathData);
211+
assertThat(e, hasToString(containsString(message)));
212+
verify(mock)
213+
.error("detected index data in default.path.data [{}] where there should not be any", defaultNodePath.indicesPath);
214+
for (final String index : indices) {
215+
verify(mock).info(
216+
"index folder [{}] in default.path.data [{}] must be moved to any of {}",
217+
index,
218+
defaultNodePath.indicesPath,
219+
Arrays.stream(nodeEnv.nodePaths()).map(np -> np.indicesPath).collect(Collectors.toList()));
220+
}
221+
verifyNoMoreInteractions(mock);
222+
} else {
223+
Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock);
224+
verifyNoMoreInteractions(mock);
225+
}
226+
}
227+
}
228+
229+
public void testDefaultPathDataNotSet() throws IOException {
230+
final Path zero = createTempDir().toAbsolutePath();
231+
final Path one = createTempDir().toAbsolutePath();
232+
final Settings settings = Settings.builder()
233+
.put("path.home", "/home")
234+
.put("path.data.0", zero)
235+
.put("path.data.1", one)
236+
.build();
237+
try (NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings))) {
238+
final Logger mock = mock(Logger.class);
239+
Node.checkForIndexDataInDefaultPathData(settings, nodeEnv, mock);
240+
verifyNoMoreInteractions(mock);
241+
}
242+
}
243+
168244
private static Settings.Builder baseSettings() {
169245
final Path tempDir = createTempDir();
170246
return Settings.builder()
171-
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
172-
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
173-
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
174-
.put("transport.type", "mock-socket-network")
175-
.put(Node.NODE_DATA_SETTING.getKey(), true);
247+
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
248+
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
249+
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
250+
.put("transport.type", "mock-socket-network")
251+
.put(Node.NODE_DATA_SETTING.getKey(), true);
176252
}
177253

254+
178255
}

0 commit comments

Comments
 (0)