New setting to prevent automatically importing dangling indices

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -70,6 +70,7 @@
 import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.gateway.DanglingIndicesState;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.gateway.IncrementalClusterStateWriter;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -184,6 +185,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             BalancedShardsAllocator.THRESHOLD_SETTING,
             ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING,
             ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
+            DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING,
             EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING,
             EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING,
             FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING,

server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -55,19 +56,35 @@ public class DanglingIndicesState implements ClusterStateListener {
 
     private static final Logger logger = LogManager.getLogger(DanglingIndicesState.class);
 
+    public static final Setting<Boolean> AUTO_IMPORT_DANGLING_INDICES_SETTING = Setting.boolSetting(
+        "gateway.auto_import_dangling_indices",
+        true,
+        Setting.Property.NodeScope
+    );
+
     private final NodeEnvironment nodeEnv;
     private final MetaStateService metaStateService;
     private final LocalAllocateDangledIndices allocateDangledIndices;
 
     private final Map<Index, IndexMetaData> danglingIndices = ConcurrentCollections.newConcurrentMap();
 
+    private boolean allocateDanglingIndices;
+
     @Inject
     public DanglingIndicesState(NodeEnvironment nodeEnv, MetaStateService metaStateService,
                                 LocalAllocateDangledIndices allocateDangledIndices, ClusterService clusterService) {
         this.nodeEnv = nodeEnv;
         this.metaStateService = metaStateService;
         this.allocateDangledIndices = allocateDangledIndices;
-        clusterService.addListener(this);
+        this.allocateDanglingIndices = AUTO_IMPORT_DANGLING_INDICES_SETTING.get(clusterService.getSettings());
+
+        if (this.allocateDanglingIndices) {
+            clusterService.addListener(this);
+        }
+    }
+
+    public void setAllocateDanglingIndicesSetting(boolean allocateDanglingIndices) {
+        this.allocateDanglingIndices = allocateDanglingIndices;
     }
 
     /**
@@ -171,10 +188,15 @@ private IndexMetaData stripAliases(IndexMetaData indexMetaData) {
      * Allocates the provided list of the dangled indices by sending them to the master node
      * for allocation.
      */
-    private void allocateDanglingIndices() {
+    void allocateDanglingIndices() {
+        if (this.allocateDanglingIndices == false) {
+            return;
+        }
+
         if (danglingIndices.isEmpty()) {
             return;
         }
+
         try {
             allocateDangledIndices.allocateDangled(Collections.unmodifiableCollection(new ArrayList<>(danglingIndices.values())),
                 new ActionListener<>() {

server/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -35,16 +36,21 @@
 import java.nio.file.StandardCopyOption;
 import java.util.Map;
 
+import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING;
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class DanglingIndicesStateTests extends ESTestCase {
 
     private static Settings indexSettings = Settings.builder()
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .build();
+        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+        .build();
 
     public void testCleanupWhenEmpty() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
@@ -57,11 +63,11 @@ public void testCleanupWhenEmpty() throws Exception {
             assertTrue(danglingState.getDanglingIndices().isEmpty());
         }
     }
+
     public void testDanglingIndicesDiscovery() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
             MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
-
             assertTrue(danglingState.getDanglingIndices().isEmpty());
             MetaData metaData = MetaData.builder().build();
             final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
@@ -155,7 +161,6 @@ public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exceptio
             final IndexGraveyard graveyard = IndexGraveyard.builder().addTombstone(dangledIndex.getIndex()).build();
             final MetaData metaData = MetaData.builder().indexGraveyard(graveyard).build();
             assertThat(danglingState.findNewDanglingIndices(metaData).size(), equalTo(0));
-
         }
     }
 
@@ -181,7 +186,68 @@ public void testDanglingIndicesStripAliases() throws Exception {
         }
     }
 
+    public void testDanglingIndicesAreNotAllocatedWhenDisabled() throws Exception {
+        try (NodeEnvironment env = newNodeEnvironment()) {
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
+            LocalAllocateDangledIndices localAllocateDangledIndices = mock(LocalAllocateDangledIndices.class);
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService, localAllocateDangledIndices, false);
+
+            assertTrue(danglingState.getDanglingIndices().isEmpty());
+
+            MetaData metaData = MetaData.builder().build();
+
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
+
+            danglingState.findNewAndAddDanglingIndices(metaData);
+
+            danglingState.allocateDanglingIndices();
+
+            verify(localAllocateDangledIndices, never()).allocateDangled(any(), any());
+        }
+    }
+
+    public void testDanglingIndicesAreAllocatedWhenEnabled() throws Exception {
+        try (NodeEnvironment env = newNodeEnvironment()) {
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
+            LocalAllocateDangledIndices localAllocateDangledIndices = mock(LocalAllocateDangledIndices.class);
+            DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService, localAllocateDangledIndices, true);
+
+            assertTrue(danglingState.getDanglingIndices().isEmpty());
+
+            MetaData metaData = MetaData.builder().build();
+
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
+
+            danglingState.findNewAndAddDanglingIndices(metaData);
+
+            danglingState.allocateDanglingIndices();
+
+            verify(localAllocateDangledIndices).allocateDangled(any(), any());
+        }
+    }
+
     private DanglingIndicesState createDanglingIndicesState(NodeEnvironment env, MetaStateService metaStateService) {
-        return new DanglingIndicesState(env, metaStateService, null, mock(ClusterService.class));
+        return createDanglingIndicesState(env, metaStateService, null, true);
+    }
+
+    private DanglingIndicesState createDanglingIndicesState(
+        NodeEnvironment env,
+        MetaStateService metaStateService,
+        LocalAllocateDangledIndices indexAllocator,
+        boolean shouldAutoImport
+    ) {
+        final Settings allocateSettings = Settings.builder().put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), shouldAutoImport).build();
+
+        final ClusterService clusterServiceMock = mock(ClusterService.class);
+        when(clusterServiceMock.getClusterSettings()).thenReturn(
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        when(clusterServiceMock.getSettings()).thenReturn(allocateSettings);
+
+        return new DanglingIndicesState(env, metaStateService, indexAllocator, clusterServiceMock);
     }
 }

server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java

@@ -0,0 +1,148 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.recovery;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.InternalTestCluster;
+
+import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES;
+import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+@ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST)
+public class DanglingIndicesIT extends ESIntegTestCase {
+    private static final String INDEX_NAME = "test-idx-1";
+
+    private static final int MIN_DOC_COUNT = 500;
+    private static final int SHARD_COUNT = 1;
+    private static final int REPLICA_COUNT = 2;
+
+    private Settings buildSettings(boolean importDanglingIndices) {
+        return Settings.builder()
+            // Don't keep any indices in the graveyard, so that when we delete an index,
+            // it's definitely considered to be dangling.
+            .put(SETTING_MAX_TOMBSTONES.getKey(), 0)
+            .put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), importDanglingIndices)
+            .build();
+    }
+
+    /**
+     * Check that when dangling indices are discovered, then they are recovered into
+     * the cluster, so long as the recovery setting is enabled.
+     */
+    public void testDanglingIndicesAreRecoveredWhenSettingIsEnabled() throws Exception {
+        logger.info("--> starting cluster");
+        final Settings settings = buildSettings(true);
+        internalCluster().startNodes(3, settings);
+
+        // Create an index and distribute it across the 3 nodes
+        createAndPopulateIndex(INDEX_NAME, SHARD_COUNT, REPLICA_COUNT);
+        ensureGreen();
+
+        // This is so that when then node comes back up, we have a dangling index that can be recovered.
+        logger.info("--> restarted a random node and deleting the index while it's down");
+        internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                deleteIndex(INDEX_NAME);
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        ensureGreen();
+
+        assertTrue("Expected dangling index " + INDEX_NAME + " to be recovered", indexExists(INDEX_NAME));
+    }
+
+    /**
+     * Check that when dangling indices are discovered, then they are not recovered into
+     * the cluster when the recovery setting is disabled.
+     */
+    public void testDanglingIndicesAreNotRecoveredWhenSettingIsDisabled() throws Exception {
+        logger.info("--> starting cluster");
+        internalCluster().startNodes(3, buildSettings(false));
+
+        // Create an index and distribute it across the 3 nodes
+        createAndPopulateIndex(INDEX_NAME, SHARD_COUNT, REPLICA_COUNT);
+
+        // Create another index so that once we drop the first index, we
+        // can still assert that the cluster is green.
+        createAndPopulateIndex(INDEX_NAME + "-other", SHARD_COUNT, REPLICA_COUNT);
+
+        ensureGreen();
+
+        // This is so that when then node comes back up, we have a dangling index that could
+        // be recovered, but shouldn't be.
+        logger.info("--> restarted a random node and deleting the index while it's down");
+        internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                deleteIndex(INDEX_NAME);
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        ensureGreen();
+
+        assertFalse("Did not expect dangling index " + INDEX_NAME + " to be recovered", indexExists(INDEX_NAME));
+    }
+
+    private void createAndPopulateIndex(String name, int shardCount, int replicaCount) throws InterruptedException {
+        logger.info("--> creating test index: {}", name);
+        assertAcked(
+            prepareCreate(
+                name,
+                Settings.builder()
+                    .put("number_of_shards", shardCount)
+                    .put("number_of_replicas", replicaCount)
+                    .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
+            )
+        );
+        ensureGreen();
+
+        logger.info("--> indexing sample data");
+        final int numDocs = between(MIN_DOC_COUNT, 1000);
+        final IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+
+        for (int i = 0; i < numDocs; i++) {
+            docs[i] = client().prepareIndex(name)
+                .setSource("foo-int", randomInt(), "foo-string", randomAlphaOfLength(32), "foo-float", randomFloat());
+        }
+
+        indexRandom(true, docs);
+        flush();
+        assertThat(client().prepareSearch(name).setSize(0).get().getHits().getTotalHits().value, equalTo((long) numDocs));
+
+        client().admin().indices().prepareStats(name).execute().actionGet();
+    }
+
+    private void deleteIndex(String indexName) {
+        logger.info("--> deleting test index: {}", indexName);
+
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+    }
+}

server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java

@@ -142,7 +142,6 @@
 public class IndexRecoveryIT extends ESIntegTestCase {
 
     private static final String INDEX_NAME = "test-idx-1";
-    private static final String INDEX_TYPE = "test-type-1";
     private static final String REPO_NAME = "test-repo-1";
     private static final String SNAP_NAME = "test-snap-1";