[8.0] All system indices are hidden indices (#79512)

docs/changelog/79512.yaml

@@ -0,0 +1,5 @@
+pr: 79512
+summary: All system indices are hidden indices
+area: Infra/Core
+type: enhancement
+issues: []

modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java

@@ -14,6 +14,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -218,6 +219,27 @@ public void testUpdateIndexSettings() throws IOException {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
+    public void testCannotCreateVisibleSystemIndex() {
+        Request request = request("PUT", "/" + indexName);
+        request.setJsonEntity("{\"settings\": {\"index.hidden\":\"false\"}}");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(
+            exception.getMessage(),
+            containsString("Cannot create system index [" + indexName + "] with [index.hidden] set to 'false'")
+        );
+    }
+
+    public void testCannotSetVisible() throws IOException {
+        Request putIndexRequest = request("PUT", "/" + indexName);
+        Response response = client().performRequest(putIndexRequest);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+
+        Request putSettingsRequest = request("PUT", "/" + indexName + "/_settings");
+        putSettingsRequest.setJsonEntity("{ \"index.hidden\" : false }");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(putSettingsRequest));
+        assertThat(exception.getMessage(), containsString("Cannot set [index.hidden] to 'false' on system indices: [" + indexName + "]"));
+    }
+
     public void testGetIndex() throws IOException {
         Request request = request("PUT", "/" + indexName);
         Response response = client().performRequest(request);

modules/reindex/build.gradle

@@ -159,6 +159,18 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
 tasks.named("yamlRestTestV7CompatTransform").configure { task ->
     task.skipTest("reindex/20_validation/reindex without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
     task.skipTest("update_by_query/20_validation/update_by_query without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
+
+    // these tests are all relying on a call to refresh all indices, when they could easily be changed
+    // in 7.x to call the specific index they want to refresh.
+    // See https://github.com/elastic/elasticsearch/issues/81188
+    task.skipTest("delete_by_query/70_throttle/Rethrottle to -1 which turns off throttling", "test relies on system index being non-hidden")
+    task.skipTest("delete_by_query/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("delete_by_query/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+    task.skipTest("reindex/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("reindex/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+    task.skipTest("update_by_query/70_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("update_by_query/70_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+
     task.addAllowedWarningRegex("\\[types removal\\].*")
 }
 

modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java

@@ -299,20 +299,22 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr
         String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old"));
         CreateIndexRequestBuilder createRequest = prepareCreate(indexName);
         createRequest.setWaitForActiveShards(ActiveShardCount.ALL);
-        if (descriptor.getSettings() != null) {
-            createRequest.setSettings(
-                Settings.builder()
-                    .put("index.version.created", Version.CURRENT)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .build()
-            );
-        } else {
+        if (SystemIndexDescriptor.DEFAULT_SETTINGS.equals(descriptor.getSettings())) {
+            // unmanaged
             createRequest.setSettings(
                 createSimpleSettings(
                     NEEDS_UPGRADE_VERSION,
                     descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE
                 )
             );
+        } else {
+            // managed
+            createRequest.setSettings(
+                Settings.builder()
+                    .put("index.version.created", Version.CURRENT)
+                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+                    .build()
+            );
         }
         if (descriptor.getMappings() == null) {
             createRequest.setMapping(createSimpleMapping(false, descriptor.isInternal()));

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml

@@ -124,8 +124,6 @@
         task_id: $task
 
   - do:
-      warnings:
-        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
 
   - do:

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml

@@ -155,7 +155,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -270,7 +271,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml

@@ -164,7 +164,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -282,7 +283,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml

@@ -147,7 +147,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -256,7 +257,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -45,6 +46,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
@@ -92,6 +94,17 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception {
         // And with a total wildcard
         assertDeprecationWarningOnAccess(randomFrom("*", "_all"), SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
 
+        // If we're not expanding wildcards, we don't get anything
+        {
+            Request searchRequest = new Request("GET", "/" + randomFrom("*", "_all") + randomFrom("/_count", "/_search"));
+            searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
+            searchRequest.addParameter("allow_no_indices", "false");
+
+            ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(searchRequest));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+            assertThat(exception.getMessage(), containsString("no such index"));
+        }
+
         // Try to index a doc directly
         {
             String expectedWarning = "this request accesses system indices: ["
@@ -115,6 +128,7 @@ private void assertDeprecationWarningOnAccess(String queryPattern, String warnin
         searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
         // Disallow no indices to cause an exception if this resolves to zero indices, so that we're sure it resolved the index
         searchRequest.addParameter("allow_no_indices", "false");
+        searchRequest.addParameter("expand_wildcards", "open,hidden");
         searchRequest.setOptions(expectWarnings(expectedWarning));
 
         Response response = getRestClient().performRequest(searchRequest);