All system indices are hidden indices (#79512) (#81388)

System indices should be hidden from users. Since they are already restricted
indices, a users that can't view restricted indices already can't see or access
them, but they should also be hidden for superusers or users that are otherwise
granted advanced privileges.

To the greatest degree possible, we apply hidden settings in the transport
layer, so that the system can create an index or alias that is set to visible,
for example, when operating in a mixed cluster mode. However, in the case of
aliases created by templates, we hide the alias in the service layer.

This change has broken a number of tests that were relaying unnecessarily on
wildcard searches. In general, the fix for these issues was to apply
expand_wildcards=open,hidden to the request.

* Force system indices to be hidden in IndexMetadata
* Hide system data streams
* Update feature migration tests
* ML datafeed config defaults to searching hidden indices
* Prevent unmanaged system indices from becoming visible
* Change validation in TransportUpdateSettingsAction
* Validate index creation settings in transport action
* Make sure system data stream backing indices are hidden
* Make sure transport request adds hidden index setting if missing
* Validate and set default for autocreated system indices
* Add some code to hide system aliases
* Hide system aliases in create index service
* Hide system aliases when adding them via alias endpoints
* Check system indices when simulating and validating templates
* Add known issue for reenabling tests

* Update docs/changelog/79512.yaml

Author: williamrandolph

docs/changelog/79512.yaml

@@ -0,0 +1,5 @@
+pr: 79512
+summary: All system indices are hidden indices
+area: Infra/Core
+type: enhancement
+issues: []

modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java

@@ -14,6 +14,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -218,6 +219,27 @@ public void testUpdateIndexSettings() throws IOException {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
+    public void testCannotCreateVisibleSystemIndex() {
+        Request request = request("PUT", "/" + indexName);
+        request.setJsonEntity("{\"settings\": {\"index.hidden\":\"false\"}}");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(
+            exception.getMessage(),
+            containsString("Cannot create system index [" + indexName + "] with [index.hidden] set to 'false'")
+        );
+    }
+
+    public void testCannotSetVisible() throws IOException {
+        Request putIndexRequest = request("PUT", "/" + indexName);
+        Response response = client().performRequest(putIndexRequest);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+
+        Request putSettingsRequest = request("PUT", "/" + indexName + "/_settings");
+        putSettingsRequest.setJsonEntity("{ \"index.hidden\" : false }");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(putSettingsRequest));
+        assertThat(exception.getMessage(), containsString("Cannot set [index.hidden] to 'false' on system indices: [" + indexName + "]"));
+    }
+
     public void testGetIndex() throws IOException {
         Request request = request("PUT", "/" + indexName);
         Response response = client().performRequest(request);

modules/reindex/build.gradle

@@ -159,6 +159,18 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
 tasks.named("yamlRestTestV7CompatTransform").configure { task ->
     task.skipTest("reindex/20_validation/reindex without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
     task.skipTest("update_by_query/20_validation/update_by_query without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
+
+    // these tests are all relying on a call to refresh all indices, when they could easily be changed
+    // in 7.x to call the specific index they want to refresh.
+    // See https://github.com/elastic/elasticsearch/issues/81188
+    task.skipTest("delete_by_query/70_throttle/Rethrottle to -1 which turns off throttling", "test relies on system index being non-hidden")
+    task.skipTest("delete_by_query/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("delete_by_query/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+    task.skipTest("reindex/80_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("reindex/80_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+    task.skipTest("update_by_query/70_slices/Multiple slices with rethrottle", "test relies on system index being non-hidden")
+    task.skipTest("update_by_query/70_slices/Multiple slices with wait_for_completion=false", "test relies on system index being non-hidden")
+
     task.addAllowedWarningRegex("\\[types removal\\].*")
 }
 

modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java

@@ -299,20 +299,22 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr
         String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old"));
         CreateIndexRequestBuilder createRequest = prepareCreate(indexName);
         createRequest.setWaitForActiveShards(ActiveShardCount.ALL);
-        if (descriptor.getSettings() != null) {
-            createRequest.setSettings(
-                Settings.builder()
-                    .put("index.version.created", Version.CURRENT)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .build()
-            );
-        } else {
+        if (SystemIndexDescriptor.DEFAULT_SETTINGS.equals(descriptor.getSettings())) {
+            // unmanaged
             createRequest.setSettings(
                 createSimpleSettings(
                     NEEDS_UPGRADE_VERSION,
                     descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE
                 )
             );
+        } else {
+            // managed
+            createRequest.setSettings(
+                Settings.builder()
+                    .put("index.version.created", Version.CURRENT)
+                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+                    .build()
+            );
         }
         if (descriptor.getMappings() == null) {
             createRequest.setMapping(createSimpleMapping(false, descriptor.isInternal()));

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml

@@ -124,8 +124,6 @@
         task_id: $task
 
   - do:
-      warnings:
-        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
 
   - do:

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml

@@ -155,7 +155,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -270,7 +271,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml

@@ -164,7 +164,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -282,7 +283,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml

@@ -147,7 +147,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -256,7 +257,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -45,6 +46,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
@@ -92,6 +94,17 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception {
         // And with a total wildcard
         assertDeprecationWarningOnAccess(randomFrom("*", "_all"), SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
 
+        // If we're not expanding wildcards, we don't get anything
+        {
+            Request searchRequest = new Request("GET", "/" + randomFrom("*", "_all") + randomFrom("/_count", "/_search"));
+            searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
+            searchRequest.addParameter("allow_no_indices", "false");
+
+            ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(searchRequest));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+            assertThat(exception.getMessage(), containsString("no such index"));
+        }
+
         // Try to index a doc directly
         {
             String expectedWarning = "this request accesses system indices: ["
@@ -115,6 +128,7 @@ private void assertDeprecationWarningOnAccess(String queryPattern, String warnin
         searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
         // Disallow no indices to cause an exception if this resolves to zero indices, so that we're sure it resolved the index
         searchRequest.addParameter("allow_no_indices", "false");
+        searchRequest.addParameter("expand_wildcards", "open,hidden");
         searchRequest.setOptions(expectWarnings(expectedWarning));
 
         Response response = getRestClient().performRequest(searchRequest);