From 5ee2be927ee830a333a271558fbe241050720906 Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Mon, 23 Jan 2023 12:00:20 +0000
Subject: [PATCH] [ML] Making indices/aliases hidden takes a while after
 cluster startup (#93148)

When a cluster is upgraded from a version that didn't have hidden
indices/aliases to one that does, we make the relevant ML indices
and aliases hidden. However, the steps to do this are split over
5 actions that run sequentially, so the work might take a second or
two after the upgraded cluster starts up. This can cause the test
that asserts on the final state to fail if it runs very quickly
after the upgraded cluster startup. The solution is to retry the
test for a few seconds to give a chance for all the upgrade actions
to complete.

Relates #93062
---
 .../MlHiddenIndicesFullClusterRestartIT.java  | 61 +++++++++++--------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
index b4e3cbe0b1d7b..63c566b73d126 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
@@ -102,34 +102,41 @@ public void testMlIndicesBecomeHidden() throws Exception {
                 }
             }
         } else {
-            Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
-            Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
-
-            assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
-            for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
-                String indexName = e.getKey();
-                @SuppressWarnings("unchecked")
-                Map<String, Object> settings = (Map<String, Object>) e.getValue();
-                assertThat(settings, is(notNullValue()));
-                assertThat(
-                    "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
-                    XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
-                    is(equalTo("true"))
-                );
-            }
+            // The 5 operations in MlInitializationService.makeMlInternalIndicesHidden() run sequentially, so might
+            // not all be finished when this test runs. The desired state should exist within a few seconds of startup,
+            // hence the assertBusy().
+            assertBusy(() -> {
+                Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
+                Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
 
-            for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
-                List<String> indices = indexAndAlias.v1();
-                String alias = indexAndAlias.v2();
-                assertThat(
-                    indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
-                    indices.stream()
-                        .anyMatch(
-                            index -> Boolean.TRUE.equals(XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"))
-                        ),
-                    is(true)
-                );
-            }
+                assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
+                for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
+                    String indexName = e.getKey();
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> settings = (Map<String, Object>) e.getValue();
+                    assertThat(settings, is(notNullValue()));
+                    assertThat(
+                        "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
+                        XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
+                        is(equalTo("true"))
+                    );
+                }
+
+                for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
+                    List<String> indices = indexAndAlias.v1();
+                    String alias = indexAndAlias.v2();
+                    assertThat(
+                        indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
+                        indices.stream()
+                            .anyMatch(
+                                index -> Boolean.TRUE.equals(
+                                    XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden")
+                                )
+                            ),
+                        is(true)
+                    );
+                }
+            });
         }
     }