[CI] MixedClusterClientYamlTestSuiteIT test {p0=field_caps/40_time_series/Get time series field caps with conflicts} failing

[kkrik-es]

Build scan:
https://gradle-enterprise.elastic.co/s/asvd4tktq6iim/tests/:qa:mixed-cluster:v8.9.1%23mixedClusterTest/org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT/test%20%7Bp0=field_caps%2F40_time_series%2FGet%20time%20series%20field%20caps%20with%20conflicts%7D
Reproduction line:

./gradlew ':qa:mixed-cluster:v8.9.1#mixedClusterTest' -Dtests.class="org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT" -Dtests.method="test {p0=field_caps/40_time_series/Get time series field caps with conflicts}" -Dtests.seed=64A0A6F67A7C3C07 -Dtests.bwc=true -Dtests.locale=ar-QA -Dtests.timezone=Africa/Kampala -Druntime.java=21

Applicable branches:
main

Reproduces locally?:
No

Failure history:
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT&tests.test=test%20%7Bp0%3Dfield_caps/40_time_series/Get%20time%20series%20field%20caps%20with%20conflicts%7D
Failure excerpt:

java.lang.AssertionError: Failure at [field_caps/40_time_series:181]: field [fields.metricset.keyword.time_series_dimension] has a true value but it shouldn't
Expected: ("" or a string equal to "false" ignoring case or "0")
     but: was "true"

  at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:582)
  at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:534)
  at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
  at java.lang.reflect.Method.invoke(Method.java:580)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
  at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
  at java.lang.Thread.run(Thread.java:1583)

  Caused by: java.lang.AssertionError: field [fields.metricset.keyword.time_series_dimension] has a true value but it shouldn't
  Expected: ("" or a string equal to "false" ignoring case or "0")
       but: was "true"

    at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:18)
    at org.junit.Assert.assertThat(Assert.java:956)
    at org.elasticsearch.test.rest.yaml.section.IsFalseAssertion.doAssert(IsFalseAssertion.java:48)
    at org.elasticsearch.test.rest.yaml.section.Assertion.execute(Assertion.java:65)
    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:562)
    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:534)
    at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
    at java.lang.reflect.Method.invoke(Method.java:580)
    at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
    at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
    at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
    at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
    at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
    at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
    at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
    at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
    at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
    at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
    at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
    at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
    at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
    at java.lang.Thread.run(Thread.java:1583)

[elasticsearchmachine]

Pinging @elastic/es-analytics-geo (Team:Analytics)

[kkrik-es]

The output also shows that there's a failure:

  | 1>       "failed_indices" : 1, 
  | 1>       "failures" : [ 
  | 1>         { 
  | 1>           "indices" : [ 
  | 1>             "tsdb_index2" 
  | 1>           ], 
  | 1>           "failure" : { 
  | 1>             "error" : {   
  | 1>               "root_cause" : [ 
  | 1>                 { 
  | 1>                   "type" : "node_not_connected_exception", 
  | 1>                   "reason" : "[v8.9.1-2][127.0.0.1:38403] Node not connected" 
  | 1>                 } 
  | 1>               ], 
  | 1>               "type" : "node_not_connected_exception", 
  | 1>               "reason" : "[v8.9.1-2][127.0.0.1:38403] Node not connected" 
  | 1>             } 
  | 1>           } 
  | 1>         } 
  | 1>       ]

This likely leads to the following delta in metricset:

Expected:

  1>         "metricset" : {
  1>           "keyword" : {
  1>             "type" : "keyword",
  1>             "metadata_field" : false,
  1>             "searchable" : true,
  1>             "aggregatable" : true,
  1>             "non_dimension_indices" : [
  1>               "tsdb_index2"
  1>             ]
  1>           }
  1>         },

Actual:

  | 1>         "metricset" : { 
  | 1>           "keyword" : { 
  | 1>             "type" : "keyword", 
  | 1>             "metadata_field" : false, 
  | 1>             "searchable" : true, 
  | 1>             "aggregatable" : true, 
  | 1>             "time_series_dimension" : true 
  | 1>           }  
  | 1>         },

It's interesting to see why we get this failure. Regardless, we also need an assert that there are no index failures.

[kkrik-es]

Well, turns out it's another instance of a test affected by the synthetic source change, #100562:

» [2023-10-16T04:18:49,353][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [v8.9.1-2] fatal error in thread [elasticsearch[v8.9.1-2][clusterApplierService#updateTask][T#1]], exiting java.lang.AssertionError: provided source [{"_doc":{"_data_stream_timestamp":{"enabled":true},"_source":{"mode":"synthetic"},"properties":{"@timestamp":{"type":"date"},"k8s":{"properties":{"pod":{"properties":{"availability_zone":{"type":"short","time_series_dimension":true},"ip":{"type":"ip","time_series_dimension":true},"name":{"type":"keyword"},"network":{"properties":{"latency":{"type":"double","time_series_metric":"gauge"},"packets_dropped":{"type":"long","time_series_metric":"gauge"},"rx":{"type":"integer"},"tx":{"type":"long","time_series_metric":"gauge"}}},"uid":{"type":"keyword","time_series_dimension":true}}}}},"metricset":{"type":"keyword"},"non_tsdb_field":{"type":"keyword"}}}}] differs from mapping [{"_doc":{"_data_stream_timestamp":{"enabled":true},"properties":{"@timestamp":{"type":"date"},"k8s":{"properties":{"pod":{"properties":{"availability_zone":{"type":"short","time_series_dimension":true},"ip":{"type":"ip","time_series_dimension":true},"name":{"type":"keyword"},"network":{"properties":{"latency":{"type":"double","time_series_metric":"gauge"},"packets_dropped":{"type":"long","time_series_metric":"gauge"},"rx":{"type":"integer"},"tx":{"type":"long","time_series_metric":"gauge"}}},"uid":{"type":"keyword","time_series_dimension":true}}}}},"metricset":{"type":"keyword"},"non_tsdb_field":{"type":"keyword"}}}}]
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.index.mapper.DocumentMapper.<init>(DocumentMapper.java:42)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.index.mapper.MapperService.newDocumentMapper(MapperService.java:376)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.index.mapper.MapperService.updateMapping(MapperService.java:285)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.index.IndexService.updateMapping(IndexService.java:744)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:494)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:251)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:536)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:522)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:495)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:426)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:154)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:217)
»  	at org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:183)
»  	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
»  	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
»  	at java.base/java.lang.Thread.run(Thread.java:1623)