Core: Backport java time date formatters (#31997)

* Date: Add DateFormatters class that uses java.time (#31856)

A newly added class called DateFormatters now contains java.time based
builders for dates, which also intends to be fully backwards compatible,
when the name based date formatters are picked. Also a new class named
CompoundDateTimeFormatter for being able to parse multiple different
formats has been added.

A duelling test class has been added that ensures the same dates when
parsing java or joda time formatted dates for the name based dates.

Note, that java.time and joda time are not fully backwards compatible,
which also means that old formats will currently not work with this
setup.

* Tests: Remove use of joda time in some tests (#31922)

This also extends the dateformatters test to ensure that the printers
are acting the same in java time and joda time.

* Tests: Fix SearchFieldsIT.testDocValueFields

This test produced different implementations of joda time classes,
depending on if the data was serialized or not (DateTime vs
MutableDateTime). This now uses a common base class to extract the
milliseconds from the data.

Closes #31992

Author: spinscale

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java

@@ -24,9 +24,10 @@
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -36,19 +37,21 @@
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.joda.time.DateTimeZone.UTC;
 
 public class DateProcessorTests extends ESTestCase {
+
     private TemplateScript.Factory templatize(Locale locale) {
         return new TestTemplateService.MockTemplateScript.Factory(locale.getLanguage());
     }
 
-    private TemplateScript.Factory templatize(DateTimeZone timezone) {
-        return new TestTemplateService.MockTemplateScript.Factory(timezone.getID());
+    private TemplateScript.Factory templatize(ZoneId timezone) {
+        // prevent writing "UTC" as string, as joda time does not parse it
+        String id = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId();
+        return new TestTemplateService.MockTemplateScript.Factory(id);
     }
     public void testJodaPattern() {
         DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
-            templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
+            templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
                 "date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "2010 12 06 11:05:15");
@@ -63,7 +66,7 @@ public void testJodaPatternMultipleFormats() {
         matchFormats.add("dd/MM/yyyy");
         matchFormats.add("dd-MM-yyyy");
         DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
-            templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
+            templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
                 "date_as_string", matchFormats, "date_as_date");
 
         Map<String, Object> document = new HashMap<>();
@@ -98,7 +101,7 @@ public void testJodaPatternMultipleFormats() {
     public void testInvalidJodaPattern() {
         try {
             DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),
-                templatize(UTC), templatize(randomLocale(random())),
+                templatize(ZoneOffset.UTC), templatize(randomLocale(random())),
                 "date_as_string", Collections.singletonList("invalid pattern"), "date_as_date");
             Map<String, Object> document = new HashMap<>();
             document.put("date_as_string", "2010");
@@ -112,7 +115,7 @@ public void testInvalidJodaPattern() {
 
     public void testJodaPatternLocale() {
             DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
-            templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ITALIAN),
+            templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN),
                 "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "2010 12 giugno");
@@ -123,18 +126,18 @@ public void testJodaPatternLocale() {
 
     public void testJodaPatternDefaultYear() {
         DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
-            templatize(DateTimeZone.forID("Europe/Amsterdam")), templatize(Locale.ENGLISH),
+            templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
             "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "12/06");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         dateProcessor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("date_as_date", String.class),
-            equalTo(DateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
+            equalTo(ZonedDateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
     }
 
     public void testTAI64N() {
-        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(DateTimeZone.forOffsetHours(2)),
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.ofHours(2)),
             templatize(randomLocale(random())),
                 "date_as_string", Collections.singletonList("TAI64N"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
@@ -146,8 +149,8 @@ public void testTAI64N() {
     }
 
     public void testUnixMs() {
-        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(UTC), templatize(randomLocale(random())),
-                "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC),
+            templatize(randomLocale(random())), "date_as_string", Collections.singletonList("UNIX_MS"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "1000500");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@@ -162,7 +165,7 @@ public void testUnixMs() {
     }
 
     public void testUnix() {
-        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(UTC),
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC),
             templatize(randomLocale(random())),
                 "date_as_string", Collections.singletonList("UNIX"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
@@ -186,7 +189,7 @@ public void testInvalidTimezone() {
 
     public void testInvalidLocale() {
         DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),
-            templatize(UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"),
+            templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Factory("invalid_locale"),
             "date_as_string", Collections.singletonList("yyyy"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "2010");

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -23,7 +23,6 @@
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -56,10 +55,11 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
@@ -1346,7 +1346,7 @@ public static Settings addHumanReadableSettings(Settings settings) {
         }
         Long creationDate = settings.getAsLong(SETTING_CREATION_DATE, null);
         if (creationDate != null) {
-            DateTime creationDateTime = new DateTime(creationDate, DateTimeZone.UTC);
+            ZonedDateTime creationDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(creationDate), ZoneOffset.UTC);
             builder.put(SETTING_CREATION_DATE_STRING, creationDateTime.toString());
         }
         return builder.build();

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -75,11 +75,10 @@
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 
 import java.io.UnsupportedEncodingException;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -390,7 +389,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 }
 
                 if (indexSettingsBuilder.get(SETTING_CREATION_DATE) == null) {
-                    indexSettingsBuilder.put(SETTING_CREATION_DATE, new DateTime(DateTimeZone.UTC).getMillis());
+                    indexSettingsBuilder.put(SETTING_CREATION_DATE, Instant.now().toEpochMilli());
                 }
                 indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
                 indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());

server/src/main/java/org/elasticsearch/common/time/CompoundDateTimeFormatter.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.time;
+
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
+
+/**
+ * wrapper class around java.time.DateTimeFormatter that supports multiple formats for easier parsing,
+ * and one specific format for printing
+ */
+public class CompoundDateTimeFormatter {
+
+    final DateTimeFormatter printer;
+    final DateTimeFormatter[] parsers;
+
+    CompoundDateTimeFormatter(DateTimeFormatter ... parsers) {
+        if (parsers.length == 0) {
+            throw new IllegalArgumentException("at least one date time formatter is required");
+        }
+        this.printer = parsers[0];
+        this.parsers = parsers;
+    }
+
+    public TemporalAccessor parse(String input) {
+        DateTimeParseException failure = null;
+        for (int i = 0; i < parsers.length; i++) {
+            try {
+                return parsers[i].parse(input);
+            } catch (DateTimeParseException e) {
+                if (failure == null) {
+                    failure = e;
+                } else {
+                    failure.addSuppressed(e);
+                }
+            }
+        }
+
+        // ensure that all parsers exceptions are returned instead of only the last one
+        throw failure;
+    }
+
+    public CompoundDateTimeFormatter withZone(ZoneId zoneId) {
+        final DateTimeFormatter[] parsersWithZone = new DateTimeFormatter[parsers.length];
+        for (int i = 0; i < parsers.length; i++) {
+            parsersWithZone[i] = parsers[i].withZone(zoneId);
+        }
+
+        return new CompoundDateTimeFormatter(parsersWithZone);
+    }
+
+    public String format(TemporalAccessor accessor) {
+        return printer.format(accessor);
+    }
+}