Super randomized tests for fetch fields API (#70278)

We've had a few bugs in the fields API where is doesn't behave like we'd
expect. Typically this happens because it isn't obvious what we expct. So
we'll try and use randomized testing to ferret out what we want. This adds
a test for most field types that asserts that `fields` works similarly
to `docvalues_fields`. We expect this to be true for most fields.

It does so by forcing all subclasses of `MapperTestCase` to define a
method that makes random values. It declares a few other hooks that
subclasses can override to further randomize the test.

We skip the test for a few field types that don't have doc values:
* `annotated_text`
* `completion`
* `search_as_you_type`
* `text`
We should come up with some way to test these without doc values, even
if it isn't as nice. But that is a problem for another time, I think.

We skip the test for a few more types just because I wanted to cut this
PR in half so we could get to reviewing it earlier. We'll get to those
in a follow up change.

I've filed a few bugs for things that are inconsistent with
`docvalues_fields`. Typically that means that we have to limit the
random values that we generate to those that *do* round trip properly.

Author: nik9000

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -136,4 +136,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
         assertEquals("[rank_feature] fields do not support indexing multiple values for the same field [foo.field] in the same document",
                 e.getCause().getMessage());
     }
+
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("Test implemented in a follow up", true);
+        return null;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -135,4 +135,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
         assertEquals("[rank_features] fields do not support indexing multiple values for the same rank feature [foo.field.bar] in " +
                 "the same document", e.getCause().getMessage());
     }
+
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("Test implemented in a follow up", true);
+        return null;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -271,4 +271,30 @@ public void testRejectIndexOptions() {
             containsString("Failed to parse mapping: unknown parameter [index_options] on mapper [field] of type [scaled_float]"));
     }
 
+    @Override
+    protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
+        // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field
+        b.field("type", "scaled_float").field("scaling_factor", randomDoubleBetween(0, Float.MAX_VALUE, true));
+    }
+
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        /*
+         * randomDoubleBetween will smear the random values out across a huge
+         * range of valid values.
+         */
+        double v = randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true);
+        switch (between(0, 3)) {
+            case 0:
+                return v;
+            case 1:
+                return (float) v;
+            case 2:
+                return Double.toString(v);
+            case 3:
+                return Float.toString((float) v);
+            default:
+                throw new IllegalArgumentException();
+        }
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

@@ -745,4 +745,10 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp
         assertThat(mapper, instanceOf(PrefixFieldMapper.class));
         return (PrefixFieldMapper) mapper;
     }
+
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("We don't have doc values or fielddata", true);
+        return null;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java

@@ -177,4 +177,20 @@ private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse
         return mapper.parse(request)
             .docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed"));
     }
+
+    @Override
+    protected String generateRandomInputValue(MappedFieldType ft) {
+        int words = between(1, 1000);
+        StringBuilder b = new StringBuilder(words * 5);
+        b.append(randomAlphaOfLength(4));
+        for (int w = 1; w < words; w++) {
+            b.append(' ').append(randomAlphaOfLength(4));
+        }
+        return b.toString();
+    }
+
+    @Override
+    protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
+        b.field("type", "token_count").field("analyzer", "standard");
+    }
 }

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -136,7 +136,7 @@ public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxD
             throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields.");
         }
 
-        public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() {
+        public static final DocValueFormat COLLATE_FORMAT = new DocValueFormat() {
             @Override
             public String getWriteableName() {
                 return "collate";

plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

@@ -10,6 +10,7 @@
 import com.ibm.icu.text.Collator;
 import com.ibm.icu.text.RawCollationKey;
 import com.ibm.icu.util.ULocale;
+
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
@@ -280,4 +281,13 @@ public void testUpdateIgnoreAbove() throws IOException {
         assertEquals(0, fields.length);
     }
 
+    @Override
+    protected String generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("docvalue_fields is broken", true);
+        // https://github.com/elastic/elasticsearch/issues/70276
+        /*
+         * docvalue_fields loads garbage bytes.
+         */
+        return null;
+    }
 }

plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.index.analysis.StandardTokenizerFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperTestCase;
@@ -552,4 +553,9 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() {
         }
     }
 
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("annotated_text doesn't have fielddata so we can't check against anything here.", true);
+        return null;
+    }
 }

plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin;
@@ -56,4 +57,9 @@ public void testDefaults() throws Exception {
         assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType());
     }
 
+    @Override
+    protected Object generateRandomInputValue(MappedFieldType ft) {
+        assumeFalse("Test implemented in a follow up", true);
+        return null;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/RangeType.java

@@ -546,6 +546,10 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu
         }
     };
 
+    public final String name;
+    private final NumberFieldMapper.NumberType numberType;
+    public final LengthType lengthType;
+
     RangeType(String name, LengthType lengthType) {
         this.name = name;
         this.numberType = null;
@@ -699,9 +703,9 @@ public final Mapper.TypeParser parser() {
         return new FieldMapper.TypeParser((n, c) -> new RangeFieldMapper.Builder(n, this, c.getSettings()));
     }
 
-    public final String name;
-    private final NumberFieldMapper.NumberType numberType;
-    public final LengthType lengthType;
+    NumberFieldMapper.NumberType numberType() {
+        return numberType;
+    }
 
     public enum LengthType {
         FIXED_4 {